@sequenceholdings/storage 0.1.0 → 0.1.1

package/README.md

@@ -16,6 +16,15 @@ const result = await workspace.uploads.put({
 ```
 
 `uploads.start` returns an upload ID before transfer. Keep that ID for
+recovery. Replaying an aborted session returns `409 upload_aborted` after
+authorization and request-identity checks. For a new-file upload, that terminal
+signal permits a new deterministic attempt key **and a new path** for the same
+content (the aborted attempt's node/history remains reserved). Replacements
+still require the file's current revision; the signal does not bypass that CAS.
+Generic `409 conflict`, expired
+but unreconciled sessions, permission failures and lost responses do **not**.
+Completed sessions replay their original file/version even after expiry. Keep
+the original source for
 `uploads.resume(id, originalSource)`. Source can be a Blob, Web ReadableStream or
 Node async iterable. The client retains fixed-size chunks, probes GCS offsets and
 retries transient failures. Supply `size` when known (up to 50 GiB); a stream without
@@ -26,7 +35,11 @@ to keep it and `resume` later. `put` generates an `idempotencyKey` unless you pa
 one and reuses it whenever it resends `start`, which `start` itself does after a
 timeout, a lost reply or a 5xx: the server replays a start that committed, so it
 never leaves an unnamed session holding one of your 20 pending uploads per
-workspace. Small Blobs can use
+workspace (100 per workspace). `uploads.startBatch(items)` reserves up to 20 of
+those sessions under one lock. `uploads.list()` returns the caller's live pending
+sessions without bearer URLs, and `uploads.reconcile({ abortIncomplete })`
+publishes sessions whose bytes already match or aborts the ones that never arrived.
+A `429 uploads_busy` names how many sessions are live and the cap. Small Blobs can use
 `{ method: 'simple' }`; sequential resumable is the default. Parallel multipart is
 not included. Do not log the session URL: it is a bearer credential.
 
@@ -83,8 +96,8 @@ const file = await storage.workspace(workspaceId).getNode(fileId)
 User mode is the default. `{ mode: 'service' }` explicitly selects the manifest's
 attached platform service account, which needs its own storage grants. Declare
 `service_account`, `capabilities.storage` and hosted network egress to your Atlas
-host and `storage.googleapis.com`. Delegation expires after 15 minutes; resume in a
-fresh invocation when necessary. Signed download leases expire after 60 seconds.
+host and `storage.googleapis.com`. Delegation expires shortly after the function's
+`timeout_seconds` (never sooner than 15 minutes); resume in a fresh invocation when necessary. Signed download leases expire after 60 seconds.
 
 For local integration use the normal local Atlas environment, `seq-studio storage`
 and `functions deploy -e local`. File references in ORM remain application-owned;

package/dist/client.d.ts

@@ -36,6 +36,15 @@ export declare function storageClient(transport: StorageTransport, { permissions
     capabilities: () => Promise<{
         canProvision: boolean;
         uploadMethods: string[];
+        pendingUploads: {
+            perPrincipal: number;
+            perWorkspace: number;
+        };
+        uploadBatchLimit: number;
+        importConcurrency: {
+            default: number;
+            max: number;
+        };
     }>;
     list: (options?: StorageListOptions) => Promise<{
         items: {
@@ -405,6 +414,73 @@ export declare function storageClient(transport: StorageTransport, { permissions
                 versionId: string | null;
                 maxBytes: number;
             }>;
+            startBatch: (items: StartStorageUpload[]) => Promise<{
+                items: {
+                    id: string;
+                    fileId: string;
+                    uploadUrl: string | null;
+                    expiresAt: string;
+                    state: "pending" | "complete" | "aborted";
+                    versionId: string | null;
+                    maxBytes: number;
+                }[];
+            }>;
+            list: () => Promise<{
+                items: {
+                    id: string;
+                    fileId: string;
+                    name: string;
+                    state: "pending" | "complete" | "aborted";
+                    expiresAt: string;
+                    maxBytes: number;
+                    expectedSize: number | null;
+                    versionId: string | null;
+                }[];
+                limits: {
+                    perPrincipal: number;
+                    perWorkspace: number;
+                };
+            }>;
+            reconcile: (input?: {
+                abortIncomplete?: boolean;
+            }) => Promise<{
+                completed: {
+                    id: string;
+                    fileId: string;
+                    name: string;
+                    state: "pending" | "complete" | "aborted";
+                    expiresAt: string;
+                    maxBytes: number;
+                    expectedSize: number | null;
+                    versionId: string | null;
+                    code: string;
+                    message: string;
+                }[];
+                aborted: {
+                    id: string;
+                    fileId: string;
+                    name: string;
+                    state: "pending" | "complete" | "aborted";
+                    expiresAt: string;
+                    maxBytes: number;
+                    expectedSize: number | null;
+                    versionId: string | null;
+                    code: string;
+                    message: string;
+                }[];
+                pending: {
+                    id: string;
+                    fileId: string;
+                    name: string;
+                    state: "pending" | "complete" | "aborted";
+                    expiresAt: string;
+                    maxBytes: number;
+                    expectedSize: number | null;
+                    versionId: string | null;
+                    code: string;
+                    message: string;
+                }[];
+            }>;
             resume(id: string, source: StorageUploadSource, options?: StorageTransferOptions & {
                 size?: number;
             }): Promise<{

package/dist/client.js

@@ -133,6 +133,15 @@ function storageClient(transport, { permissionsRetry = {}, } = {}) {
         capabilities: () => replayable('GET', '/capabilities', zod_1.z.object({
             canProvision: zod_1.z.boolean(),
             uploadMethods: zod_1.z.array(zod_1.z.string()),
+            pendingUploads: zod_1.z.object({
+                perPrincipal: zod_1.z.number().int(),
+                perWorkspace: zod_1.z.number().int(),
+            }),
+            uploadBatchLimit: zod_1.z.number().int(),
+            importConcurrency: zod_1.z.object({
+                default: zod_1.z.number().int(),
+                max: zod_1.z.number().int(),
+            }),
         })),
         list: (options = {}) => replayable('GET', `/workspaces${query({ ...options })}`, (0, contracts_js_1.storagePage)(contracts_js_1.workspaceSchema)),
         shared: (options = {}) => replayable('GET', `/shared${query({ ...options })}`, (0, contracts_js_1.storagePage)(contracts_js_1.nodeSchema)),
@@ -161,6 +170,21 @@ function storageClient(transport, { permissionsRetry = {}, } = {}) {
                 // Abort is idempotent: an aborted session stays aborted, and a completed
                 // one deterministically rejects, so a replay cannot change the outcome.
                 abort: (id) => replayable('DELETE', session(id), contracts_js_1.uploadSessionSchema),
+                startBatch: (items) => send({
+                    method: 'POST',
+                    path: `${root}/uploads/batch`,
+                    schema: zod_1.z.object({ items: zod_1.z.array(contracts_js_1.uploadSessionSchema) }),
+                    body: contracts_js_1.startUploadBatchSchema.parse({ items }),
+                    retryable: (error) => isRetryableConflict(error) || isUnacknowledged(error),
+                }),
+                list: () => replayable('GET', `${root}/uploads`, contracts_js_1.uploadListSchema),
+                reconcile: (input = {}) => send({
+                    method: 'POST',
+                    path: `${root}/uploads/reconcile`,
+                    schema: contracts_js_1.reconcileUploadsResultSchema,
+                    body: contracts_js_1.reconcileUploadsSchema.parse(input),
+                    retryable: isRetryableConflict,
+                }),
                 async resume(id, source, options = {}) {
                     const current = await upload.get(id);
                     if (current.state === 'complete')

package/dist/contracts.d.ts

@@ -149,6 +149,17 @@ export declare const updateNodeSchema: z.ZodObject<{
     parentId: z.ZodOptional<z.ZodNullable<z.ZodUUID>>;
     metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
 }, z.core.$strict>;
+/**
+ * Live pending upload sessions. Published on the capabilities and upload-list
+ * responses so callers can size a batch instead of discovering the cap from a
+ * 429. A batch registers at most one principal's cap in a single request.
+ */
+export declare const STORAGE_PENDING_UPLOADS_PER_PRINCIPAL = 20;
+export declare const STORAGE_PENDING_UPLOADS_PER_WORKSPACE = 100;
+export declare const STORAGE_UPLOAD_BATCH_LIMIT = 20;
+/** Parallel byte transfers `seq-studio storage import` opens. */
+export declare const STORAGE_IMPORT_CONCURRENCY_DEFAULT = 8;
+export declare const STORAGE_IMPORT_CONCURRENCY_MAX = 16;
 /** CRC32C in GCS's encoding: base64 of the big-endian uint32. */
 export declare const storageChecksum: z.ZodString;
 export declare const startUploadSchema: z.ZodObject<{
@@ -166,6 +177,19 @@ export declare const completeUploadSchema: z.ZodObject<{
     crc32c: z.ZodOptional<z.ZodString>;
 }, z.core.$strict>;
 export type CompleteStorageUpload = z.infer<typeof completeUploadSchema>;
+export declare const startUploadBatchSchema: z.ZodObject<{
+    items: z.ZodArray<z.ZodObject<{
+        parentId: z.ZodDefault<z.ZodNullable<z.ZodUUID>>;
+        path: z.ZodPipe<z.ZodString, z.ZodTransform<string, string>>;
+        fileId: z.ZodOptional<z.ZodUUID>;
+        expectedRevision: z.ZodOptional<z.ZodNumber>;
+        size: z.ZodOptional<z.ZodNumber>;
+        contentType: z.ZodDefault<z.ZodString>;
+        checksum: z.ZodOptional<z.ZodString>;
+        idempotencyKey: z.ZodString;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+export type StartStorageUploadBatch = z.infer<typeof startUploadBatchSchema>;
 export declare const uploadSessionSchema: z.ZodObject<{
     id: z.ZodUUID;
     fileId: z.ZodUUID;
@@ -180,6 +204,100 @@ export declare const uploadSessionSchema: z.ZodObject<{
     maxBytes: z.ZodNumber;
 }, z.core.$strip>;
 export type StorageUploadSession = z.infer<typeof uploadSessionSchema>;
+/** A pending session without its bearer upload URL. */
+export declare const uploadListItemSchema: z.ZodObject<{
+    id: z.ZodUUID;
+    fileId: z.ZodUUID;
+    name: z.ZodString;
+    state: z.ZodEnum<{
+        pending: "pending";
+        complete: "complete";
+        aborted: "aborted";
+    }>;
+    expiresAt: z.ZodString;
+    maxBytes: z.ZodNumber;
+    expectedSize: z.ZodNullable<z.ZodNumber>;
+    versionId: z.ZodNullable<z.ZodUUID>;
+}, z.core.$strip>;
+export type StorageUploadListItem = z.infer<typeof uploadListItemSchema>;
+export declare const uploadLimitsSchema: z.ZodObject<{
+    perPrincipal: z.ZodNumber;
+    perWorkspace: z.ZodNumber;
+}, z.core.$strip>;
+export declare const uploadListSchema: z.ZodObject<{
+    items: z.ZodArray<z.ZodObject<{
+        id: z.ZodUUID;
+        fileId: z.ZodUUID;
+        name: z.ZodString;
+        state: z.ZodEnum<{
+            pending: "pending";
+            complete: "complete";
+            aborted: "aborted";
+        }>;
+        expiresAt: z.ZodString;
+        maxBytes: z.ZodNumber;
+        expectedSize: z.ZodNullable<z.ZodNumber>;
+        versionId: z.ZodNullable<z.ZodUUID>;
+    }, z.core.$strip>>;
+    limits: z.ZodObject<{
+        perPrincipal: z.ZodNumber;
+        perWorkspace: z.ZodNumber;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+export declare const reconcileUploadsSchema: z.ZodObject<{
+    abortIncomplete: z.ZodDefault<z.ZodBoolean>;
+}, z.core.$strict>;
+export declare const reconcileUploadsResultSchema: z.ZodObject<{
+    completed: z.ZodArray<z.ZodObject<{
+        id: z.ZodUUID;
+        fileId: z.ZodUUID;
+        name: z.ZodString;
+        state: z.ZodEnum<{
+            pending: "pending";
+            complete: "complete";
+            aborted: "aborted";
+        }>;
+        expiresAt: z.ZodString;
+        maxBytes: z.ZodNumber;
+        expectedSize: z.ZodNullable<z.ZodNumber>;
+        versionId: z.ZodNullable<z.ZodUUID>;
+        code: z.ZodString;
+        message: z.ZodString;
+    }, z.core.$strip>>;
+    aborted: z.ZodArray<z.ZodObject<{
+        id: z.ZodUUID;
+        fileId: z.ZodUUID;
+        name: z.ZodString;
+        state: z.ZodEnum<{
+            pending: "pending";
+            complete: "complete";
+            aborted: "aborted";
+        }>;
+        expiresAt: z.ZodString;
+        maxBytes: z.ZodNumber;
+        expectedSize: z.ZodNullable<z.ZodNumber>;
+        versionId: z.ZodNullable<z.ZodUUID>;
+        code: z.ZodString;
+        message: z.ZodString;
+    }, z.core.$strip>>;
+    pending: z.ZodArray<z.ZodObject<{
+        id: z.ZodUUID;
+        fileId: z.ZodUUID;
+        name: z.ZodString;
+        state: z.ZodEnum<{
+            pending: "pending";
+            complete: "complete";
+            aborted: "aborted";
+        }>;
+        expiresAt: z.ZodString;
+        maxBytes: z.ZodNumber;
+        expectedSize: z.ZodNullable<z.ZodNumber>;
+        versionId: z.ZodNullable<z.ZodUUID>;
+        code: z.ZodString;
+        message: z.ZodString;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export type ReconcileStorageUploads = z.infer<typeof reconcileUploadsSchema>;
 export declare const storagePrincipalSchema: z.ZodObject<{
     type: z.ZodEnum<{
         user: "user";

package/dist/contracts.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.storageErrorSchema = exports.storageAccessSchema = exports.updateStorageAccessSchema = exports.storagePrincipalSchema = exports.uploadSessionSchema = exports.completeUploadSchema = exports.startUploadSchema = exports.storageChecksum = exports.updateNodeSchema = exports.createFolderSchema = exports.listStorageSchema = exports.storagePage = exports.listStorageOperationsSchema = exports.storageOperationScope = exports.operationSchema = exports.versionSchema = exports.nodeSchema = exports.workspaceSchema = exports.workspaceManifestSchema = exports.storageMetadata = exports.storagePath = exports.storageName = exports.storageLifecycle = exports.storageRole = exports.storageId = void 0;
+exports.storageErrorSchema = exports.storageAccessSchema = exports.updateStorageAccessSchema = exports.storagePrincipalSchema = exports.reconcileUploadsResultSchema = exports.reconcileUploadsSchema = exports.uploadListSchema = exports.uploadLimitsSchema = exports.uploadListItemSchema = exports.uploadSessionSchema = exports.startUploadBatchSchema = exports.completeUploadSchema = exports.startUploadSchema = exports.storageChecksum = exports.STORAGE_IMPORT_CONCURRENCY_MAX = exports.STORAGE_IMPORT_CONCURRENCY_DEFAULT = exports.STORAGE_UPLOAD_BATCH_LIMIT = exports.STORAGE_PENDING_UPLOADS_PER_WORKSPACE = exports.STORAGE_PENDING_UPLOADS_PER_PRINCIPAL = exports.updateNodeSchema = exports.createFolderSchema = exports.listStorageSchema = exports.storagePage = exports.listStorageOperationsSchema = exports.storageOperationScope = exports.operationSchema = exports.versionSchema = exports.nodeSchema = exports.workspaceSchema = exports.workspaceManifestSchema = exports.storageMetadata = exports.storagePath = exports.storageName = exports.storageLifecycle = exports.storageRole = exports.storageId = void 0;
 const zod_1 = require("zod");
 const paths_js_1 = require("./paths.js");
 exports.storageId = zod_1.z.uuid();
@@ -124,6 +124,17 @@ exports.updateNodeSchema = zod_1.z.strictObject({
     parentId: exports.storageId.nullable().optional(),
     metadata: exports.storageMetadata.optional(),
 });
+/**
+ * Live pending upload sessions. Published on the capabilities and upload-list
+ * responses so callers can size a batch instead of discovering the cap from a
+ * 429. A batch registers at most one principal's cap in a single request.
+ */
+exports.STORAGE_PENDING_UPLOADS_PER_PRINCIPAL = 20;
+exports.STORAGE_PENDING_UPLOADS_PER_WORKSPACE = 100;
+exports.STORAGE_UPLOAD_BATCH_LIMIT = exports.STORAGE_PENDING_UPLOADS_PER_PRINCIPAL;
+/** Parallel byte transfers `seq-studio storage import` opens. */
+exports.STORAGE_IMPORT_CONCURRENCY_DEFAULT = 8;
+exports.STORAGE_IMPORT_CONCURRENCY_MAX = 16;
 /** CRC32C in GCS's encoding: base64 of the big-endian uint32. */
 exports.storageChecksum = zod_1.z.string().regex(/^[A-Za-z0-9+/]{6}==$/);
 exports.startUploadSchema = zod_1.z
@@ -153,6 +164,9 @@ exports.completeUploadSchema = zod_1.z.strictObject({
      */
     crc32c: exports.storageChecksum.optional(),
 });
+exports.startUploadBatchSchema = zod_1.z.strictObject({
+    items: zod_1.z.array(exports.startUploadSchema).min(1).max(exports.STORAGE_UPLOAD_BATCH_LIMIT),
+});
 exports.uploadSessionSchema = zod_1.z.object({
     id: exports.storageId,
     fileId: exports.storageId,
@@ -162,6 +176,38 @@ exports.uploadSessionSchema = zod_1.z.object({
     versionId: exports.storageId.nullable(),
     maxBytes: zod_1.z.number().int(),
 });
+/** A pending session without its bearer upload URL. */
+exports.uploadListItemSchema = zod_1.z.object({
+    id: exports.storageId,
+    fileId: exports.storageId,
+    name: zod_1.z.string(),
+    state: zod_1.z.enum(['pending', 'complete', 'aborted']),
+    expiresAt: zod_1.z.string(),
+    maxBytes: zod_1.z.number().int(),
+    expectedSize: zod_1.z.number().int().nullable(),
+    versionId: exports.storageId.nullable(),
+});
+exports.uploadLimitsSchema = zod_1.z.object({
+    perPrincipal: zod_1.z.number().int(),
+    perWorkspace: zod_1.z.number().int(),
+});
+exports.uploadListSchema = zod_1.z.object({
+    items: zod_1.z.array(exports.uploadListItemSchema),
+    limits: exports.uploadLimitsSchema,
+});
+exports.reconcileUploadsSchema = zod_1.z.strictObject({
+    /** Abort sessions whose bytes never arrived. Terminal mismatches are already released. */
+    abortIncomplete: zod_1.z.boolean().default(false),
+});
+const reconcileEntrySchema = exports.uploadListItemSchema.extend({
+    code: zod_1.z.string(),
+    message: zod_1.z.string(),
+});
+exports.reconcileUploadsResultSchema = zod_1.z.object({
+    completed: zod_1.z.array(reconcileEntrySchema),
+    aborted: zod_1.z.array(reconcileEntrySchema),
+    pending: zod_1.z.array(reconcileEntrySchema),
+});
 exports.storagePrincipalSchema = zod_1.z.strictObject({
     type: zod_1.z.enum(['user', 'group', 'organization', 'service_account']),
     id: zod_1.z.string().min(1).max(255),

package/dist/transfer.js

@@ -128,9 +128,8 @@ async function transferStorageUpload({ uploadUrl, source, size, maxBytes, ...opt
     const url = checkedUploadUrl(uploadUrl);
     const fetcher = options.fetch ?? globalThis.fetch;
     const total = source instanceof Blob ? source.size : size;
-    if (total !== undefined &&
-        (!Number.isSafeInteger(total) || total < 0 || total > maxBytes))
-        throw new StorageTransferError('Upload exceeds its reserved size');
+    if (total !== undefined && (!Number.isSafeInteger(total) || total < 0 || total > maxBytes))
+        throw new StorageTransferError(`Upload source is ${total} bytes, above the ${maxBytes}-byte reservation`);
     let chunkBytes = options.chunkBytes ?? 8 * 1024 * 1024;
     if (chunkBytes < 262144 ||
         chunkBytes > 32 * 1024 * 1024 ||
@@ -181,7 +180,7 @@ async function transferStorageUpload({ uploadUrl, source, size, maxBytes, ...opt
     }
     let offset = initial.offset;
     if (offset > maxBytes || (total !== undefined && offset > total))
-        throw new StorageTransferError('Upload offset exceeds the source size');
+        throw new StorageTransferError(`Upload session offset is ${offset} bytes, past the source${total === undefined ? '' : ` of ${total} bytes`} and the ${maxBytes}-byte reservation`);
     options.onProgress?.(offset);
     const hash = new crc32c_js_1.Crc32c();
     const iterable = chunks(source, chunkBytes, offset, hash, options.signal);
@@ -197,7 +196,7 @@ async function transferStorageUpload({ uploadUrl, source, size, maxBytes, ...opt
                 throw new StorageTransferError(`Upload stream exceeds its ${maxBytes}-byte reservation; declare \`size\` when starting the upload to send a larger file`);
             if (end > maxBytes ||
                 (total !== undefined && (end > total || (last && end !== total))))
-                throw new StorageTransferError('Upload source does not match its reserved size');
+                throw new StorageTransferError(`Upload source does not match its reserved size: session offset ${offset} bytes, read ${end} bytes, reservation ${maxBytes} bytes${total === undefined ? '' : `, declared ${total} bytes`}`);
             const length = total ?? (last ? end : '*');
             let acknowledged = offset;
             for (let attempt = 0;; attempt++) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sequenceholdings/storage",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Atlas workspace storage contracts and streaming client",
   "license": "UNLICENSED",
   "type": "commonjs",