@omelhorsite/sdk 0.1.0

package/dist/types/resources/storage/upload.d.ts

@@ -0,0 +1,459 @@
+/**
+ * The direct-upload driver.
+ *
+ * Bytes never pass through Rails. The flow the backend implements, and the one
+ * this module follows exactly:
+ *
+ * 1. `POST /fs_nodes/batch_upload_urls` with a manifest of up to 100 files
+ *    (`client_id`, `name`, `size`, and optionally `relative_path`,
+ *    `content_type`, `checksum`). ONE request does directory resolution, one
+ *    quota reservation under a single lock, node creation and plan minting.
+ * 2. Per file the server answers a {@link UploadPlan}:
+ *    - `strategy: "direct"` below 32 MiB: PUT the whole body to a presigned URL
+ *      with the exact headers returned, then remember `blob_signed_id`;
+ *    - `strategy: "multipart"` at or above 32 MiB: drive
+ *      `/fs_nodes/:id/multipart/{start,part_urls,complete}` in 32 MiB parts.
+ * 3. `POST /fs_nodes/batch_attach_blobs` binds the direct-tier blobs to their
+ *    nodes. Multipart finalises itself at `multipart/complete`.
+ *
+ * Two rules that are easy to get wrong and expensive to get wrong:
+ *
+ * - The direct tier REQUIRES a base64 MD5 `checksum`. It is bound into the
+ *   presigned signature as `Content-MD5`, so a wrong or missing one makes the
+ *   PUT fail at the object store with a signature error that says nothing
+ *   about checksums. WebCrypto has no MD5, so {@link md5Base64} carries a
+ *   self-contained implementation - see the note there.
+ * - The presigned PUTs go to the object store, NOT to the API. They must be
+ *   sent with the injected fetch but WITHOUT the `Authorization` header, or
+ *   MinIO rejects the request for having two authentication schemes.
+ *
+ * A node whose bytes never landed is adopted on the next attempt rather than
+ * failing, so a torn batch is simply retried with the same manifest.
+ *
+ * Multipart is not an optimisation. The object store sits behind Cloudflare on
+ * a plan that caps a request body at roughly 100 MB, so anything larger has no
+ * other way in.
+ */
+import { Resource, type ApiClient } from "../../http";
+import { type FetchLike, type FileInput, type Id, type OperationOptions, type ProgressCallback, type RequestOptions } from "../../types";
+import type { FsNode } from "../storage";
+/** Files at or above this size take the multipart path. Mirrors the backend. */
+export declare const MULTIPART_THRESHOLD: number;
+/**
+ * Part size for the multipart path. Mirrors the backend, but it is only a
+ * fallback: the real part size is whatever `multipart/start` answered, and that
+ * is the number the driver slices with.
+ */
+export declare const MULTIPART_PART_SIZE: number;
+/** Maximum files in one `batch_upload_urls` call. Mirrors the backend. */
+export declare const MAX_BATCH = 100;
+/**
+ * Files sent in one `batch_upload_urls` call by default.
+ *
+ * Deliberately half of {@link MAX_BATCH}: a batch is one quota reservation
+ * under one row lock, and a smaller batch narrows the window in which a torn
+ * run leaves reserved-but-unused bytes behind.
+ */
+export declare const DEFAULT_BATCH_SIZE = 50;
+/** Maximum part URLs requested in one call. Mirrors the backend. */
+export declare const MAX_PART_URLS = 100;
+/** Parts one multipart upload may have. Mirrors the backend's `MAX_PARTS`. */
+export declare const MAX_PARTS = 10000;
+/** Part URLs asked for in one round trip. Below {@link MAX_PART_URLS} on purpose. */
+export declare const PART_URL_WINDOW = 32;
+/** Parallel presigned PUTs in flight by default. The store is a Pi behind Cloudflare. */
+export declare const DEFAULT_UPLOAD_CONCURRENCY = 4;
+/**
+ * The `fs_upload/authed` throttle: `batch_upload_urls`, `batch_attach_blobs`
+ * and every `multipart/*` call share 300 requests a minute, keyed by session.
+ * {@link UploadManager} paces itself against this so a large run degrades into
+ * waiting rather than into a wall of 429s.
+ */
+export declare const FS_UPLOAD_RATE_LIMIT = 300;
+/**
+ * The `fs_bulk_job/authed` throttle: `copy`, `create_directories`,
+ * `empty_trash` and `move_to_trash` share TWELVE requests a minute. It is the
+ * tightest limit in the API and the easiest one to trip by looping.
+ */
+export declare const FS_BULK_JOB_RATE_LIMIT = 12;
+/** One entry of the manifest sent to `batch_upload_urls`. */
+export interface UploadManifestEntry {
+    /** Caller-chosen correlation key. The response is matched back on this. */
+    readonly client_id: string;
+    readonly name: string;
+    readonly size: number;
+    /**
+     * Path INCLUDING the filename, e.g. `"fotos/2024/praia.jpg"`. Everything
+     * before the last segment becomes directories. A `..` segment is rejected.
+     */
+    readonly relative_path?: string;
+    readonly content_type?: string;
+    /** Base64 MD5 of the whole file. Required below {@link MULTIPART_THRESHOLD}. */
+    readonly checksum?: string;
+}
+/** The presigned PUT for a small file. */
+export interface DirectUploadTarget {
+    readonly url: string;
+    /** Send these verbatim. They include the `Content-MD5` the signature covers. */
+    readonly headers: Record<string, string>;
+    /** Hand this back to `batch_attach_blobs` once the PUT succeeds. */
+    readonly blob_signed_id: string;
+}
+/** What the server decided for one manifest entry. */
+export interface UploadPlan {
+    readonly client_id: string;
+    readonly fs_node_id: Id;
+    readonly strategy: "direct" | "multipart";
+    /** Present only for `strategy: "direct"`. */
+    readonly upload?: DirectUploadTarget;
+}
+/**
+ * One entry the server refused, with a machine-readable reason.
+ *
+ * The server names the entry by `client_id` only - it never echoes the
+ * filename back - so correlate on that and look the name up yourself.
+ */
+export interface UploadPlanError {
+    readonly client_id: string;
+    /** `"quota_exceeded"`, `"invalid"`, `"mint_failed"`. */
+    readonly code: string;
+    readonly message: string;
+}
+/** The answer to `POST /fs_nodes/batch_upload_urls`. */
+export interface UploadBatch {
+    readonly parent_id: Id;
+    /** Directories resolved or created while walking the relative paths. */
+    readonly directories: Array<{
+        readonly path: string;
+        readonly id: Id;
+    }>;
+    readonly results: UploadPlan[];
+    readonly errors: UploadPlanError[];
+}
+/** One entry of the answer to `POST /fs_nodes/batch_attach_blobs`. */
+export interface AttachBlobResult {
+    readonly fs_node_id: Id;
+    readonly attached: boolean;
+    /** `"not_found"`, `"already_attached"`, `"blob_not_found"`, `"attach_failed"`. */
+    readonly code?: string;
+    readonly message?: string;
+}
+/** The answer to `POST /fs_nodes/:id/multipart/start`. */
+export interface MultipartSession {
+    /** Signed `(key, upload_id)` pair. The server keeps no upload state. */
+    readonly upload_token: string;
+    readonly part_size: number;
+    readonly part_count: number;
+}
+/** A presigned PUT for one part. */
+export interface MultipartPartUrl {
+    readonly part_number: number;
+    readonly url: string;
+}
+/** A part that has landed, identified by the ETag the store returned. */
+export interface MultipartPart {
+    readonly part_number: number;
+    readonly etag: string;
+}
+/** The answer to `POST /fs_nodes/:id/multipart/complete`. */
+export interface MultipartCompletion {
+    readonly attached: boolean;
+    /** Size the store measured. Authoritative: the declared size is reconciled to it. */
+    readonly byte_size: number;
+}
+/**
+ * Computes the base64 MD5 of a blob.
+ *
+ * Supply one to replace the built-in {@link md5Base64} with a faster native or
+ * WASM digest, on a host that has one.
+ */
+export type Md5Base64Fn = (data: Blob) => Promise<string>;
+/** Arguments for {@link UploadManager.upload}. */
+export interface UploadInput {
+    /** Directory to upload into. */
+    readonly parentId: Id;
+    readonly files: FileInput[];
+    /**
+     * Relative path per file, parallel to `files`, when uploading a folder. Use
+     * {@link FileInput.filename} alone for a flat upload.
+     *
+     * The path INCLUDES the filename (`"fotos/2024/praia.jpg"`); everything
+     * before the last segment is created as directories in the same call, so a
+     * folder upload costs no extra round trips. A `..` segment is a 400.
+     */
+    readonly relativePaths?: string[];
+    /** Parallel PUTs in flight. Keep it modest: the store is a Pi behind Cloudflare. */
+    readonly concurrency?: number;
+    /** Files per `batch_upload_urls` call. Clamped to {@link MAX_BATCH}. */
+    readonly batchSize?: number;
+    /**
+     * Read the finished nodes back with one extra listing per batch. On by
+     * default, and worth it: a normal listing hides nodes whose bytes never
+     * landed, so a node coming back at all is proof the blob is bound.
+     */
+    readonly verify?: boolean;
+    /** Replacement for the built-in MD5. See {@link Md5Base64Fn}. */
+    readonly md5?: Md5Base64Fn;
+}
+/** What one file's upload ended as. */
+export interface UploadResult {
+    readonly client_id: string;
+    /** Name that was uploaded, echoed back so a caller can report on it. */
+    readonly filename: string;
+    /** The node the bytes belong to. `null` when the file never got a plan. */
+    readonly fs_node_id: Id | null;
+    /**
+     * The finished node, when it was read back (see {@link UploadInput.verify}).
+     * `null` when verification was off or the node did not come back.
+     */
+    readonly node: FsNode | null;
+    /** Set when this file failed while the rest of the batch succeeded. */
+    readonly error?: UploadPlanError;
+}
+/**
+ * A sliding-window pacer for one throttle bucket.
+ *
+ * The API throttles by session, not by connection, so parallelism inside the
+ * SDK is exactly what trips it. Requests wait their turn here instead of
+ * racing into a 429 and paying the server's `Retry-After` afterwards.
+ *
+ * Isolate-safe: `Date.now` and `setTimeout`, nothing else. Per instance, so two
+ * clients in the same isolate do not share (and must not share) a window.
+ */
+export declare class StorageRateGate {
+    private readonly limit;
+    private readonly windowMs;
+    private readonly headroom;
+    private readonly hits;
+    /**
+     * @param limit Requests allowed inside the window.
+     * @param windowMs Length of the window in milliseconds.
+     * @param headroom Fraction of the limit actually used, so the SDK is never
+     *   the request that trips the bucket. `0.9` spends 270 of 300.
+     */
+    constructor(limit: number, windowMs?: number, headroom?: number);
+    /** Resolves when another request may be sent. */
+    wait(signal?: AbortSignal): Promise<void>;
+}
+/** Constructor options for {@link UploadManager}. */
+export interface UploadManagerOptions {
+    /**
+     * The fetch used for the presigned PUTs.
+     *
+     * Defaults to the one the {@link ApiClient} was built with, so bytes travel
+     * on the same transport as everything else. Pass one only to send the object
+     * store's traffic somewhere different from the API's.
+     */
+    readonly fetch?: FetchLike;
+    /** Replacement for the built-in MD5. See {@link Md5Base64Fn}. */
+    readonly md5?: Md5Base64Fn;
+}
+/**
+ * Drives presigned uploads into the virtual filesystem.
+ *
+ * Reached as `oms.storage.uploads`, and wrapped by `oms.storage.upload()` for
+ * the common case. Instantiate it directly only when you need the low-level
+ * steps, for instance to resume a torn multipart.
+ */
+export declare class UploadManager extends Resource {
+    /** Paces every control-plane call against the `fs_upload` throttle. */
+    readonly gate: StorageRateGate;
+    private readonly transport;
+    private readonly md5;
+    constructor(http: ApiClient, options?: UploadManagerOptions);
+    /**
+     * Uploads files end to end: manifest, presign, transfer, bind, verify.
+     *
+     * Splits into batches of {@link UploadInput.batchSize}, picks the strategy
+     * per file from the server's plan, and reports bytes moved through
+     * `onProgress`.
+     *
+     * Per-file failures do NOT abort the run: they come back as
+     * {@link UploadResult.error} with the rest of the batch intact, exactly as
+     * the server reports them. Only a structural failure (bad parent, empty or
+     * oversized batch) throws.
+     *
+     * Progress granularity is a whole direct PUT or a whole multipart part, not
+     * a byte counter. `fetch` has no upload-progress event and a streamed request
+     * body is neither universally supported nor usable against a presigned PUT,
+     * which needs a `Content-Length`.
+     *
+     * Every multipart session this call opened is aborted before the error
+     * leaves, so a torn run does not strand parts (and reserved quota) on the
+     * store.
+     */
+    upload(input: UploadInput, options?: OperationOptions): Promise<UploadResult[]>;
+    /**
+     * `POST /fs_nodes/batch_upload_urls` - step 1 on its own.
+     *
+     * One request buys directory resolution, a single quota reservation under one
+     * lock, node creation and plan minting for the whole manifest. Per-file
+     * rejections come back in `errors` with a 200; only a structural problem
+     * (invalid parent, empty batch, over {@link MAX_BATCH}, a `..` segment)
+     * is a 400.
+     *
+     * @throws {OmsApiError} 400 on a structural problem, 404 when the parent is
+     *   not a directory the caller may write to.
+     */
+    createBatch(input: {
+        parentId: Id;
+        files: UploadManifestEntry[];
+    }, options?: RequestOptions): Promise<UploadBatch>;
+    /**
+     * PUTs one whole body to a presigned URL.
+     *
+     * Sends the plan's headers verbatim and NO `Authorization`: this request goes
+     * to the object store, not to the API. MinIO refuses a request that carries
+     * both a presigned signature and a bearer header.
+     *
+     * Retries a network fault or a 5xx from the store. A 4xx is never retried: an
+     * expired signature or a checksum mismatch is deterministic and a retry only
+     * costs the bytes again.
+     *
+     * @returns The ETag the store reported, or `null` when it was not readable -
+     *   which happens in a browser whose bucket CORS policy does not expose
+     *   `ETag`. The direct tier does not need it; multipart does.
+     */
+    putDirect(target: DirectUploadTarget, body: Blob, options?: RequestOptions & {
+        onProgress?: ProgressCallback;
+    }): Promise<string | null>;
+    /**
+     * `POST /fs_nodes/batch_attach_blobs` - binds finished direct uploads to
+     * their nodes. Until this lands the node exists with no bytes and is hidden
+     * from every listing.
+     *
+     * Answers 200 with per-item results even when every item failed. Read
+     * `attached` on each one; do not infer success from the status.
+     */
+    attachBlobs(attachments: Array<{
+        fs_node_id: Id;
+        blob_signed_id: string;
+    }>, options?: RequestOptions): Promise<AttachBlobResult[]>;
+    /**
+     * `POST /fs_nodes/:id/multipart/start`.
+     *
+     * The server keeps no upload state: the `(key, upload_id)` pair lives signed
+     * inside `upload_token`, which is good for 48 hours. Persist it and a torn
+     * upload can be resumed later with {@link uploadMultipart}'s `resume`.
+     *
+     * @throws {OmsApiError} 400 when the node already has data, is a directory,
+     *   or has no size set.
+     */
+    multipartStart(fsNodeId: Id, options?: RequestOptions): Promise<MultipartSession>;
+    /**
+     * `POST /fs_nodes/:id/multipart/part_urls` - presigns up to
+     * {@link MAX_PART_URLS} parts at a time. The URLs live one hour.
+     */
+    multipartPartUrls(fsNodeId: Id, input: {
+        uploadToken: string;
+        partNumbers: number[];
+    }, options?: RequestOptions): Promise<MultipartPartUrl[]>;
+    /**
+     * `POST /fs_nodes/:id/multipart/complete` - assembles the parts. This is what
+     * creates the blob row; there is no separate attach step.
+     *
+     * `byte_size` is what the store measured, and it wins: the node's declared
+     * size and the root's quota counter are reconciled to it server-side.
+     */
+    multipartComplete(fsNodeId: Id, input: {
+        uploadToken: string;
+        parts: MultipartPart[];
+    }, options?: RequestOptions): Promise<MultipartCompletion>;
+    /**
+     * `POST /fs_nodes/:id/multipart/abort` - releases the parts already stored
+     * and, when no blob was ever bound, destroys the node and refunds its quota
+     * reservation.
+     *
+     * Call it whenever an upload fails or is cancelled. {@link upload} does; a
+     * caller driving the steps by hand must too. Skipping it leaves the bytes
+     * charged against the quota until a server-side reaper notices, which is a
+     * schedule, not a promise.
+     */
+    multipartAbort(fsNodeId: Id, input: {
+        uploadToken: string;
+    }, options?: RequestOptions): Promise<void>;
+    /**
+     * Drives one file through the multipart flow: start, presign a window of
+     * parts, PUT them in parallel, complete.
+     *
+     * Parts are sliced with the size the SERVER reported, never with
+     * {@link MULTIPART_PART_SIZE}, so a change on the backend does not silently
+     * corrupt an upload here.
+     *
+     * @param options.resume A session from a previous attempt plus the parts that
+     *   already landed. The matching parts are skipped.
+     * @param options.onSession Called with the upload token as soon as there is
+     *   one, so a caller can persist it and abort later.
+     * @param options.onPart Called with the byte count of each finished part.
+     */
+    uploadMultipart(fsNodeId: Id, body: Blob, options?: RequestOptions & {
+        concurrency?: number;
+        resume?: {
+            uploadToken: string;
+            partSize: number;
+            parts: MultipartPart[];
+        };
+        onSession?: (uploadToken: string, partSize: number) => void;
+        onPart?: (bytes: number, part: MultipartPart) => void;
+    }): Promise<MultipartCompletion>;
+    /**
+     * PUTs bytes at the object store on the injected transport, with no
+     * `Authorization` header and a bounded retry.
+     */
+    private putBytes;
+    /**
+     * Reads finished nodes back in ONE listing.
+     *
+     * A normal listing hides files whose bytes never landed, so a node coming
+     * back is proof the blob is bound - which is why this is a verification and
+     * not just a convenience.
+     */
+    private readBack;
+    /** Best-effort abort of every session still open. Never throws. */
+    private abortAll;
+}
+/**
+ * Base64 MD5 of a blob, in the exact form `Content-MD5` wants.
+ *
+ * The algorithm is not negotiable: the backend binds this digest into the
+ * presigned PUT signature as `Content-MD5`, so anything else makes the object
+ * store reject the upload with a signature error that never mentions checksums.
+ *
+ * WebCrypto implements SHA only, so this cannot be done with platform APIs, and
+ * the core carries no runtime dependencies - it has to run in an isolate where
+ * a Node-flavoured hashing package is not an option. Hence the implementation
+ * below: small, streaming, and self-contained. A host that already owns a
+ * faster digest can replace it through {@link UploadManagerOptions.md5} or
+ * {@link UploadInput.md5}.
+ *
+ * Streams the blob rather than materialising it, so hashing a 31 MiB file costs
+ * one 64 KiB window and not 31 MiB of extra heap.
+ */
+export declare function md5Base64(data: Blob): Promise<string>;
+/**
+ * Builds a manifest entry from a {@link FileInput}, computing the checksum only
+ * when the file is below {@link MULTIPART_THRESHOLD} and therefore needs one.
+ *
+ * Buffers a `ReadableStream` input: the manifest needs an exact byte count up
+ * front, and the direct tier needs a digest of the whole body. Prefer handing
+ * in a `Blob` when you have one.
+ */
+export declare function manifestEntryFor(file: FileInput, input: {
+    clientId: string;
+    relativePath?: string;
+    md5?: Md5Base64Fn;
+}): Promise<UploadManifestEntry>;
+/**
+ * The `fetch` a request to the object store should travel on.
+ *
+ * It has to be the one the host injected into the {@link ApiClient} - a
+ * Worker's proxy, a test double, a policy wrapper - or the bytes silently
+ * escape whatever the host put around its network. `ApiClient` keeps that
+ * function private and `http.ts` is shared, so it is read here through a
+ * defensive probe instead of by growing the transport a new public member.
+ *
+ * When the probe finds nothing (a hand-built stub, a future refactor) the
+ * ambient `fetch` is used, and when there is none either the failure is loud
+ * rather than an upload sent through something unexpected.
+ */
+export declare function objectStoreFetch(http: ApiClient): FetchLike;