@oatscenter/trellis 0.100.0-rc.1

package/service/runtime/transfer/download.d.ts

@@ -0,0 +1,27 @@
+import { sha256 } from "@noble/hashes/sha256";
+import type { Subscription } from "@nats-io/nats-core";
+import type { PermissionAtom } from "../../../participant_runtime/api.ts";
+import type { TypedStore } from "../../../store.ts";
+import type { FileInfo } from "../../../transfer.ts";
+export type DownloadSession = {
+    kind: "download";
+    subject: string;
+    transferId: string;
+    sessionKey: string;
+    permission: PermissionAtom;
+    requiredCapabilities: readonly string[];
+    inboxPrefix: string;
+    expiresAtMs: number;
+    store: TypedStore;
+    key: string;
+    info: FileInfo;
+    subscription: Subscription;
+    timeoutId: ReturnType<typeof setTimeout>;
+    reader?: ReadableStreamDefaultReader<Uint8Array>;
+    pending?: Uint8Array;
+    pendingOffset: number;
+    nextSeq: number;
+    sentBytes: number;
+    hasher: ReturnType<typeof sha256.create>;
+    cancellation: AbortController;
+};

package/service/runtime/transfer/download.js

@@ -0,0 +1 @@
+import { sha256 } from "@noble/hashes/sha256";

package/service/runtime/transfer/protocol.d.ts

@@ -0,0 +1,15 @@
+import { type Result as ResultType } from "@oatscenter/result";
+import { type Msg, type NatsConnection } from "@nats-io/nats-core";
+import { TransferError } from "../../../errors/TransferError.ts";
+import type { StoreInfo } from "../../../store.ts";
+import type { FileInfo } from "../../../transfer.ts";
+export declare const UPLOAD_SUBJECT_PREFIX = "transfer.v1.upload";
+export declare const DOWNLOAD_SUBJECT_PREFIX = "transfer.v1.download";
+export declare const TRANSFER_SEQUENCE_HEADER = "trellis-transfer-seq";
+export declare const TRANSFER_EOF_HEADER = "trellis-transfer-eof";
+export declare const TRANSFER_CONTROL_HEADER = "trellis-transfer-control";
+export declare const DEFAULT_TRANSFER_CHUNK_BYTES: number;
+export declare function fileInfoFromStoreInfo(info: StoreInfo): FileInfo;
+export declare function replyError(msg: Msg, error: TransferError): void;
+export declare function publishError(nc: NatsConnection, subject: string, error: TransferError): void;
+export declare function parseSeq(msg: Msg): ResultType<number, TransferError>;

package/service/runtime/transfer/protocol.js

@@ -0,0 +1,46 @@
+import { Result } from "@oatscenter/result";
+import { headers as natsHeaders, } from "@nats-io/nats-core";
+import { TransferError } from "../../../errors/TransferError.js";
+export const UPLOAD_SUBJECT_PREFIX = "transfer.v1.upload";
+export const DOWNLOAD_SUBJECT_PREFIX = "transfer.v1.download";
+export const TRANSFER_SEQUENCE_HEADER = "trellis-transfer-seq";
+export const TRANSFER_EOF_HEADER = "trellis-transfer-eof";
+export const TRANSFER_CONTROL_HEADER = "trellis-transfer-control";
+export const DEFAULT_TRANSFER_CHUNK_BYTES = 256 * 1024;
+export function fileInfoFromStoreInfo(info) {
+    return {
+        key: info.key,
+        size: info.size,
+        updatedAt: info.updatedAt,
+        ...(info.digest ? { digest: info.digest } : {}),
+        ...(info.contentType ? { contentType: info.contentType } : {}),
+        metadata: info.metadata,
+    };
+}
+export function replyError(msg, error) {
+    const headers = natsHeaders();
+    headers.set("status", "error");
+    msg.respond(JSON.stringify(error.toSerializable()), { headers });
+}
+export function publishError(nc, subject, error) {
+    const headers = natsHeaders();
+    headers.set("status", "error");
+    nc.publish(subject, JSON.stringify(error.toSerializable()), { headers });
+}
+export function parseSeq(msg) {
+    const raw = msg.headers?.get(TRANSFER_SEQUENCE_HEADER);
+    if (!raw) {
+        return Result.err(new TransferError({
+            operation: "transfer",
+            context: { reason: "missing_sequence" },
+        }));
+    }
+    const value = Number(raw);
+    if (!Number.isInteger(value) || value < 0) {
+        return Result.err(new TransferError({
+            operation: "transfer",
+            context: { reason: "invalid_sequence", raw },
+        }));
+    }
+    return Result.ok(value);
+}

package/service/runtime/transfer/queue.d.ts

@@ -0,0 +1,18 @@
+export declare function deferred<T>(): {
+    promise: Promise<T>;
+    resolve(value: T): void;
+};
+export declare class AsyncValueBroadcaster<T> {
+    #private;
+    push(value: T): void;
+    close(): void;
+    subscribe(): AsyncIterable<T>;
+}
+export declare class AsyncChunkQueue implements AsyncIterable<Uint8Array> {
+    #private;
+    push(chunk: Uint8Array): Promise<void>;
+    close(): void;
+    fail(error: unknown): void;
+    next(): Promise<IteratorResult<Uint8Array>>;
+    [Symbol.asyncIterator](): AsyncIterator<Uint8Array>;
+}

package/service/runtime/transfer/queue.js

@@ -0,0 +1,136 @@
+export function deferred() {
+    let resolve;
+    const promise = new Promise((r) => {
+        resolve = r;
+    });
+    return { promise, resolve };
+}
+class AsyncValueQueue {
+    #values = [];
+    #resolvers = [];
+    #closed = false;
+    push(value) {
+        if (this.#closed)
+            return;
+        const resolver = this.#resolvers.shift();
+        if (resolver) {
+            resolver({ value, done: false });
+        }
+        else {
+            this.#values.push(value);
+        }
+    }
+    close() {
+        if (this.#closed)
+            return;
+        this.#closed = true;
+        for (const resolver of this.#resolvers.splice(0)) {
+            resolver({ value: undefined, done: true });
+        }
+    }
+    [Symbol.asyncIterator]() {
+        return {
+            next: async () => {
+                const value = this.#values.shift();
+                if (value !== undefined)
+                    return { value, done: false };
+                if (this.#closed)
+                    return { value: undefined, done: true };
+                return await new Promise((resolve) => {
+                    this.#resolvers.push(resolve);
+                });
+            },
+        };
+    }
+}
+export class AsyncValueBroadcaster {
+    #subscribers = new Set();
+    #closed = false;
+    push(value) {
+        if (this.#closed)
+            return;
+        for (const subscriber of this.#subscribers)
+            subscriber.push(value);
+    }
+    close() {
+        if (this.#closed)
+            return;
+        this.#closed = true;
+        for (const subscriber of this.#subscribers)
+            subscriber.close();
+        this.#subscribers.clear();
+    }
+    subscribe() {
+        const subscriber = new AsyncValueQueue();
+        if (this.#closed) {
+            subscriber.close();
+        }
+        else {
+            this.#subscribers.add(subscriber);
+        }
+        const subscribers = this.#subscribers;
+        return {
+            async *[Symbol.asyncIterator]() {
+                try {
+                    yield* subscriber;
+                }
+                finally {
+                    subscribers.delete(subscriber);
+                }
+            },
+        };
+    }
+}
+export class AsyncChunkQueue {
+    #values = [];
+    #pending = [];
+    #closed = false;
+    #error;
+    async push(chunk) {
+        if (this.#closed)
+            return;
+        const pending = this.#pending.shift();
+        if (pending) {
+            pending.resolve({ value: chunk, done: false });
+            return;
+        }
+        await new Promise((consumed) => {
+            this.#values.push({ chunk, consumed });
+        });
+    }
+    close() {
+        if (this.#closed)
+            return;
+        this.#closed = true;
+        for (const pending of this.#pending.splice(0)) {
+            pending.resolve({ value: undefined, done: true });
+        }
+    }
+    fail(error) {
+        if (this.#closed)
+            return;
+        this.#error = error;
+        this.#closed = true;
+        for (const value of this.#values.splice(0))
+            value.consumed();
+        for (const pending of this.#pending.splice(0))
+            pending.reject(error);
+    }
+    async next() {
+        const value = this.#values.shift();
+        if (value) {
+            value.consumed();
+            return { value: value.chunk, done: false };
+        }
+        if (this.#error)
+            throw this.#error;
+        if (this.#closed)
+            return { value: undefined, done: true };
+        return await new Promise((resolve, reject) => {
+            this.#pending.push({ resolve, reject });
+        });
+    }
+    [Symbol.asyncIterator]() {
+        return { next: () => this.next() };
+    }
+}

package/service/runtime/transfer/upload.d.ts

@@ -0,0 +1,40 @@
+import type { Subscription } from "@nats-io/nats-core";
+import type { PermissionAtom } from "../../../participant_runtime/api.ts";
+import type { RuntimeOperationTransferProgress } from "../../../session.ts";
+import type { StoreError } from "../../../errors/StoreError.ts";
+import type { TransferError } from "../../../errors/TransferError.ts";
+import type { TypedStore } from "../../../store.ts";
+import type { FileInfo } from "../../../transfer.ts";
+import type { AsyncResult } from "@oatscenter/result";
+import type { StoredTransfer } from "../transfer.ts";
+import type { AsyncChunkQueue } from "./queue.ts";
+import { sha256 } from "@noble/hashes/sha256";
+export type UploadSession = {
+    kind: "upload";
+    subject: string;
+    transferId: string;
+    sessionKey: string;
+    permission: PermissionAtom | undefined;
+    requiredCapabilities: readonly string[];
+    expiresAtMs: number;
+    store: TypedStore;
+    key: string;
+    maxBytes?: number;
+    contentType?: string;
+    metadata?: Record<string, string>;
+    onProgress?: (progress: RuntimeOperationTransferProgress) => Promise<void> | void;
+    onComplete?: (info: FileInfo) => Promise<void> | void;
+    onError?: (error: TransferError) => Promise<void> | void;
+    onStored?: (stored: StoredTransfer) => Promise<void> | void;
+    subscription: Subscription;
+    timeoutId: ReturnType<typeof setTimeout>;
+    queue: AsyncChunkQueue;
+    putPromise: AsyncResult<void, StoreError>;
+    cancellation: AbortController;
+    committing: boolean;
+    nextSeq: number;
+    receivedBytes: number;
+    hasher: ReturnType<typeof sha256.create>;
+};
+export declare function raceCancellation<T>(promise: Promise<T>, signal: AbortSignal): Promise<T>;
+export declare function effectiveUploadMaxBytes(argsMaxBytes?: number, storeMaxObjectBytes?: number): number | undefined;

package/service/runtime/transfer/upload.js

@@ -0,0 +1,27 @@
+import { sha256 } from "@noble/hashes/sha256";
+export function raceCancellation(promise, signal) {
+    if (signal.aborted)
+        return Promise.reject(signal.reason);
+    return new Promise((resolve, reject) => {
+        const cancel = () => {
+            finish();
+            reject(signal.reason);
+        };
+        const finish = () => signal.removeEventListener("abort", cancel);
+        signal.addEventListener("abort", cancel, { once: true });
+        promise.then((value) => {
+            finish();
+            resolve(value);
+        }, (error) => {
+            finish();
+            reject(error);
+        });
+    });
+}
+export function effectiveUploadMaxBytes(argsMaxBytes, storeMaxObjectBytes) {
+    if (argsMaxBytes === undefined)
+        return storeMaxObjectBytes;
+    if (storeMaxObjectBytes === undefined)
+        return argsMaxBytes;
+    return Math.min(argsMaxBytes, storeMaxObjectBytes);
+}

package/service/runtime/transfer.d.ts

@@ -0,0 +1,62 @@
+import { AsyncResult, type Result as ResultType } from "@oatscenter/result";
+import { type NatsConnection } from "@nats-io/nats-core";
+import type { PermissionAtom } from "../../participant_runtime/api.ts";
+import { type OperationTransferHandle, type RuntimeOperationTransferProgress, type TrellisAuth } from "../../session.ts";
+import type { StoreError } from "../../errors/StoreError.ts";
+import { TransferError } from "../../errors/TransferError.ts";
+import { TypedStore, TypedStoreEntry } from "../../store.ts";
+import type { FileInfo, ReceiveTransferGrant, SendTransferGrant } from "../../transfer.ts";
+export type TransferStoreHandle = {
+    open(): AsyncResult<TypedStore, StoreError>;
+};
+export type InitiateUploadArgs = {
+    sessionKey: string;
+    permission: PermissionAtom | undefined;
+    requiredCapabilities: readonly string[];
+    store: string;
+    key: string;
+    expiresInMs: number;
+    maxBytes?: number;
+    contentType?: string;
+    metadata?: Record<string, string>;
+    onProgress?: (progress: RuntimeOperationTransferProgress) => Promise<void> | void;
+    onComplete?: (info: FileInfo) => Promise<void> | void;
+    onError?: (error: TransferError) => Promise<void> | void;
+    onStored?: (stored: StoredTransfer) => Promise<void> | void;
+};
+export type OperationUploadTransfer = {
+    grant: SendTransferGrant;
+    transfer: OperationTransferHandle;
+};
+export type InitiateDownloadArgs = {
+    sessionKey: string;
+    permission: PermissionAtom;
+    requiredCapabilities?: readonly string[];
+    inboxPrefix: string;
+    store: string;
+    key: string;
+    expiresInMs: number;
+};
+type ServiceTransferOpts = {
+    name: string;
+    nc: NatsConnection;
+    auth: TrellisAuth;
+    stores: Record<string, TransferStoreHandle>;
+    chunkBytes?: number;
+};
+export type StoredTransfer = {
+    transferId: string;
+    sessionKey: string;
+    store: TypedStore;
+    entry: TypedStoreEntry;
+    info: FileInfo;
+};
+export declare class ServiceTransfer {
+    #private;
+    constructor(opts: ServiceTransferOpts);
+    initiateUpload(args: InitiateUploadArgs): Promise<ResultType<SendTransferGrant, TransferError>>;
+    createOperationUpload(args: InitiateUploadArgs): AsyncResult<OperationUploadTransfer, TransferError>;
+    initiateDownload(args: InitiateDownloadArgs): Promise<ResultType<ReceiveTransferGrant, TransferError>>;
+    stop(): Promise<void>;
+}
+export {};