@elaraai/e3-types 1.0.76 → 1.0.78

package/src/transfer.spec.ts

@@ -0,0 +1,98 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+
+/**
+ * The dataset transfer protocol's two invariants that live in e3-types.
+ *
+ * Part arithmetic is shared by the client that cuts the bytes and the server
+ * that places them, so an off-by-one on either side would corrupt uploads
+ * silently until the commit's hash check. And protocol 2 only adds variant
+ * cases: a version-1 client decodes a version-1 answer from a new server, and a
+ * new client decodes an old server's answers, only while every added case sorts
+ * after the version-1 cases and so leaves their tags unchanged.
+ */
+
+import { describe, it } from 'node:test';
+import * as assert from 'node:assert';
+import { NullType, StringType, StructType, VariantType, decodeBeast2For, encodeBeast2For, variant } from '@elaraai/east';
+import {
+  TransferDoneResponseType,
+  TransferUploadResponseType,
+  transferPartCount,
+  transferPartRange,
+} from './transfer.js';
+
+describe('transferPartCount / transferPartRange', () => {
+  it('cuts an upload into full parts and a remainder', () => {
+    assert.strictEqual(transferPartCount(10, 4), 3);
+    assert.deepStrictEqual(transferPartRange(10, 4, 1), { start: 0, end: 4 });
+    assert.deepStrictEqual(transferPartRange(10, 4, 2), { start: 4, end: 8 });
+    assert.deepStrictEqual(transferPartRange(10, 4, 3), { start: 8, end: 10 });
+  });
+
+  it('gives an exact multiple no empty trailing part', () => {
+    assert.strictEqual(transferPartCount(8, 4), 2);
+    assert.deepStrictEqual(transferPartRange(8, 4, 2), { start: 4, end: 8 });
+    assert.strictEqual(transferPartRange(8, 4, 3), null);
+  });
+
+  it('sends an upload no larger than a part as one part', () => {
+    assert.strictEqual(transferPartCount(3, 4), 1);
+    assert.strictEqual(transferPartCount(4, 4), 1);
+    assert.deepStrictEqual(transferPartRange(3, 4, 1), { start: 0, end: 3 });
+  });
+
+  it('has no part 0, no fractional part and none past the end', () => {
+    assert.strictEqual(transferPartRange(10, 4, 0), null);
+    assert.strictEqual(transferPartRange(10, 4, 1.5), null);
+    assert.strictEqual(transferPartRange(10, 4, 4), null);
+  });
+
+  it('stays exact past 2^32 bytes, taking sizes as bigint or number', () => {
+    const size = 5n * 1024n ** 4n + 1n; // 5 TiB and a byte
+    const partBytes = 512n * 1024n ** 2n;
+    assert.strictEqual(transferPartCount(size, partBytes), 10241);
+    assert.deepStrictEqual(transferPartRange(Number(size), Number(partBytes), 10241), {
+      start: Number(size) - 1,
+      end: Number(size),
+    });
+  });
+});
+
+describe('transfer wire compatibility across protocol versions', () => {
+  // The version-1 shapes, as a client and server built before protocol 2 hold them.
+  const UploadResponseV1 = VariantType({
+    completed: NullType,
+    upload: StructType({ id: StringType, uploadUrl: StringType }),
+  });
+  const DoneResponseV1 = VariantType({
+    completed: NullType,
+    error: StructType({ message: StringType }),
+  });
+
+  it('keeps the init answers a version-1 client decodes', () => {
+    const upload = variant('upload', { id: 'abc', uploadUrl: 'http://h/api/uploads/abc' });
+    assert.deepStrictEqual(
+      decodeBeast2For(UploadResponseV1)(encodeBeast2For(TransferUploadResponseType)(upload)),
+      upload,
+    );
+    assert.deepStrictEqual(
+      decodeBeast2For(TransferUploadResponseType)(encodeBeast2For(UploadResponseV1)(upload)),
+      upload,
+    );
+    const completed = variant('completed', null);
+    assert.deepStrictEqual(
+      decodeBeast2For(UploadResponseV1)(encodeBeast2For(TransferUploadResponseType)(completed)),
+      completed,
+    );
+  });
+
+  it('keeps the commit answers a version-1 client decodes', () => {
+    for (const done of [variant('completed', null), variant('error', { message: 'hash mismatch' })] as const) {
+      assert.deepStrictEqual(decodeBeast2For(DoneResponseV1)(encodeBeast2For(TransferDoneResponseType)(done)), done);
+      assert.deepStrictEqual(decodeBeast2For(TransferDoneResponseType)(encodeBeast2For(DoneResponseV1)(done)), done);
+    }
+  });
+});

package/src/transfer.ts

@@ -8,9 +8,60 @@
  *
  * Used by both the API server and client for large dataset uploads
  * that exceed inline body size limits.
+ *
+ * Two protocol versions share these types. A client names the version it speaks
+ * with `?protocol=N` on the init and commit requests (a request without it is
+ * version 1), and a server answers only in the forms that version understands —
+ * every case version 2 adds sorts after the version-1 cases, so the tags a
+ * version-1 client decodes keep their numbers.
+ */
+
+import { VariantType, StructType, StringType, IntegerType, NullType, DictType, type ValueTypeOf } from '@elaraai/east';
+
+/**
+ * The dataset transfer protocol version this build speaks.
+ *
+ * @remarks
+ * Version 2 lets the server plan an upload as parts — each sent to its own URL
+ * with the request headers the server names, so an object store can bind a
+ * checksum to a single PUT or take a multipart upload larger than one PUT
+ * allows — and lets a commit answer `processing` while the server verifies the
+ * bytes, for the client to poll.
+ */
+export const TRANSFER_PROTOCOL_VERSION = 2;
+
+/**
+ * How many parts an upload of `size` bytes has when planned with `partBytes`.
+ *
+ * @param size - The upload's size in bytes
+ * @param partBytes - The size of every part but the last
+ * @returns The part count — at least 1
  */
+export function transferPartCount(size: number | bigint, partBytes: number | bigint): number {
+  const total = BigInt(size);
+  const each = BigInt(partBytes);
+  return total <= each ? 1 : Number((total + each - 1n) / each);
+}
 
-import { VariantType, StructType, StringType, IntegerType, NullType, type ValueTypeOf } from '@elaraai/east';
+/**
+ * The byte range part `part` of an upload covers.
+ *
+ * @param size - The upload's size in bytes
+ * @param partBytes - The size of every part but the last
+ * @param part - The part's number, from 1
+ * @returns The half-open range `[start, end)`, or `null` when the upload has no
+ *   such part
+ */
+export function transferPartRange(
+  size: number | bigint,
+  partBytes: number | bigint,
+  part: number
+): { start: number; end: number } | null {
+  if (!Number.isInteger(part) || part < 1 || part > transferPartCount(size, partBytes)) return null;
+  const start = BigInt(part - 1) * BigInt(partBytes);
+  const end = start + BigInt(partBytes) < BigInt(size) ? start + BigInt(partBytes) : BigInt(size);
+  return { start: Number(start), end: Number(end) };
+}
 
 /**
  * Transfer upload init request.
@@ -30,7 +81,15 @@ export type TransferUploadRequest = ValueTypeOf<typeof TransferUploadRequestType
  * Transfer upload init response.
  *
  * - `completed`: Object already exists (dedup), dataset ref updated
- * - `upload`: Staging slot created, client should upload then call done
+ * - `upload`: Staging slot created; the client PUTs every byte to `uploadUrl`,
+ *   then commits
+ * - `upload_parts` (protocol 2): the client sends the bytes as parts of
+ *   `partBytes` bytes each (the last part holds the remainder, and an upload no
+ *   larger than `partBytes` is one part). Part `n` (1-based) is the byte range
+ *   `[(n - 1) * partBytes, min(size, n * partBytes))`; its URL and required
+ *   headers come from `GET …/upload/<id>/parts/<n>`. Parts may be sent in any
+ *   order and concurrently, and re-sending a part replaces it. The client
+ *   commits once every part is sent.
  */
 export const TransferUploadResponseType = VariantType({
   completed: NullType,
@@ -38,19 +97,41 @@ export const TransferUploadResponseType = VariantType({
     id: StringType,
     uploadUrl: StringType,
   }),
+  upload_parts: StructType({
+    id: StringType,
+    partBytes: IntegerType,
+  }),
 });
 export type TransferUploadResponse = ValueTypeOf<typeof TransferUploadResponseType>;
 
 /**
- * Transfer done response.
+ * Where and how to send one part of a protocol-2 upload.
+ *
+ * @property url - The URL the client PUTs the part's bytes to, without an
+ *   `Authorization` header (it may be a presigned object-store URL)
+ * @property headers - Request headers the PUT must carry exactly as given — a
+ *   presigned URL can sign them, e.g. the checksum an object store verifies the
+ *   bytes against
+ */
+export const TransferPartResponseType = StructType({
+  url: StringType,
+  headers: DictType(StringType, StringType),
+});
+export type TransferPartResponse = ValueTypeOf<typeof TransferPartResponseType>;
+
+/**
+ * Transfer done response — the commit's answer, and the poll's.
  *
  * - `completed`: Hash verified, object stored, dataset ref updated
  * - `error`: Hash mismatch or other failure
+ * - `processing` (protocol 2): the server is still verifying the bytes; poll
+ *   `GET …/upload/<id>` until it answers `completed` or `error`
  */
 export const TransferDoneResponseType = VariantType({
   completed: NullType,
   error: StructType({
     message: StringType,
   }),
+  processing: NullType,
 });
 export type TransferDoneResponse = ValueTypeOf<typeof TransferDoneResponseType>;