@volter/world-core 3.0.156 → 3.0.158

package/src/blob-store.ts

@@ -19,7 +19,7 @@
 // local worlds change NOTHING — scrub, backup, and inspection all still see the same tree.
 // A remote adapter maps the path to an object key (namespace prefix + path).
 import { createHash, randomUUID } from 'node:crypto';
-import { closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync, writeSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { volterHome } from './volter-home.ts';
 import { sharedBlobIndex } from './shared-blob-index.ts';
@@ -46,6 +46,38 @@ export interface BlobStore {
    *  whole. Optional: `readBlobRange` falls back to `get` (a whole read) for a backend without it —
    *  the hosted World's R2 store today. */
   getRange?(key: string, start: number, endInclusive: number): Promise<Uint8Array | null>;
+  /** A body's chunks written to `key` as they arrive and hashed as they are written, kept (atomically, as `put`)
+   *  only when `accept` holds for their size and SHA-256; otherwise nothing is kept. True when kept. So an upload
+   *  of a large object (a model's weights) never holds more than a chunk. Optional: `writeBlobStream` collects
+   *  the chunks and `put`s them for a backend without it. */
+  putStream?(key: string, chunks: AsyncIterable<Uint8Array>, accept: (written: BlobWritten) => boolean): Promise<boolean>;
+}
+
+/** What a streamed write wrote: its byte length and its SHA-256 (hex). */
+export type BlobWritten = { size: number; sha256: string };
+
+/** A body's chunks to `key` through the active store, kept when `accept` holds for what was written (the store's
+ *  own streamed write where it has one, else the chunks collected and `put`). True when kept. */
+export async function writeBlobStream(key: string, chunks: AsyncIterable<Uint8Array>, accept: (written: BlobWritten) => boolean): Promise<boolean> {
+  const store = getActiveBlobStore();
+  if (store.putStream) return store.putStream(key, chunks, accept);
+  const parts: Uint8Array[] = [];
+  let size = 0;
+  for await (const chunk of chunks) { parts.push(chunk); size += chunk.byteLength; }
+  const bytes = new Uint8Array(size);
+  let at = 0;
+  for (const part of parts) { bytes.set(part, at); at += part.byteLength; }
+  if (!accept({ size, sha256: blobDigest(bytes) })) return false;
+  await store.put(key, bytes);
+  return true;
+}
+
+/** A web stream's chunks, read by its reader (a browser's ReadableStream need not be async-iterable). */
+export async function* streamChunks(body: ReadableStream<Uint8Array>): AsyncGenerator<Uint8Array> {
+  const reader = body.getReader();
+  try {
+    for (;;) { const { done, value } = await reader.read(); if (done) return; if (value) yield value; }
+  } finally { reader.releaseLock(); }
 }
 
 /** A range of the blob at `key` through the active store: native when the store has it, else sliced. */
@@ -86,6 +118,30 @@ export class FsBlobStore implements BlobStore {
       index?.record(key);
     } finally { rmSync(tmp, { force: true }); }
   }
+  async putStream(key: string, chunks: AsyncIterable<Uint8Array>, accept: (written: BlobWritten) => boolean): Promise<boolean> {
+    mkdirSync(dirname(key), { recursive: true });
+    const tmp = `${key}.tmp.${randomUUID()}`;
+    const hash = createHash('sha256');
+    let size = 0;
+    try {
+      const fd = openSync(tmp, 'wx');
+      try {
+        for await (const chunk of chunks) {
+          hash.update(chunk);
+          size += chunk.byteLength;
+          for (let at = 0; at < chunk.byteLength;) at += writeSync(fd, chunk, at, chunk.byteLength - at);
+        }
+      } finally { closeSync(fd); }
+      const sha256 = hash.digest('hex');
+      if (!accept({ size, sha256 })) return false;
+      renameSync(tmp, key);
+      if (this.options.sharedIndexRoot !== false) {
+        try { sharedBlobIndex(this.options.sharedIndexRoot ?? join(volterHome(), 'blob-index-v1'), statSync(dirname(key)).dev, sha256).record(key); }
+        catch { /* The index is an optimization, never a prerequisite for a blob write. */ }
+      }
+      return true;
+    } finally { rmSync(tmp, { force: true }); }
+  }
   async exists(key: string): Promise<boolean> {
     return existsSync(key);
   }
@@ -178,7 +234,9 @@ function readOnlyBlobs(store: BlobStore): BlobStore {
     exists: (key) => store.exists(key),
     list: (prefix) => store.list(prefix),
     size: (key) => store.size(key),
+    ...(store.getRange ? { getRange: (key: string, start: number, endInclusive: number) => store.getRange!(key, start, endInclusive) } : {}),
     put: async () => { refuseReadOnlyWrite('blobs'); },
+    putStream: async () => { refuseReadOnlyWrite('blobs'); return false; },
     remove: async () => { refuseReadOnlyWrite('blobs'); },
   };
 }