@simulatte/doppler 0.1.0

package/src/storage/backends/memory-store.js

@@ -0,0 +1,242 @@
+function ensureModel(models, modelId) {
+  if (!models.has(modelId)) {
+    models.set(modelId, { files: new Map(), meta: new Map() });
+  }
+  return models.get(modelId);
+}
+
+export function createMemoryStore(config) {
+  const { maxBytes } = config;
+  const models = new Map();
+  let currentModelId = null;
+  let totalBytes = 0;
+
+  async function init() {}
+
+  async function openModel(modelId, options = {}) {
+    const create = options.create !== false;
+    if (!create && !models.has(modelId)) {
+      throw new Error('Model not found');
+    }
+    currentModelId = modelId;
+    if (create) {
+      ensureModel(models, modelId);
+    }
+    return null;
+  }
+
+  function getCurrentModelId() {
+    return currentModelId;
+  }
+
+  function requireModel() {
+    if (!currentModelId) {
+      throw new Error('No model open. Call openModelStore first.');
+    }
+  }
+
+  function normalizeWriteStreamOptions(options = {}) {
+    const append = options?.append === true;
+    const expectedOffsetRaw = options?.expectedOffset;
+    const expectedOffset = expectedOffsetRaw == null
+      ? null
+      : Number(expectedOffsetRaw);
+    if (
+      expectedOffset != null
+      && (!Number.isInteger(expectedOffset) || expectedOffset < 0)
+    ) {
+      throw new Error('createWriteStream expectedOffset must be a non-negative integer');
+    }
+    return { append, expectedOffset };
+  }
+
+  function adjustBytes(delta) {
+    totalBytes += delta;
+    if (totalBytes > maxBytes) {
+      totalBytes -= delta;
+      throw new Error(`Memory store exceeded maxBytes (${maxBytes})`);
+    }
+  }
+
+  async function readFile(filename) {
+    requireModel();
+    const model = ensureModel(models, currentModelId);
+    const entry = model.files.get(filename);
+    if (!entry) {
+      throw new Error(`File not found: ${filename}`);
+    }
+    return entry.buffer.slice(entry.byteOffset, entry.byteOffset + entry.byteLength);
+  }
+
+  async function getFileSize(filename) {
+    requireModel();
+    const model = ensureModel(models, currentModelId);
+    const entry = model.files.get(filename);
+    if (!entry) {
+      throw new Error(`File not found: ${filename}`);
+    }
+    return entry.byteLength;
+  }
+
+  async function readText(filename) {
+    try {
+      const buffer = await readFile(filename);
+      return new TextDecoder().decode(buffer);
+    } catch (error) {
+      if (error.message?.includes('File not found')) {
+        return null;
+      }
+      throw error;
+    }
+  }
+
+  async function writeFile(filename, data) {
+    requireModel();
+    const model = ensureModel(models, currentModelId);
+    const bytes = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
+    const prev = model.files.get(filename);
+    if (prev) {
+      adjustBytes(-prev.byteLength);
+    }
+    adjustBytes(bytes.byteLength);
+    model.files.set(filename, bytes.slice(0));
+  }
+
+  async function createWriteStream(filename, options = {}) {
+    requireModel();
+    const { append, expectedOffset } = normalizeWriteStreamOptions(options);
+    const model = ensureModel(models, currentModelId);
+    const previous = model.files.get(filename) ?? null;
+    const base = append && previous
+      ? previous.slice(0)
+      : new Uint8Array(0);
+    if (expectedOffset != null && expectedOffset !== base.byteLength) {
+      throw new Error(
+        `createWriteStream expectedOffset mismatch for ${filename}: expected ${expectedOffset}, got ${base.byteLength}`
+      );
+    }
+    let chunks = [];
+    let total = base.byteLength;
+    let closed = false;
+    return {
+      write: async (chunk) => {
+        if (closed) {
+          throw new Error('Write after close');
+        }
+        const bytes = chunk instanceof ArrayBuffer ? new Uint8Array(chunk) : chunk;
+        chunks.push(bytes.slice(0));
+        total += bytes.byteLength;
+      },
+      close: async () => {
+        if (closed) return;
+        closed = true;
+        const combined = new Uint8Array(total);
+        let offset = 0;
+        if (base.byteLength > 0) {
+          combined.set(base, offset);
+          offset += base.byteLength;
+        }
+        for (const chunk of chunks) {
+          combined.set(chunk, offset);
+          offset += chunk.byteLength;
+        }
+        if (previous) {
+          adjustBytes(-previous.byteLength);
+        }
+        adjustBytes(combined.byteLength);
+        model.files.set(filename, combined);
+        chunks = [];
+      },
+      abort: async () => {
+        if (closed) return;
+        closed = true;
+        chunks = [];
+      },
+    };
+  }
+
+  async function deleteFile(filename) {
+    requireModel();
+    const model = ensureModel(models, currentModelId);
+    const prev = model.files.get(filename);
+    if (prev) {
+      adjustBytes(-prev.byteLength);
+      model.files.delete(filename);
+    }
+    return true;
+  }
+
+  async function listFiles() {
+    requireModel();
+    const model = ensureModel(models, currentModelId);
+    return Array.from(model.files.keys());
+  }
+
+  async function listModels() {
+    return Array.from(models.keys());
+  }
+
+  async function deleteModel(modelId) {
+    const model = models.get(modelId);
+    if (model) {
+      for (const value of model.files.values()) {
+        adjustBytes(-value.byteLength);
+      }
+      models.delete(modelId);
+    }
+    if (currentModelId === modelId) {
+      currentModelId = null;
+    }
+    return true;
+  }
+
+  async function writeManifest(text) {
+    requireModel();
+    const model = ensureModel(models, currentModelId);
+    model.meta.set('manifest', text);
+  }
+
+  async function readManifest() {
+    requireModel();
+    const model = ensureModel(models, currentModelId);
+    return model.meta.get('manifest') ?? null;
+  }
+
+  async function writeTokenizer(text) {
+    requireModel();
+    const model = ensureModel(models, currentModelId);
+    model.meta.set('tokenizer', text);
+  }
+
+  async function readTokenizer() {
+    requireModel();
+    const model = ensureModel(models, currentModelId);
+    return model.meta.get('tokenizer') ?? null;
+  }
+
+  async function cleanup() {
+    models.clear();
+    currentModelId = null;
+    totalBytes = 0;
+  }
+
+  return {
+    init,
+    openModel,
+    getCurrentModelId,
+    getFileSize,
+    readFile,
+    readText,
+    writeFile,
+    createWriteStream,
+    deleteFile,
+    listFiles,
+    listModels,
+    deleteModel,
+    writeManifest,
+    readManifest,
+    writeTokenizer,
+    readTokenizer,
+    cleanup,
+  };
+}

package/src/storage/backends/opfs-store.d.ts

@@ -0,0 +1,41 @@
+export interface OpfsStoreConfig {
+  opfsRootDir: string;
+  useSyncAccessHandle: boolean;
+  maxConcurrentHandles: number;
+}
+
+export interface OpfsWriteStream {
+  write(chunk: Uint8Array | ArrayBuffer): Promise<void>;
+  close(): Promise<void>;
+  abort(): Promise<void>;
+}
+
+export interface OpfsWriteStreamOptions {
+  append?: boolean;
+  expectedOffset?: number | null;
+}
+
+export interface OpfsStore {
+  init(): Promise<void>;
+  openModel(modelId: string, options?: { create?: boolean }): Promise<FileSystemDirectoryHandle>;
+  getCurrentModelId(): string | null;
+  getFileSize(filename: string): Promise<number>;
+  readFile(filename: string): Promise<ArrayBuffer>;
+  readFileRange(filename: string, offset?: number, length?: number | null): Promise<ArrayBuffer>;
+  readFileRangeStream(
+    filename: string,
+    offset?: number,
+    length?: number | null,
+    options?: { chunkBytes?: number }
+  ): AsyncIterable<Uint8Array>;
+  readText(filename: string): Promise<string | null>;
+  writeFile(filename: string, data: Uint8Array | ArrayBuffer): Promise<void>;
+  createWriteStream(filename: string, options?: OpfsWriteStreamOptions): Promise<OpfsWriteStream>;
+  deleteFile(filename: string): Promise<boolean>;
+  listFiles(): Promise<string[]>;
+  listModels(): Promise<string[]>;
+  deleteModel(modelId: string): Promise<boolean>;
+  cleanup(): Promise<void>;
+}
+
+export function createOpfsStore(config: OpfsStoreConfig): OpfsStore;

package/src/storage/backends/opfs-store.js

@@ -0,0 +1,429 @@
+import { isOPFSAvailable } from '../quota.js';
+
+function createLimiter(maxConcurrent) {
+  let active = 0;
+  const queue = [];
+
+  const acquire = async () => {
+    if (active < maxConcurrent) {
+      active += 1;
+      return;
+    }
+    await new Promise((resolve) => queue.push(resolve));
+    active += 1;
+  };
+
+  const release = () => {
+    active = Math.max(0, active - 1);
+    const next = queue.shift();
+    if (next) {
+      next();
+    }
+  };
+
+  return { acquire, release };
+}
+
+export function createOpfsStore(config) {
+  const {
+    opfsRootDir,
+    useSyncAccessHandle,
+    maxConcurrentHandles,
+  } = config;
+  let rootDir = null;
+  let modelsDir = null;
+  let currentModelDir = null;
+  let currentModelId = null;
+  // SyncAccessHandle is typically only available in dedicated workers, but we
+  // allow an optimistic attempt anywhere and fall back gracefully if the
+  // browser rejects it (NotAllowedError / InvalidStateError).
+  const syncAccessEnabled = !!useSyncAccessHandle
+    && typeof FileSystemSyncAccessHandle !== 'undefined';
+  const handleLimiter = syncAccessEnabled ? createLimiter(maxConcurrentHandles) : null;
+
+  if (syncAccessEnabled && (!Number.isInteger(maxConcurrentHandles) || maxConcurrentHandles < 1)) {
+    throw new Error('Invalid opfs.maxConcurrentHandles');
+  }
+
+  async function init() {
+    if (!isOPFSAvailable()) {
+      throw new Error('OPFS not available in this browser');
+    }
+    rootDir = await navigator.storage.getDirectory();
+    modelsDir = await rootDir.getDirectoryHandle(opfsRootDir, { create: true });
+  }
+
+  async function openModel(modelId, options = {}) {
+    if (!modelsDir) {
+      await init();
+    }
+    const create = options.create !== false;
+    currentModelDir = await modelsDir.getDirectoryHandle(modelId, { create });
+    currentModelId = modelId;
+    return currentModelDir;
+  }
+
+  function getCurrentModelId() {
+    return currentModelId;
+  }
+
+  async function openSyncAccessHandle(fileHandle) {
+    if (!syncAccessEnabled || !handleLimiter || typeof fileHandle.createSyncAccessHandle !== 'function') {
+      return null;
+    }
+    await handleLimiter.acquire();
+    try {
+      const handle = await fileHandle.createSyncAccessHandle();
+      return {
+        handle,
+        release: () => {
+          handle.close();
+          handleLimiter.release();
+        }
+      };
+    } catch (error) {
+      handleLimiter.release();
+      if (error?.name === 'InvalidStateError' || error?.name === 'NotAllowedError') {
+        return null;
+      }
+      throw error;
+    }
+  }
+
+  async function ensureModelDir() {
+    if (!currentModelDir) {
+      throw new Error('No model directory open. Call openModelStore first.');
+    }
+  }
+
+  function normalizeWriteStreamOptions(options = {}) {
+    const append = options?.append === true;
+    const expectedOffsetRaw = options?.expectedOffset;
+    const expectedOffset = expectedOffsetRaw == null
+      ? null
+      : Number(expectedOffsetRaw);
+    if (
+      expectedOffset != null
+      && (!Number.isInteger(expectedOffset) || expectedOffset < 0)
+    ) {
+      throw new Error('createWriteStream expectedOffset must be a non-negative integer');
+    }
+    return { append, expectedOffset };
+  }
+
+  async function getFileSize(filename) {
+    await ensureModelDir();
+    const fileHandle = await currentModelDir.getFileHandle(filename);
+    const access = await openSyncAccessHandle(fileHandle);
+    if (access) {
+      try {
+        return access.handle.getSize();
+      } finally {
+        access.release();
+      }
+    }
+    const file = await fileHandle.getFile();
+    return file.size;
+  }
+
+  async function readFile(filename) {
+    await ensureModelDir();
+    const fileHandle = await currentModelDir.getFileHandle(filename);
+    const access = await openSyncAccessHandle(fileHandle);
+    if (access) {
+      try {
+        const size = access.handle.getSize();
+        const buffer = new Uint8Array(size);
+        let offset = 0;
+        while (offset < size) {
+          const view = buffer.subarray(offset);
+          const read = access.handle.read(view, { at: offset });
+          if (read <= 0) {
+            break;
+          }
+          offset += read;
+        }
+        return buffer.buffer;
+      } finally {
+        access.release();
+      }
+    }
+
+    const file = await fileHandle.getFile();
+    return file.arrayBuffer();
+  }
+
+  async function readFileRange(filename, offset = 0, length = null) {
+    await ensureModelDir();
+    const fileHandle = await currentModelDir.getFileHandle(filename);
+    const access = await openSyncAccessHandle(fileHandle);
+
+    const startRaw = Number(offset);
+    const start = Number.isFinite(startRaw) ? Math.max(0, Math.floor(startRaw)) : 0;
+
+    if (access) {
+      try {
+        const size = access.handle.getSize();
+        const end = length == null
+          ? size
+          : Math.min(size, start + Math.max(0, Number.isFinite(Number(length)) ? Math.floor(Number(length)) : 0));
+        const want = Math.max(0, end - start);
+        const buffer = new Uint8Array(want);
+        let readOffset = 0;
+        while (readOffset < want) {
+          const view = buffer.subarray(readOffset);
+          const read = access.handle.read(view, { at: start + readOffset });
+          if (read <= 0) break;
+          readOffset += read;
+        }
+        return buffer.buffer;
+      } finally {
+        access.release();
+      }
+    }
+
+    const file = await fileHandle.getFile();
+    const end = length == null
+      ? file.size
+      : Math.min(file.size, start + Math.max(0, Number.isFinite(Number(length)) ? Math.floor(Number(length)) : 0));
+    return file.slice(start, end).arrayBuffer();
+  }
+
+  async function* readFileRangeStream(filename, offset = 0, length = null, options = {}) {
+    const rawChunkBytes = options?.chunkBytes;
+    const chunkBytes = Number.isFinite(rawChunkBytes) && rawChunkBytes > 0
+      ? Math.floor(rawChunkBytes)
+      : (4 * 1024 * 1024);
+    const startRaw = Number(offset);
+    const start = Number.isFinite(startRaw) ? Math.max(0, Math.floor(startRaw)) : 0;
+
+    await ensureModelDir();
+    const fileHandle = await currentModelDir.getFileHandle(filename);
+    const access = await openSyncAccessHandle(fileHandle);
+
+    if (access) {
+      try {
+        const size = access.handle.getSize();
+        const end = length == null
+          ? size
+          : Math.min(size, start + Math.max(0, Number.isFinite(Number(length)) ? Math.floor(Number(length)) : 0));
+        let at = start;
+        const scratch = new Uint8Array(chunkBytes);
+        while (at < end) {
+          const want = Math.min(chunkBytes, end - at);
+          const view = want === scratch.byteLength ? scratch : scratch.subarray(0, want);
+          const read = access.handle.read(view, { at });
+          if (read <= 0) break;
+          // Copy out to avoid consumers seeing a mutated scratch buffer.
+          yield view.slice(0, read);
+          at += read;
+        }
+        return;
+      } finally {
+        access.release();
+      }
+    }
+
+    // Fallback: repeated slice reads (allocates per-chunk, but avoids full-file materialization).
+    const file = await fileHandle.getFile();
+    const end = length == null
+      ? file.size
+      : Math.min(file.size, start + Math.max(0, Number.isFinite(Number(length)) ? Math.floor(Number(length)) : 0));
+    for (let at = start; at < end; at += chunkBytes) {
+      const ab = await file.slice(at, Math.min(end, at + chunkBytes)).arrayBuffer();
+      yield new Uint8Array(ab);
+    }
+  }
+
+  async function readText(filename) {
+    await ensureModelDir();
+    try {
+      const fileHandle = await currentModelDir.getFileHandle(filename);
+      const file = await fileHandle.getFile();
+      return await file.text();
+    } catch (error) {
+      if (error.name === 'NotFoundError') {
+        return null;
+      }
+      throw error;
+    }
+  }
+
+  async function writeFile(filename, data) {
+    await ensureModelDir();
+    const fileHandle = await currentModelDir.getFileHandle(filename, { create: true });
+    const bytes = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
+    const access = await openSyncAccessHandle(fileHandle);
+    if (access) {
+      try {
+        access.handle.truncate(0);
+        access.handle.write(bytes, { at: 0 });
+        access.handle.flush();
+      } finally {
+        access.release();
+      }
+      return;
+    }
+
+    const writable = await fileHandle.createWritable();
+    await writable.write(bytes);
+    await writable.close();
+  }
+
+  async function createWriteStream(filename, options = {}) {
+    await ensureModelDir();
+    const { append, expectedOffset } = normalizeWriteStreamOptions(options);
+    const fileHandle = await currentModelDir.getFileHandle(filename, { create: true });
+    const existingSize = await getFileSize(filename).catch((error) => {
+      if (error?.name === 'NotFoundError') {
+        return 0;
+      }
+      throw error;
+    });
+    if (expectedOffset != null && expectedOffset !== existingSize) {
+      throw new Error(
+        `createWriteStream expectedOffset mismatch for ${filename}: expected ${expectedOffset}, got ${existingSize}`
+      );
+    }
+    const access = await openSyncAccessHandle(fileHandle);
+    if (access) {
+      const initialOffset = append ? existingSize : 0;
+      let offset = initialOffset;
+      let closed = false;
+      if (!append) {
+        access.handle.truncate(0);
+      }
+      return {
+        write: async (chunk) => {
+          if (closed) {
+            throw new Error('Write after close');
+          }
+          const bytes = chunk instanceof ArrayBuffer ? new Uint8Array(chunk) : chunk;
+          access.handle.write(bytes, { at: offset });
+          offset += bytes.byteLength;
+        },
+        close: async () => {
+          if (closed) return;
+          closed = true;
+          access.handle.flush();
+          access.release();
+        },
+        abort: async () => {
+          if (closed) return;
+          closed = true;
+          access.handle.truncate(initialOffset);
+          access.release();
+        },
+      };
+    }
+
+    const writable = await fileHandle.createWritable({ keepExistingData: append });
+    let closed = false;
+    let offset = append ? existingSize : 0;
+    return {
+      write: async (chunk) => {
+        if (closed) {
+          throw new Error('Write after close');
+        }
+        const bytes = chunk instanceof ArrayBuffer ? new Uint8Array(chunk) : chunk;
+        await writable.write({
+          type: 'write',
+          position: offset,
+          data: bytes,
+        });
+        offset += bytes.byteLength;
+      },
+      close: async () => {
+        if (closed) return;
+        closed = true;
+        await writable.close();
+      },
+      abort: async () => {
+        if (closed) return;
+        closed = true;
+        await writable.abort();
+      },
+    };
+  }
+
+  async function deleteFile(filename) {
+    await ensureModelDir();
+    try {
+      await currentModelDir.removeEntry(filename);
+      return true;
+    } catch (error) {
+      if (error.name === 'NotFoundError') {
+        return false;
+      }
+      throw error;
+    }
+  }
+
+  async function listFiles() {
+    await ensureModelDir();
+    const files = [];
+    for await (const [name, handle] of currentModelDir.entries()) {
+      if (handle.kind === 'file') {
+        files.push(name);
+      }
+    }
+    return files;
+  }
+
+  async function listModels() {
+    if (!modelsDir) {
+      await init();
+    }
+    const models = [];
+    for await (const [name, handle] of modelsDir.entries()) {
+      if (handle.kind === 'directory') {
+        models.push(name);
+      }
+    }
+    return models;
+  }
+
+  async function deleteModel(modelId) {
+    if (!modelsDir) {
+      await init();
+    }
+    try {
+      await modelsDir.removeEntry(modelId, { recursive: true });
+      if (currentModelId === modelId) {
+        currentModelId = null;
+        currentModelDir = null;
+      }
+      return true;
+    } catch (error) {
+      if (error.name === 'NotFoundError') {
+        return false;
+      }
+      throw error;
+    }
+  }
+
+  async function cleanup() {
+    rootDir = null;
+    modelsDir = null;
+    currentModelDir = null;
+    currentModelId = null;
+  }
+
+  return {
+    init,
+    openModel,
+    getCurrentModelId,
+    getFileSize,
+    readFile,
+    readFileRange,
+    readFileRangeStream,
+    readText,
+    writeFile,
+    createWriteStream,
+    deleteFile,
+    listFiles,
+    listModels,
+    deleteModel,
+    cleanup,
+  };
+}