@simulatte/doppler 0.1.5 → 0.1.7

package/src/storage/index.js

@@ -20,6 +20,7 @@ export {
   writeShard,
   createShardWriter,
   createConversionShardWriter,
+  createFileWriter,
   loadShard,
   loadShardRange,
   streamShardRange,
@@ -32,6 +33,7 @@ export {
   listModels,
   listFilesInStore,
   loadFileFromStore,
+  loadFileRangeFromStore,
   streamFileFromStore,
   getModelInfo,
   modelExists,
@@ -46,6 +48,7 @@ export {
   saveAuxFile,
   loadAuxFile,
   loadAuxText,
+  deleteFileFromStore,
   cleanup,
 } from './shard-manager.js';
 

package/src/storage/preflight.d.ts

@@ -113,8 +113,8 @@ export declare const GEMMA_1B_REQUIREMENTS: ModelRequirements;
 
 /**
  * All available model requirements
- * Naming convention: {family}-{version}-{size}-{variant}-{quant}
- * Examples: gemma-3-1b-it-wq4k-ef16-hf16, llama-3.2-1b-q4, mistral-7b-v0.3-q4
+ * Naming convention: {family}-{version}-{size}-{weights}[-{override-groups}]
+ * Examples: gemma-3-1b-it-q4k-ehf16-af32, llama-3.2-1b-q4, mistral-7b-v0.3-q4
  */
 export declare const MODEL_REQUIREMENTS: Record<string, ModelRequirements>;
 

package/src/storage/preflight.js

@@ -12,7 +12,7 @@ import { GB, MB } from '../config/schema/index.js';
 
 
 export const GEMMA_1B_REQUIREMENTS = {
-  modelId: 'gemma-3-1b-it-wq4k-ef16-hf16',
+  modelId: 'gemma-3-1b-it-q4k-ehf16-af32',
   displayName: 'Gemma 3 1B IT (Q4_K_M)',
   downloadSize: 565 * MB,       // ~565MB actual size from manifest shards
   vramRequired: 1.5 * GB,       // ~1.5GB VRAM for inference (weights + KV cache)
@@ -21,9 +21,31 @@ export const GEMMA_1B_REQUIREMENTS = {
   architecture: 'Gemma3ForCausalLM',
 };
 
+const GEMMA_270M_REQUIREMENTS = {
+  modelId: 'gemma-3-270m-it-q4k-ehf16-af32',
+  displayName: 'Gemma 3 270M IT (Q4K)',
+  downloadSize: 418512553,
+  vramRequired: 1.0 * GB,
+  paramCount: '270M',
+  quantization: 'Q4K',
+  architecture: 'Gemma3ForCausalLM',
+};
+
+const EMBEDDINGGEMMA_300M_REQUIREMENTS = {
+  modelId: 'google-embeddinggemma-300m-q4k-ehf16-af32',
+  displayName: 'EmbeddingGemma 300M (Q4K)',
+  downloadSize: 480562497,
+  vramRequired: 1.0 * GB,
+  paramCount: '300M',
+  quantization: 'Q4K',
+  architecture: 'EmbeddingGemmaForEmbedding',
+};
+
 
 export const MODEL_REQUIREMENTS = {
-  'gemma-3-1b-it-wq4k-ef16-hf16': GEMMA_1B_REQUIREMENTS,
+  'gemma-3-270m-it-q4k-ehf16-af32': GEMMA_270M_REQUIREMENTS,
+  'google-embeddinggemma-300m-q4k-ehf16-af32': EMBEDDINGGEMMA_300M_REQUIREMENTS,
+  'gemma-3-1b-it-q4k-ehf16-af32': GEMMA_1B_REQUIREMENTS,
 };
 
 // ============================================================================

package/src/storage/quickstart-downloader.js

@@ -3,7 +3,7 @@
 import { downloadModel } from './downloader.js';
 import {
   runPreflightChecks,
-  GEMMA_1B_REQUIREMENTS,
+  MODEL_REQUIREMENTS,
 } from './preflight.js';
 import { formatBytes } from './quota.js';
 import { getCdnBasePath } from './download-types.js';
@@ -53,11 +53,17 @@ export function getCDNBaseUrl() {
 
 
 export const QUICKSTART_MODELS = {
-  'gemma-3-1b-it-wq4k-ef16-hf16': {
-    modelId: 'gemma-3-1b-it-wq4k-ef16-hf16',
-    displayName: 'Gemma 3 1B IT (Q4_K_M)',
+  'gemma-3-270m-it-q4k-ehf16-af32': {
+    modelId: 'gemma-3-270m-it-q4k-ehf16-af32',
+    displayName: 'Gemma 3 270M IT (Q4K)',
     baseUrl: null,
-    requirements: GEMMA_1B_REQUIREMENTS,
+    requirements: MODEL_REQUIREMENTS['gemma-3-270m-it-q4k-ehf16-af32'],
+  },
+  'google-embeddinggemma-300m-q4k-ehf16-af32': {
+    modelId: 'google-embeddinggemma-300m-q4k-ehf16-af32',
+    displayName: 'EmbeddingGemma 300M (Q4K)',
+    baseUrl: null,
+    requirements: MODEL_REQUIREMENTS['google-embeddinggemma-300m-q4k-ehf16-af32'],
   },
 };
 

package/src/storage/registry.js

@@ -44,13 +44,16 @@ async function readRegistryFromIdb() {
   const store = createIdbStore(runtime.loading.storage.backend.indexeddb);
   await store.openModel(REGISTRY_MODEL_ID, { create: true });
   try {
-    const text = await store.readFile(REGISTRY_FILENAME);
+    const text = await store.readText(REGISTRY_FILENAME);
     await store.cleanup();
     if (!text) return { models: [] };
     return normalizeRegistry(JSON.parse(text));
-  } catch {
+  } catch (error) {
     await store.cleanup();
-    return { models: [] };
+    if (error?.message?.includes('not found')) {
+      return { models: [] };
+    }
+    throw error;
   }
 }
 
@@ -58,7 +61,10 @@ async function writeRegistryToIdb(registry) {
   const runtime = getRuntimeConfig();
   const store = createIdbStore(runtime.loading.storage.backend.indexeddb);
   await store.openModel(REGISTRY_MODEL_ID, { create: true });
-  await store.writeFile(REGISTRY_FILENAME, JSON.stringify(registry, null, 2));
+  await store.writeFile(
+    REGISTRY_FILENAME,
+    new TextEncoder().encode(JSON.stringify(registry, null, 2))
+  );
   await store.cleanup();
   return { backend: 'indexeddb', path: REGISTRY_FILENAME };
 }

package/src/storage/reports.js

@@ -48,7 +48,7 @@ async function saveReportToIdb(modelId, filename, payload) {
   const store = createIdbStore(idbConfig);
   const reportModelId = `${REPORT_MODEL_PREFIX}${modelId}`;
   await store.openModel(reportModelId, { create: true });
-  await store.writeFile(filename, payload);
+  await store.writeFile(filename, new TextEncoder().encode(payload));
   await store.cleanup();
   return `reports/${modelId}/${filename}`;
 }

package/src/storage/shard-manager.d.ts

@@ -84,6 +84,11 @@ export function createConversionShardWriter(
   shardIndex: number
 ): Promise<ShardWriteStream>;
 
+export function createFileWriter(
+  filename: string,
+  options?: ShardWriterOptions
+): Promise<ShardWriteStream>;
+
 export function loadShard(
   shardIndex: number,
   options?: ShardReadOptions
@@ -117,7 +122,15 @@ export function deleteModel(modelId: string): Promise<boolean>;
 export function listModels(): Promise<string[]>;
 export function listFilesInStore(): Promise<string[]>;
 export function loadFileFromStore(filename: string): Promise<ArrayBuffer>;
-export function streamFileFromStore(filename: string, options?: { chunkBytes?: number }): AsyncIterable<Uint8Array> | null;
+export function loadFileRangeFromStore(
+  filename: string,
+  offset?: number,
+  length?: number | null
+): Promise<ArrayBuffer>;
+export function streamFileFromStore(
+  filename: string,
+  options?: { chunkBytes?: number; offset?: number; length?: number | null }
+): AsyncIterable<Uint8Array> | null;
 export function getModelInfo(modelId: string): Promise<ModelInfo>;
 export function modelExists(modelId: string): Promise<boolean>;
 
@@ -133,5 +146,6 @@ export function loadTokenizerModelFromStore(): Promise<ArrayBuffer | null>;
 export function saveAuxFile(filename: string, data: string | ArrayBuffer | Uint8Array): Promise<void>;
 export function loadAuxFile(filename: string): Promise<ArrayBuffer | null>;
 export function loadAuxText(filename: string): Promise<string | null>;
+export function deleteFileFromStore(filename: string): Promise<boolean>;
 
 export function cleanup(): Promise<void>;

package/src/storage/shard-manager.js

@@ -341,6 +341,19 @@ export async function createConversionShardWriter(shardIndex) {
   return backend.createWriteStream(filename);
 }
 
+export async function createFileWriter(filename, options = {}) {
+  await ensureBackend();
+  requireModel();
+  if (!filename || typeof filename !== 'string') {
+    throw new Error('createFileWriter requires a filename');
+  }
+  if (!backend.createWriteStream) {
+    throw new Error('Storage backend does not support streaming writes');
+  }
+  const writerOptions = normalizeShardWriterOptions(options);
+  return backend.createWriteStream(filename, writerOptions);
+}
+
 export async function loadShard(shardIndex, options = { verify: false }) {
   await ensureBackend();
   requireModel();
@@ -581,6 +594,25 @@ export async function loadFileFromStore(filename) {
   return backend.readFile(filename);
 }
 
+export async function loadFileRangeFromStore(filename, offset = 0, length = null) {
+  await ensureBackend();
+  requireModel();
+  if (!filename || typeof filename !== 'string') {
+    throw new Error('loadFileRangeFromStore requires a filename');
+  }
+  const start = Number.isFinite(Number(offset)) ? Math.max(0, Math.floor(Number(offset))) : 0;
+  const want = length == null
+    ? null
+    : (Number.isFinite(Number(length)) ? Math.max(0, Math.floor(Number(length))) : 0);
+  if (backend && typeof backend.readFileRange === 'function') {
+    return backend.readFileRange(filename, start, want);
+  }
+  const buffer = await backend.readFile(filename);
+  const bytes = new Uint8Array(buffer);
+  const end = want == null ? bytes.byteLength : Math.min(bytes.byteLength, start + want);
+  return bytes.slice(start, end).buffer;
+}
+
 export function streamFileFromStore(filename, options = {}) {
   if (!backend || !filename || typeof filename !== 'string') {
     return null;
@@ -589,8 +621,12 @@ export function streamFileFromStore(filename, options = {}) {
   const runtimeDefault = runtime?.loading?.storage?.backend?.streaming?.readChunkBytes ?? (4 * 1024 * 1024);
   const raw = options.chunkBytes ?? runtimeDefault;
   const chunkBytes = Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : (4 * 1024 * 1024);
+  const offset = Number.isFinite(Number(options.offset)) ? Math.max(0, Math.floor(Number(options.offset))) : 0;
+  const length = options.length == null
+    ? null
+    : (Number.isFinite(Number(options.length)) ? Math.max(0, Math.floor(Number(options.length))) : 0);
   if (typeof backend.readFileRangeStream === 'function') {
-    return backend.readFileRangeStream(filename, 0, null, { chunkBytes });
+    return backend.readFileRangeStream(filename, offset, length, { chunkBytes });
   }
   return null;
 }
@@ -731,8 +767,11 @@ export async function loadTokenizerModelFromStore() {
   requireModel();
   try {
     return await backend.readFile('tokenizer.model');
-  } catch (_error) {
-    return null;
+  } catch (error) {
+    if (error?.name === 'NotFoundError' || error?.message?.includes('not found')) {
+      return null;
+    }
+    throw error;
   }
 }
 
@@ -771,6 +810,15 @@ export async function loadAuxFile(filename) {
   }
 }
 
+export async function deleteFileFromStore(filename) {
+  await ensureBackend();
+  requireModel();
+  if (!filename || typeof filename !== 'string') {
+    throw new Error('deleteFileFromStore requires a filename');
+  }
+  return backend.deleteFile(filename);
+}
+
 export async function loadAuxText(filename) {
   await ensureBackend();
   requireModel();

package/src/storage/source-artifact-store.d.ts

@@ -0,0 +1,52 @@
+import type { RDRRManifest } from '../formats/rdrr/types.js';
+import type {
+  SourceRuntimeMetadata,
+  SourceStorageContext,
+} from '../tooling/source-runtime-bundle.js';
+
+export interface StoredSourceArtifactFile {
+  path: string;
+  size: number | null;
+  hash: string | null;
+  hashAlgorithm: string | null;
+  kind: string;
+}
+
+export interface StoredSourceArtifactSourceFile extends StoredSourceArtifactFile {
+  index: number;
+}
+
+export interface StoredSourceArtifactDescriptor {
+  sourceRuntime: SourceRuntimeMetadata;
+  sourceFiles: StoredSourceArtifactSourceFile[];
+  auxiliaryFiles: StoredSourceArtifactFile[];
+  files: StoredSourceArtifactFile[];
+  totalBytes: number;
+  fingerprint: string;
+}
+
+export interface StoredSourceArtifactIntegrity {
+  valid: boolean;
+  missingFiles: string[];
+  corruptFiles: string[];
+}
+
+export declare function normalizeSourceArtifactPath(value: unknown): string;
+
+export declare function listSourceArtifactFiles(manifest: RDRRManifest): StoredSourceArtifactFile[];
+
+export declare function resolveSourceArtifact(
+  manifest: RDRRManifest
+): StoredSourceArtifactDescriptor | null;
+
+export declare function buildSourceArtifactFingerprint(manifest: RDRRManifest): string | null;
+
+export declare function verifyStoredSourceArtifact(
+  manifest: RDRRManifest,
+  options?: { checkHashes?: boolean }
+): Promise<StoredSourceArtifactIntegrity>;
+
+export declare function createStoredSourceArtifactContext(
+  manifest: RDRRManifest,
+  options?: { verifyHashes?: boolean }
+): SourceStorageContext;

package/src/storage/source-artifact-store.js

@@ -0,0 +1,234 @@
+import {
+  createSourceStorageContext,
+  getSourceRuntimeMetadata,
+} from '../tooling/source-runtime-bundle.js';
+import {
+  computeHash,
+  loadAuxText,
+  loadFileFromStore,
+  loadFileRangeFromStore,
+  streamFileFromStore,
+} from './shard-manager.js';
+
+export function normalizeSourceArtifactPath(value) {
+  return String(value || '').trim().replace(/\\/g, '/');
+}
+
+function encodeUtf8(value) {
+  return new TextEncoder().encode(String(value ?? ''));
+}
+
+function normalizeArtifactFile(entry, kind) {
+  const path = normalizeSourceArtifactPath(entry?.path);
+  if (!path) {
+    return null;
+  }
+  return {
+    path,
+    size: Number.isFinite(entry?.size) ? Math.max(0, Math.floor(Number(entry.size))) : null,
+    hash: typeof entry?.hash === 'string' && entry.hash.trim() ? entry.hash.trim().toLowerCase() : null,
+    hashAlgorithm: typeof entry?.hashAlgorithm === 'string' && entry.hashAlgorithm.trim()
+      ? entry.hashAlgorithm.trim().toLowerCase()
+      : null,
+    kind,
+  };
+}
+
+function pushArtifactFile(files, seen, entry, kind) {
+  const normalized = normalizeArtifactFile(entry, kind);
+  if (!normalized || seen.has(normalized.path)) {
+    return;
+  }
+  seen.add(normalized.path);
+  files.push(normalized);
+}
+
+function collectSourceArtifactFiles(sourceRuntime) {
+  if (!sourceRuntime) {
+    return {
+      sourceFiles: [],
+      auxiliaryFiles: [],
+      files: [],
+    };
+  }
+
+  const files = [];
+  const seen = new Set();
+  const sourceFiles = [];
+
+  for (let index = 0; index < sourceRuntime.sourceFiles.length; index += 1) {
+    const entry = sourceRuntime.sourceFiles[index];
+    const normalized = normalizeArtifactFile(entry, 'source');
+    if (!normalized) {
+      continue;
+    }
+    sourceFiles.push({
+      ...normalized,
+      index: Number.isFinite(entry?.index) ? Math.max(0, Math.floor(Number(entry.index))) : index,
+    });
+    pushArtifactFile(files, seen, entry, 'source');
+  }
+  for (const entry of sourceRuntime.auxiliaryFiles) {
+    pushArtifactFile(files, seen, entry, entry?.kind || 'auxiliary');
+  }
+
+  const tokenizer = sourceRuntime.tokenizer ?? {};
+  const auxiliaryByPath = new Map(
+    sourceRuntime.auxiliaryFiles.map((entry) => [normalizeSourceArtifactPath(entry?.path), entry])
+  );
+  for (const [path, kind] of [
+    [tokenizer.jsonPath, 'tokenizer_json'],
+    [tokenizer.configPath, 'tokenizer_config'],
+    [tokenizer.modelPath, 'tokenizer_model'],
+  ]) {
+    const normalizedPath = normalizeSourceArtifactPath(path);
+    if (!normalizedPath) {
+      continue;
+    }
+    pushArtifactFile(files, seen, auxiliaryByPath.get(normalizedPath) ?? { path: normalizedPath }, kind);
+  }
+
+  files.sort((left, right) => left.path.localeCompare(right.path));
+  return {
+    sourceFiles,
+    auxiliaryFiles: files.filter((entry) => entry.kind !== 'source'),
+    files,
+  };
+}
+
+export function listSourceArtifactFiles(manifest) {
+  const sourceRuntime = getSourceRuntimeMetadata(manifest);
+  return collectSourceArtifactFiles(sourceRuntime).files;
+}
+
+export function resolveSourceArtifact(manifest) {
+  const sourceRuntime = getSourceRuntimeMetadata(manifest);
+  if (!sourceRuntime) {
+    return null;
+  }
+  const { sourceFiles, auxiliaryFiles, files } = collectSourceArtifactFiles(sourceRuntime);
+  const totalBytes = files.reduce((sum, entry) => sum + (entry.size || 0), 0);
+  return {
+    sourceRuntime,
+    sourceFiles,
+    auxiliaryFiles,
+    files,
+    totalBytes,
+    fingerprint: JSON.stringify({
+      mode: sourceRuntime.mode,
+      schema: sourceRuntime.schema,
+      hashAlgorithm: sourceRuntime.hashAlgorithm,
+      pathSemantics: sourceRuntime.pathSemantics,
+      sourceKind: sourceRuntime.sourceKind,
+      files: files.map((entry) => ({
+        path: entry.path,
+        size: entry.size,
+        hash: entry.hash,
+        hashAlgorithm: entry.hashAlgorithm,
+        kind: entry.kind,
+      })),
+    }),
+  };
+}
+
+export function buildSourceArtifactFingerprint(manifest) {
+  return resolveSourceArtifact(manifest)?.fingerprint ?? null;
+}
+
+async function loadStoreFile(path) {
+  try {
+    return await loadFileFromStore(path);
+  } catch (error) {
+    const message = String(error?.message || '');
+    if (error?.name === 'NotFoundError' || message.toLowerCase().includes('not found')) {
+      return null;
+    }
+    throw error;
+  }
+}
+
+export async function verifyStoredSourceArtifact(manifest, options = {}) {
+  const sourceRuntime = getSourceRuntimeMetadata(manifest);
+  if (!sourceRuntime) {
+    throw new Error('verifyStoredSourceArtifact requires a direct-source manifest.');
+  }
+
+  const checkHashes = options.checkHashes !== false;
+  const missingFiles = [];
+  const corruptFiles = [];
+  const files = listSourceArtifactFiles(manifest);
+
+  for (const entry of files) {
+    const payload = await loadStoreFile(entry.path);
+    if (!(payload instanceof ArrayBuffer)) {
+      missingFiles.push(entry.path);
+      continue;
+    }
+    if (!checkHashes || !entry.hash) {
+      continue;
+    }
+    const isTextAsset = entry.kind === 'config'
+      || entry.kind === 'tokenizer_json'
+      || entry.kind === 'tokenizer_config'
+      || entry.kind === 'safetensors_index';
+    const computedHash = isTextAsset
+      ? await computeHash(encodeUtf8(new TextDecoder().decode(payload)), entry.hashAlgorithm || sourceRuntime.hashAlgorithm)
+      : await computeHash(new Uint8Array(payload), entry.hashAlgorithm || sourceRuntime.hashAlgorithm);
+    if (computedHash !== entry.hash) {
+      corruptFiles.push(entry.path);
+    }
+  }
+
+  return {
+    valid: missingFiles.length === 0 && (!checkHashes || corruptFiles.length === 0),
+    missingFiles,
+    corruptFiles,
+  };
+}
+
+export function createStoredSourceArtifactContext(manifest, options = {}) {
+  const sourceRuntime = getSourceRuntimeMetadata(manifest);
+  if (!sourceRuntime) {
+    throw new Error('createStoredSourceArtifactContext requires a direct-source manifest.');
+  }
+
+  const readRange = async (path, offset, length) => loadFileRangeFromStore(path, offset, length);
+  const streamRange = (path, offset, length, streamOptions = {}) => {
+    const stream = streamFileFromStore(path, {
+      chunkBytes: streamOptions?.chunkBytes,
+      offset,
+      length,
+    });
+    if (!stream) {
+      return null;
+    }
+    return stream;
+  };
+  const readText = async (path) => loadAuxText(path);
+  const readBinary = async (path) => {
+    const payload = await loadStoreFile(path);
+    if (!(payload instanceof ArrayBuffer)) {
+      throw new Error(`Missing stored source binary file: ${path}`);
+    }
+    return payload;
+  };
+
+  return createSourceStorageContext({
+    manifest,
+    readRange,
+    streamRange: streamRange ? (async function* (path, offset, length, streamOptions = {}) {
+      const stream = streamRange(path, offset, length, streamOptions);
+      if (!stream) {
+        const payload = await loadFileRangeFromStore(path, offset, length);
+        yield new Uint8Array(payload);
+        return;
+      }
+      for await (const chunk of stream) {
+        yield chunk;
+      }
+    }) : null,
+    readText,
+    readBinary,
+    verifyHashes: options.verifyHashes !== false,
+  });
+}

package/src/tooling/command-api-constants.d.ts

@@ -0,0 +1,9 @@
+export declare const TOOLING_COMMAND_SET: readonly string[];
+export declare const TOOLING_SURFACE_SET: readonly string[];
+export declare const TOOLING_SUITE_SET: readonly string[];
+export declare const TOOLING_INTENT_SET: readonly string[];
+export declare const VERIFY_SUITES: readonly string[];
+export declare const TRAINING_STAGE_SET: readonly string[];
+export declare const DISTILL_ACTION_SET: readonly string[];
+export declare const LORA_ACTION_SET: readonly string[];
+export declare const TRAINING_COMMAND_SCHEMA_VERSION: number;

package/src/tooling/command-api-constants.js

@@ -0,0 +1,9 @@
+export const TOOLING_COMMAND_SET = ['convert', 'debug', 'bench', 'verify', 'lora', 'distill'];
+export const TOOLING_SURFACE_SET = ['browser', 'node'];
+export const TOOLING_SUITE_SET = ['kernels', 'inference', 'training', 'bench', 'debug', 'diffusion', 'energy'];
+export const TOOLING_INTENT_SET = ['verify', 'investigate', 'calibrate'];
+export const VERIFY_SUITES = ['kernels', 'inference', 'training', 'diffusion', 'energy'];
+export const TRAINING_STAGE_SET = ['stage1_joint', 'stage2_base', 'stage_a', 'stage_b'];
+export const DISTILL_ACTION_SET = ['run', 'stage-a', 'stage-b', 'eval', 'watch', 'compare', 'quality-gate', 'subsets'];
+export const LORA_ACTION_SET = ['run', 'eval', 'watch', 'export', 'compare', 'quality-gate', 'activate'];
+export const TRAINING_COMMAND_SCHEMA_VERSION = 1;

package/src/tooling/command-api-family-normalizers.d.ts

@@ -0,0 +1,9 @@
+export declare function normalizeConvert(raw: Record<string, unknown>): Record<string, unknown>;
+export declare function normalizeTrainingOperatorCommand(
+  raw: Record<string, unknown>,
+  command: string
+): Record<string, unknown>;
+export declare function normalizeSuiteCommand(
+  raw: Record<string, unknown>,
+  command: string
+): Record<string, unknown>;