@simulatte/doppler 0.1.6 → 0.1.8

package/src/distribution/shard-delivery.js

@@ -1,4 +1,5 @@
 import { log } from '../debug/index.js';
+import { getExpectedShardHash } from '../formats/rdrr/index.js';
 import {
   computeHash,
   createStreamingHasher,
@@ -55,22 +56,30 @@ const inFlightDeliveries = new Map();
 const p2pTransportPolicyState = new WeakMap();
 
 function normalizeDistributionSourceOrder(rawSources = []) {
-  if (!Array.isArray(rawSources)) {
+  if (rawSources === undefined || rawSources === null) {
     return [...DISTRIBUTION_SOURCES];
   }
+  if (!Array.isArray(rawSources)) {
+    throw new Error('distribution.sourceOrder must be an array when provided.');
+  }
 
   const normalized = [];
   const seen = new Set();
 
   for (const value of rawSources) {
     const source = String(value || '').trim().toLowerCase();
-    if (!DISTRIBUTION_SOURCES.includes(source)) continue;
+    if (!DISTRIBUTION_SOURCES.includes(source)) {
+      throw new Error(`distribution.sourceOrder contains unsupported source "${source || value}".`);
+    }
     if (seen.has(source)) continue;
     seen.add(source);
     normalized.push(source);
   }
 
-  return normalized.length > 0 ? normalized : [...DISTRIBUTION_SOURCES];
+  if (normalized.length === 0) {
+    throw new Error('distribution.sourceOrder must include at least one supported source.');
+  }
+  return normalized;
 }
 
 function normalizeInteger(value, fallback, allowZero = false) {
@@ -81,6 +90,23 @@ function normalizeInteger(value, fallback, allowZero = false) {
     : fallback;
 }
 
+function normalizeRequiredInteger(value, label, { allowZero = false, fallback = null } = {}) {
+  if (value === undefined || value === null) {
+    if (fallback !== null) {
+      return fallback;
+    }
+    throw new Error(`${label} is required.`);
+  }
+  const parsed = Number(value);
+  const min = allowZero ? 0 : 1;
+  if (!Number.isInteger(parsed) || parsed < min) {
+    throw new Error(
+      `${label} must be a ${allowZero ? 'non-negative' : 'positive'} integer when provided.`
+    );
+  }
+  return parsed;
+}
+
 function normalizeContentEncodings(value) {
   if (!value) return [];
   return value
@@ -95,13 +121,17 @@ function normalizeManifestVersionSet(value) {
   return normalized || null;
 }
 
-function normalizeSamplingRate(value, fallback = 1) {
+function normalizeSamplingRate(value, fallback = 1, label = 'distribution.sourceDecision.trace.samplingRate') {
+  if (value === undefined || value === null) {
+    return fallback;
+  }
   const parsed = Number(value);
   if (!Number.isFinite(parsed)) {
-    return fallback;
+    throw new Error(`${label} must be a finite number between 0 and 1 when provided.`);
+  }
+  if (parsed < 0 || parsed > 1) {
+    throw new Error(`${label} must be between 0 and 1 when provided.`);
   }
-  if (parsed <= 0) return 0;
-  if (parsed >= 1) return 1;
   return parsed;
 }
 
@@ -479,19 +509,28 @@ function normalizeP2PConfig(config = {}) {
 
   return {
     enabled,
-    timeoutMs: normalizeInteger(rawTimeoutMs, DEFAULT_P2P_TIMEOUT_MS),
-    maxRetries: normalizeInteger(rawMaxRetries, DEFAULT_P2P_MAX_RETRIES, true),
-    retryDelayMs: normalizeInteger(rawRetryDelayMs, DEFAULT_P2P_RETRY_DELAY_MS, true),
+    timeoutMs: normalizeRequiredInteger(
+      rawTimeoutMs,
+      'distribution.p2p.timeoutMs',
+      { fallback: DEFAULT_P2P_TIMEOUT_MS }
+    ),
+    maxRetries: normalizeRequiredInteger(
+      rawMaxRetries,
+      'distribution.p2p.maxRetries',
+      { allowZero: true, fallback: DEFAULT_P2P_MAX_RETRIES }
+    ),
+    retryDelayMs: normalizeRequiredInteger(
+      rawRetryDelayMs,
+      'distribution.p2p.retryDelayMs',
+      { allowZero: true, fallback: DEFAULT_P2P_RETRY_DELAY_MS }
+    ),
     transport,
     contractVersion,
     controlPlane: normalizeP2PControlPlaneConfig({
       ...DEFAULT_DISTRIBUTION_CONFIG.p2p.controlPlane,
       ...rawControlPlane,
-      tokenRefreshSkewMs: normalizeInteger(
-        rawControlPlane.tokenRefreshSkewMs,
-        DEFAULT_P2P_CONTROL_PLANE_TOKEN_REFRESH_SKEW_MS,
-        true
-      ),
+      tokenRefreshSkewMs: rawControlPlane.tokenRefreshSkewMs
+        ?? DEFAULT_P2P_CONTROL_PLANE_TOKEN_REFRESH_SKEW_MS,
     }),
     security: {
       requireSessionToken: rawSecurity.requireSessionToken === true,
@@ -499,19 +538,20 @@ function normalizeP2PConfig(config = {}) {
       tokenExpiresAtMs: normalizeOptionalTimestamp(rawSecurity.tokenExpiresAtMs),
     },
     abuse: {
-      rateLimitPerMinute: normalizeInteger(
+      rateLimitPerMinute: normalizeRequiredInteger(
         rawAbuse.rateLimitPerMinute,
-        DEFAULT_P2P_RATE_LIMIT_PER_MINUTE,
-        true
+        'distribution.p2p.abuse.rateLimitPerMinute',
+        { allowZero: true, fallback: DEFAULT_P2P_RATE_LIMIT_PER_MINUTE }
       ),
-      maxConsecutiveFailures: normalizeInteger(
+      maxConsecutiveFailures: normalizeRequiredInteger(
         rawAbuse.maxConsecutiveFailures,
-        DEFAULT_P2P_MAX_CONSECUTIVE_FAILURES
+        'distribution.p2p.abuse.maxConsecutiveFailures',
+        { fallback: DEFAULT_P2P_MAX_CONSECUTIVE_FAILURES }
       ),
-      quarantineMs: normalizeInteger(
+      quarantineMs: normalizeRequiredInteger(
         rawAbuse.quarantineMs,
-        DEFAULT_P2P_QUARANTINE_MS,
-        true
+        'distribution.p2p.abuse.quarantineMs',
+        { allowZero: true, fallback: DEFAULT_P2P_QUARANTINE_MS }
       ),
     },
   };
@@ -1293,9 +1333,21 @@ async function downloadShardFromHttp(baseUrl, shardInfo, shardIndex, options = {
   const startTime = performance.now();
   const url = buildShardUrl(baseUrl, shardInfo);
   let lastError;
-  const maxRetries = normalizeInteger(options.maxRetries, 3, true);
-  const initialRetryDelayMs = normalizeInteger(options.initialRetryDelayMs, 1000);
-  const maxRetryDelayMs = normalizeInteger(options.maxRetryDelayMs, 30000);
+  const maxRetries = normalizeRequiredInteger(
+    options.maxRetries,
+    'download.maxRetries',
+    { allowZero: true, fallback: 3 }
+  );
+  const initialRetryDelayMs = normalizeRequiredInteger(
+    options.initialRetryDelayMs,
+    'download.initialRetryDelayMs',
+    { allowZero: true, fallback: 1000 }
+  );
+  const maxRetryDelayMs = normalizeRequiredInteger(
+    options.maxRetryDelayMs,
+    'download.maxRetryDelayMs',
+    { allowZero: true, fallback: 30000 }
+  );
   const progressTotalBytes = Number.isFinite(options.expectedSize)
     ? Math.floor(options.expectedSize)
     : (Number.isFinite(shardInfo?.size) ? Math.floor(shardInfo.size) : 0);
@@ -1967,7 +2019,11 @@ export async function downloadShard(
     onDeliveryMetrics,
     signal,
     requiredEncoding: requiredEncoding ?? activeConfig.requiredContentEncoding ?? null,
-    expectedHash: options.expectedHash ?? shardInfo?.hash ?? activeConfig.expectedHash ?? null,
+    expectedHash:
+      options.expectedHash
+      ?? getExpectedShardHash(shardInfo, algorithm)
+      ?? activeConfig.expectedHash
+      ?? null,
     expectedSize: expectedSize ?? shardInfo?.size ?? null,
     expectedManifestVersionSet: options.expectedManifestVersionSet ?? null,
     writeToStore,

package/src/formats/gguf/types.js

@@ -94,6 +94,8 @@ export const GGML_TYPE_SIZE = {
 const GGUF_MAGIC = 0x46554747;
 const GGUF_VERSION_MIN = 2;
 const GGUF_VERSION_MAX = 3;
+const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);
+const MIN_SAFE_BIGINT = BigInt(Number.MIN_SAFE_INTEGER);
 
 const {
   contextLength: DEFAULT_GGUF_CONTEXT_LENGTH,
@@ -102,6 +104,13 @@ const {
   ropeFreqBase: DEFAULT_ROPE_FREQ_BASE,
 } = DEFAULT_GGUF_PARSER_DEFAULTS;
 
+function toSafeInteger(value, label) {
+  if (value > MAX_SAFE_BIGINT || value < MIN_SAFE_BIGINT) {
+    throw new Error(`GGUF ${label} exceeds JavaScript safe integer range: ${value.toString()}`);
+  }
+  return Number(value);
+}
+
 class GGUFReader {
   constructor(buffer) {
     this.view = new DataView(buffer);
@@ -144,18 +153,26 @@ class GGUFReader {
     return value;
   }
 
-  readUint64() {
-    const low = this.view.getUint32(this.offset, true);
-    const high = this.view.getUint32(this.offset + 4, true);
+  readUint64BigInt() {
+    const low = BigInt(this.view.getUint32(this.offset, true));
+    const high = BigInt(this.view.getUint32(this.offset + 4, true));
     this.offset += 8;
-    return high * 0x100000000 + low;
+    return (high << 32n) | low;
   }
 
-  readInt64() {
-    const low = this.view.getUint32(this.offset, true);
-    const high = this.view.getInt32(this.offset + 4, true);
+  readUint64(label = 'u64 value') {
+    return toSafeInteger(this.readUint64BigInt(), label);
+  }
+
+  readInt64BigInt() {
+    const low = BigInt(this.view.getUint32(this.offset, true));
+    const high = BigInt(this.view.getInt32(this.offset + 4, true));
     this.offset += 8;
-    return high * 0x100000000 + low;
+    return (high << 32n) | low;
+  }
+
+  readInt64(label = 'i64 value') {
+    return toSafeInteger(this.readInt64BigInt(), label);
   }
 
   readFloat32() {
@@ -175,7 +192,7 @@ class GGUFReader {
   }
 
   readString() {
-    const length = this.readUint64();
+    const length = this.readUint64('string length');
     const bytes = new Uint8Array(this.view.buffer, this.offset, length);
     this.offset += length;
     return new TextDecoder().decode(bytes);
@@ -196,9 +213,9 @@ class GGUFReader {
       case GGUFValueType.INT32:
         return this.readInt32();
       case GGUFValueType.UINT64:
-        return this.readUint64();
+        return this.readUint64('metadata uint64');
       case GGUFValueType.INT64:
-        return this.readInt64();
+        return this.readInt64('metadata int64');
       case GGUFValueType.FLOAT32:
         return this.readFloat32();
       case GGUFValueType.FLOAT64:
@@ -216,7 +233,7 @@ class GGUFReader {
 
   readArray() {
     const elementType = this.readUint32();
-    const length = this.readUint64();
+    const length = this.readUint64('array length');
     if (length > 10000000) {
       throw new Error(`Array too long: ${length}`);
     }
@@ -331,8 +348,8 @@ export function parseGGUF(buffer) {
     throw new Error(`Unsupported GGUF version: ${version}`);
   }
 
-  const tensorCount = reader.readUint64();
-  const metadataKVCount = reader.readUint64();
+  const tensorCount = reader.readUint64('tensor count');
+  const metadataKVCount = reader.readUint64('metadata count');
 
   const metadata = {};
   for (let i = 0; i < metadataKVCount; i++) {
@@ -351,10 +368,10 @@ export function parseGGUF(buffer) {
     const nDims = reader.readUint32();
     const shape = [];
     for (let d = 0; d < nDims; d++) {
-      shape.push(reader.readUint64());
+      shape.push(reader.readUint64(`tensor "${name}" shape[${d}]`));
     }
     const type = reader.readUint32();
-    const offset = reader.readUint64();
+    const offset = reader.readUint64(`tensor "${name}" offset`);
 
     tensors.push({
       name,

package/src/formats/rdrr/groups.d.ts

@@ -6,7 +6,7 @@
  * @module formats/rdrr/groups
  */
 
-import type { ComponentGroup } from './types.js';
+import type { ComponentGroup, RDRRManifest } from './types.js';
 
 export declare function getGroup(groupId: string): ComponentGroup | null;
 
@@ -16,11 +16,19 @@ export declare function getShardsForGroup(groupId: string): number[];
 
 export declare function getTensorsForGroup(groupId: string): string[];
 
-export declare function getShardsForExpert(layerIdx: number, expertIdx: number): number[];
+export declare function getShardsForExpert(
+  layerIdx: number,
+  expertIdx: number,
+  manifest?: RDRRManifest | null
+): number[];
 
-export declare function getTensorsForExpert(layerIdx: number, expertIdx: number): string[];
+export declare function getTensorsForExpert(
+  layerIdx: number,
+  expertIdx: number,
+  manifest?: RDRRManifest | null
+): string[];
 
-export declare function getExpertBytes(): number;
+export declare function getExpertBytes(manifest?: RDRRManifest | null): number;
 
 export declare function getLayerGroupIds(): string[];
 

package/src/formats/rdrr/groups.js

@@ -19,8 +19,7 @@ export function getTensorsForGroup(groupId) {
   return getManifest()?.groups?.[groupId]?.tensors ?? [];
 }
 
-export function getShardsForExpert(layerIdx, expertIdx) {
-  const manifest = getManifest();
+export function getShardsForExpert(layerIdx, expertIdx, manifest = getManifest()) {
   const groupId = `layer.${layerIdx}.expert.${expertIdx}`;
   const group = manifest?.groups?.[groupId];
   if (group) {
@@ -29,8 +28,7 @@ export function getShardsForExpert(layerIdx, expertIdx) {
   throw new Error(`Missing expert group mapping: ${groupId}`);
 }
 
-export function getTensorsForExpert(layerIdx, expertIdx) {
-  const manifest = getManifest();
+export function getTensorsForExpert(layerIdx, expertIdx, manifest = getManifest()) {
   const groupId = `layer.${layerIdx}.expert.${expertIdx}`;
   const group = manifest?.groups?.[groupId];
   if (group) {
@@ -39,8 +37,7 @@ export function getTensorsForExpert(layerIdx, expertIdx) {
   throw new Error(`Missing expert group mapping: ${groupId}`);
 }
 
-export function getExpertBytes() {
-  const manifest = getManifest();
+export function getExpertBytes(manifest = getManifest()) {
   const expertGroups = Object.entries(manifest?.groups || {})
     .filter(([id]) => id.includes('.expert.'));
 

package/src/formats/rdrr/parsing.d.ts

@@ -7,6 +7,10 @@
 import type { RDRRManifest, ShardInfo, TensorMap } from './types.js';
 
 export declare function parseManifest(jsonString: string): RDRRManifest;
+export declare function getExpectedShardHash(
+  shard: Partial<ShardInfo> | Record<string, unknown> | null | undefined,
+  manifestHashAlgorithm?: string | null
+): string;
 
 export declare function parseTensorMap(jsonString: string): TensorMap;
 

package/src/formats/rdrr/parsing.js

@@ -4,6 +4,19 @@ import { validateManifest } from './validation.js';
 
 let currentManifest = null;
 
+export function getExpectedShardHash(shard, manifestHashAlgorithm = null) {
+  if (!shard || typeof shard !== 'object' || Array.isArray(shard)) {
+    return '';
+  }
+  const algorithm = typeof manifestHashAlgorithm === 'string'
+    ? manifestHashAlgorithm.trim().toLowerCase()
+    : '';
+  if (algorithm === 'blake3') {
+    return shard.blake3 || shard.hash || '';
+  }
+  return shard.hash || shard.blake3 || '';
+}
+
 export function parseManifest(jsonString) {
   let manifest;
 
@@ -21,7 +34,7 @@ export function parseManifest(jsonString) {
         index: shard.index ?? i,
         filename: shard.filename || shard.fileName || '',
         size: shard.size,
-        hash: shard.hash || shard.blake3 || '',
+        hash: getExpectedShardHash(shard, manifest.hashAlgorithm),
         blake3: shard.blake3 || shard.hash,
         offset: shard.offset ?? offset,
         hashAlgorithm: shard.hashAlgorithm,
@@ -44,9 +57,13 @@ export function parseManifest(jsonString) {
 export function parseTensorMap(jsonString) {
   try {
     const tensorMap = JSON.parse(jsonString);
+    const normalizedTensorMap = {};
 
     for (const [name, loc] of Object.entries(tensorMap)) {
-      if (typeof loc.shard !== 'number') {
+      const shardIndex = typeof loc.shardIndex === 'number'
+        ? loc.shardIndex
+        : loc.shard;
+      if (typeof shardIndex !== 'number') {
         throw new Error(`Tensor '${name}' missing shard index`);
       }
       if (typeof loc.offset !== 'number') {
@@ -61,9 +78,42 @@ export function parseTensorMap(jsonString) {
       if (typeof loc.role !== 'string') {
         throw new Error(`Tensor '${name}' missing role`);
       }
+
+      let spans = undefined;
+      if (loc.spans !== undefined) {
+        if (!Array.isArray(loc.spans)) {
+          throw new Error(`Tensor '${name}' has invalid spans array`);
+        }
+        spans = loc.spans.map((span, spanIndex) => {
+          const spanShardIndex = typeof span?.shardIndex === 'number'
+            ? span.shardIndex
+            : span?.shard;
+          if (typeof spanShardIndex !== 'number') {
+            throw new Error(`Tensor '${name}' span[${spanIndex}] missing shard index`);
+          }
+          if (typeof span?.offset !== 'number') {
+            throw new Error(`Tensor '${name}' span[${spanIndex}] missing offset`);
+          }
+          if (typeof span?.size !== 'number') {
+            throw new Error(`Tensor '${name}' span[${spanIndex}] missing size`);
+          }
+          return {
+            shardIndex: spanShardIndex,
+            offset: span.offset,
+            size: span.size,
+          };
+        });
+      }
+
+      normalizedTensorMap[name] = {
+        ...loc,
+        shard: shardIndex,
+        shardIndex,
+        spans,
+      };
     }
 
-    return tensorMap;
+    return normalizedTensorMap;
   } catch (e) {
     if (e instanceof Error && e.message.includes('Tensor')) {
       throw e;

package/src/formats/rdrr/types.d.ts

@@ -75,13 +75,14 @@ export interface ComponentGroup extends ComponentGroupSchema {}
 
 export interface TensorLocation {
   shard: number;
+  shardIndex?: number;
   offset: number;
   size: number;
   shape: number[];
   dtype: string;
   role: TensorRole;
   group?: string;
-  spans?: Array<{ shardIndex: number; offset: number; size: number }>;
+  spans?: Array<{ shard?: number; shardIndex?: number; offset: number; size: number }>;
   layout?: WeightLayout;
   originalShape?: number[];
 }