@simulatte/doppler 0.1.9 → 0.2.0

package/src/formats/rdrr/manifest.js

@@ -1,108 +0,0 @@
-
-
-import {
-  RDRR_VERSION,
-  SHARD_SIZE,
-} from './types.js';
-import { validateManifest } from './validation.js';
-import { getShardInfo } from './parsing.js';
-import { createDopplerError, ERROR_CODES } from '../../errors/index.js';
-
-export function generateShardFilename(index) {
-  return `shard_${String(index).padStart(5, '0')}.bin`;
-}
-
-export function calculateShardCount(totalSize, shardSize = SHARD_SIZE) {
-  return Math.ceil(totalSize / shardSize);
-}
-
-export function createShardLayout(
-  totalSize,
-  hashes,
-  shardSize = SHARD_SIZE
-) {
-  const numShards = calculateShardCount(totalSize, shardSize);
-
-  if (hashes.length !== numShards) {
-    throw new Error(`Hash count mismatch: expected ${numShards}, got ${hashes.length}`);
-  }
-
-  const shards = [];
-  let offset = 0;
-
-  for (let i = 0; i < numShards; i++) {
-    const isLast = i === numShards - 1;
-    const size = isLast ? totalSize - offset : shardSize;
-
-    shards.push({
-      index: i,
-      filename: generateShardFilename(i),
-      size,
-      hash: hashes[i],
-      offset,
-    });
-
-    offset += size;
-  }
-
-  return shards;
-}
-
-export function createManifest(options) {
-  const manifest = {
-    version: RDRR_VERSION,
-    modelId: options.modelId,
-    modelType: options.modelType,
-    quantization: options.quantization,
-    quantizationInfo: options.quantizationInfo,
-    hashAlgorithm: options.hashAlgorithm,
-    architecture: options.architecture,
-    groups: options.groups,
-    shards: options.shards,
-    totalSize: options.totalSize,
-    tensorsFile: options.tensorsFile,
-    tensorCount: options.tensorCount,
-    tokenizer: options.tokenizer,
-    moeConfig: options.moeConfig,
-    config: options.config,
-    conversion: options.conversion,
-    blake3Full: options.blake3Full,
-    metadata: options.metadata,
-    inference: options.inference,
-  };
-
-  const validation = validateManifest(manifest);
-  if (!validation.valid) {
-    throw createDopplerError(
-      ERROR_CODES.LOADER_MANIFEST_INVALID,
-      `Created invalid manifest:\n  - ${validation.errors.join('\n  - ')}`
-    );
-  }
-
-  return manifest;
-}
-
-export function serializeTensorMap(tensorMap) {
-  return JSON.stringify(tensorMap, null, 2);
-}
-
-export function serializeManifest(manifest) {
-  return JSON.stringify(manifest, null, 2);
-}
-
-export function getShardUrl(baseUrl, shardIndex) {
-  const shard = getShardInfo(shardIndex);
-  if (!shard) {
-    throw createDopplerError(
-      ERROR_CODES.LOADER_SHARD_INDEX_INVALID,
-      `Invalid shard index: ${shardIndex}`
-    );
-  }
-  const base = baseUrl.replace(/\/$/, '');
-  return `${base}/${shard.filename}`;
-}
-
-export function getManifestUrl(baseUrl) {
-  const base = baseUrl.replace(/\/$/, '');
-  return `${base}/manifest.json`;
-}

package/src/formats/rdrr/parsing.d.ts

@@ -1,27 +0,0 @@
-/**
- * RDRR Parsing Functions
- *
- * @module formats/rdrr/parsing
- */
-
-import type { RDRRManifest, ShardInfo, TensorMap } from './types.js';
-
-export declare function parseManifest(jsonString: string): RDRRManifest;
-export declare function getExpectedShardHash(
-  shard: Partial<ShardInfo> | Record<string, unknown> | null | undefined,
-  manifestHashAlgorithm?: string | null
-): string;
-
-export declare function parseTensorMap(jsonString: string): TensorMap;
-
-export declare function getManifest(): RDRRManifest | null;
-
-export declare function setManifest(manifest: RDRRManifest): void;
-
-export declare function clearManifest(): void;
-
-export declare function getShardInfo(index: number): ShardInfo | null;
-
-export declare function getShardCount(): number;
-
-export declare function isMoE(): boolean;

package/src/formats/rdrr/parsing.js

@@ -1,151 +0,0 @@
-
-
-import { validateManifest } from './validation.js';
-
-let currentManifest = null;
-
-export function getExpectedShardHash(shard, manifestHashAlgorithm = null) {
-  if (!shard || typeof shard !== 'object' || Array.isArray(shard)) {
-    return '';
-  }
-  const algorithm = typeof manifestHashAlgorithm === 'string'
-    ? manifestHashAlgorithm.trim().toLowerCase()
-    : '';
-  if (algorithm === 'blake3') {
-    return shard.blake3 || shard.hash || '';
-  }
-  return shard.hash || shard.blake3 || '';
-}
-
-export function parseManifest(jsonString) {
-  let manifest;
-
-  try {
-    manifest = JSON.parse(jsonString);
-  } catch (e) {
-    throw new Error(`Failed to parse manifest JSON: ${e.message}`);
-  }
-
-  // Normalize shards (handle fileName vs filename, compute offset if missing)
-  if (Array.isArray(manifest.shards)) {
-    let offset = 0;
-    manifest.shards = manifest.shards.map((shard, i) => {
-      const normalized = {
-        index: shard.index ?? i,
-        filename: shard.filename || shard.fileName || '',
-        size: shard.size,
-        hash: getExpectedShardHash(shard, manifest.hashAlgorithm),
-        blake3: shard.blake3 || shard.hash,
-        offset: shard.offset ?? offset,
-        hashAlgorithm: shard.hashAlgorithm,
-      };
-      offset += shard.size;
-      return normalized;
-    });
-  }
-
-  // Validate
-  const validation = validateManifest(manifest);
-  if (!validation.valid) {
-    throw new Error(`Invalid manifest:\n  - ${validation.errors.join('\n  - ')}`);
-  }
-
-  currentManifest = manifest;
-  return manifest;
-}
-
-export function parseTensorMap(jsonString) {
-  try {
-    const tensorMap = JSON.parse(jsonString);
-    const normalizedTensorMap = {};
-
-    for (const [name, loc] of Object.entries(tensorMap)) {
-      const shardIndex = typeof loc.shardIndex === 'number'
-        ? loc.shardIndex
-        : loc.shard;
-      if (typeof shardIndex !== 'number') {
-        throw new Error(`Tensor '${name}' missing shard index`);
-      }
-      if (typeof loc.offset !== 'number') {
-        throw new Error(`Tensor '${name}' missing offset`);
-      }
-      if (typeof loc.size !== 'number') {
-        throw new Error(`Tensor '${name}' missing size`);
-      }
-      if (!Array.isArray(loc.shape)) {
-        throw new Error(`Tensor '${name}' missing shape`);
-      }
-      if (typeof loc.role !== 'string') {
-        throw new Error(`Tensor '${name}' missing role`);
-      }
-
-      let spans = undefined;
-      if (loc.spans !== undefined) {
-        if (!Array.isArray(loc.spans)) {
-          throw new Error(`Tensor '${name}' has invalid spans array`);
-        }
-        spans = loc.spans.map((span, spanIndex) => {
-          const spanShardIndex = typeof span?.shardIndex === 'number'
-            ? span.shardIndex
-            : span?.shard;
-          if (typeof spanShardIndex !== 'number') {
-            throw new Error(`Tensor '${name}' span[${spanIndex}] missing shard index`);
-          }
-          if (typeof span?.offset !== 'number') {
-            throw new Error(`Tensor '${name}' span[${spanIndex}] missing offset`);
-          }
-          if (typeof span?.size !== 'number') {
-            throw new Error(`Tensor '${name}' span[${spanIndex}] missing size`);
-          }
-          return {
-            shardIndex: spanShardIndex,
-            offset: span.offset,
-            size: span.size,
-          };
-        });
-      }
-
-      normalizedTensorMap[name] = {
-        ...loc,
-        shard: shardIndex,
-        shardIndex,
-        spans,
-      };
-    }
-
-    return normalizedTensorMap;
-  } catch (e) {
-    if (e instanceof Error && e.message.includes('Tensor')) {
-      throw e;
-    }
-    throw new Error(`Failed to parse tensors.json: ${e.message}`);
-  }
-}
-
-export function getManifest() {
-  return currentManifest;
-}
-
-export function setManifest(manifest) {
-  currentManifest = manifest;
-}
-
-export function clearManifest() {
-  currentManifest = null;
-}
-
-export function getShardInfo(index) {
-  if (!currentManifest || index < 0 || index >= currentManifest.shards.length) {
-    return null;
-  }
-  return currentManifest.shards[index];
-}
-
-export function getShardCount() {
-  return currentManifest?.shards?.length ?? 0;
-}
-
-export function isMoE() {
-  return currentManifest?.moeConfig != null ||
-    Object.keys(currentManifest?.groups || {}).some(g => g.includes('.expert.'));
-}

package/src/formats/rdrr/tensor-config-validator.d.ts

@@ -1,42 +0,0 @@
-/**
- * Tensor-Config Consistency Validator
- *
- * Validates that manifest config flags are consistent with the actual tensors present.
- */
-
-export interface TensorConfigWarning {
-  severity: 'error' | 'warning';
-  code: string;
-  message: string;
-  suggestion?: string;
-}
-
-export interface TensorConfigValidationResult {
-  valid: boolean;
-  warnings: TensorConfigWarning[];
-  errors: TensorConfigWarning[];
-}
-
-/**
- * Validate tensor-config consistency.
- *
- * Checks that config flags (like `postFeedforwardNorm`) match the actual
- * tensors present in the model. Prevents silent failures from misconfigured
- * presets.
- *
- * @param manifest - The RDRR manifest to validate
- * @returns Validation result with errors and warnings
- */
-export function validateTensorConfigConsistency(
-  manifest: object
-): TensorConfigValidationResult;
-
-/**
- * Format validation result for logging.
- *
- * @param result - The validation result to format
- * @returns Formatted string for console/log output
- */
-export function formatValidationResult(
-  result: TensorConfigValidationResult
-): string;

package/src/formats/rdrr/tensor-config-validator.js

@@ -1,156 +0,0 @@
-// Tensor-Config Consistency Validator
-//
-// Validates that manifest config flags are consistent with the actual tensors present.
-// This catches bugs like postFeedforwardNorm=false when the weights exist.
-// See tensor-config-validator.d.ts for type definitions.
-
-// Tensor name patterns for each config flag
-const TENSOR_CONFIG_MAPPINGS = [
-  {
-    configPath: 'inference.normalization.postFeedforwardNorm',
-    tensorPattern: /post_feedforward_layernorm\.weight$/,
-    description: 'post-feedforward normalization',
-    severity: 'error', // This caused complete model failure
-  },
-  {
-    configPath: 'inference.normalization.preFeedforwardNorm',
-    tensorPattern: /pre_feedforward_layernorm\.weight$/,
-    description: 'pre-feedforward normalization',
-    severity: 'error',
-  },
-  {
-    configPath: 'inference.normalization.postAttentionNorm',
-    tensorPattern: /post_attention_layernorm\.weight$/,
-    description: 'post-attention normalization',
-    severity: 'error',
-  },
-  {
-    configPath: 'inference.attention.queryKeyNorm',
-    tensorPattern: /self_attn\.(q_norm|k_norm)\.weight$/,
-    description: 'query/key normalization',
-    severity: 'warning',
-  },
-  {
-    configPath: 'inference.output.tieWordEmbeddings',
-    tensorPattern: /^(lm_head\.weight|model\.lm_head\.weight)$/,
-    description: 'separate LM head weights',
-    invertLogic: true, // If tensor exists, tieWordEmbeddings should be FALSE
-    severity: 'warning',
-  },
-];
-
-// Get nested property from object using dot notation.
-function getNestedValue(obj, path) {
-  return path.split('.').reduce((current, key) => current?.[key], obj);
-}
-
-// Collect all tensor names from manifest.
-function collectTensorNames(manifest) {
-  const tensorNames = [];
-
-  // From groups
-  if (manifest.groups) {
-    for (const group of Object.values(manifest.groups)) {
-      if (Array.isArray(group.tensors)) {
-        tensorNames.push(...group.tensors);
-      }
-    }
-  }
-
-  // From inline tensors
-  if (manifest.tensors) {
-    tensorNames.push(...Object.keys(manifest.tensors));
-  }
-
-  return tensorNames;
-}
-
-// Validate tensor-config consistency.
-export function validateTensorConfigConsistency(manifest) {
-  const warnings = [];
-  const errors = [];
-
-  const tensorNames = collectTensorNames(manifest);
-
-  for (const mapping of TENSOR_CONFIG_MAPPINGS) {
-    const configValue = getNestedValue(manifest, mapping.configPath);
-    const matchingTensors = tensorNames.filter(name => mapping.tensorPattern.test(name));
-    const hasTensors = matchingTensors.length > 0;
-
-    // Check for inconsistency
-    let isInconsistent = false;
-    let suggestion = '';
-
-    if (mapping.invertLogic) {
-      // For inverted logic (e.g., tieWordEmbeddings)
-      // If tensors exist, config should be false
-      if (hasTensors && configValue === true) {
-        isInconsistent = true;
-        suggestion = `Set ${mapping.configPath} to false (found ${matchingTensors.length} tensor(s): ${matchingTensors.slice(0, 2).join(', ')}...)`;
-      }
-    } else {
-      // Normal logic: if tensors exist, config should be true
-      if (hasTensors && configValue === false) {
-        isInconsistent = true;
-        suggestion = `Set ${mapping.configPath} to true (found ${matchingTensors.length} tensor(s): ${matchingTensors.slice(0, 2).join(', ')}...)`;
-      }
-      // Also warn if config is true but no tensors found
-      if (!hasTensors && configValue === true) {
-        isInconsistent = true;
-        suggestion = `Set ${mapping.configPath} to false (no matching tensors found for pattern: ${mapping.tensorPattern})`;
-      }
-    }
-
-    if (isInconsistent) {
-      const issue = {
-        severity: mapping.severity,
-        code: `TENSOR_CONFIG_MISMATCH_${mapping.configPath.replace(/\./g, '_').toUpperCase()}`,
-        message: `Config flag "${mapping.configPath}" (=${configValue}) is inconsistent with ${mapping.description} tensors (found=${hasTensors})`,
-        suggestion,
-      };
-
-      if (mapping.severity === 'error') {
-        errors.push(issue);
-      } else {
-        warnings.push(issue);
-      }
-    }
-  }
-
-  return {
-    valid: errors.length === 0,
-    warnings,
-    errors,
-  };
-}
-
-// Format validation result for logging.
-export function formatValidationResult(result) {
-  const lines = [];
-
-  if (result.errors.length > 0) {
-    lines.push('ERRORS:');
-    for (const err of result.errors) {
-      lines.push(`  [${err.code}] ${err.message}`);
-      if (err.suggestion) {
-        lines.push(`    -> ${err.suggestion}`);
-      }
-    }
-  }
-
-  if (result.warnings.length > 0) {
-    lines.push('WARNINGS:');
-    for (const warn of result.warnings) {
-      lines.push(`  [${warn.code}] ${warn.message}`);
-      if (warn.suggestion) {
-        lines.push(`    -> ${warn.suggestion}`);
-      }
-    }
-  }
-
-  if (result.valid && result.warnings.length === 0) {
-    lines.push('Tensor-config consistency: OK');
-  }
-
-  return lines.join('\n');
-}

package/src/formats/rdrr/types.d.ts

@@ -1,201 +0,0 @@
-/**
- * RDRR Format Types
- *
- * Core type definitions for the RDRR model format.
- *
- * @module formats/rdrr/types
- */
-
-import type {
-  HashAlgorithm as SchemaHashAlgorithm,
-  ModelType as SchemaModelType,
-  ComponentGroupType as SchemaComponentGroupType,
-  WeightLayout as SchemaWeightLayout,
-  QuantizationInfoSchema,
-  ComponentGroupSchema,
-  MoEConfigSchema,
-  AdapterConfigSchema,
-  ProvenanceSchema,
-  KernelPathRef,
-  ManifestInferenceSchema,
-  type TensorRole as SchemaTensorRole,
-} from '../../config/schema/index.js';
-
-// =============================================================================
-// Re-exports from Schema
-// =============================================================================
-
-export declare const RDRR_VERSION: number;
-export declare const SHARD_SIZE: number;
-export declare const MANIFEST_FILENAME: string;
-export declare const TENSORS_FILENAME: string;
-
-export type HashAlgorithm = SchemaHashAlgorithm;
-export type ModelType = SchemaModelType;
-export type ComponentGroupType = SchemaComponentGroupType;
-export type WeightLayout = SchemaWeightLayout;
-export type QuantizationInfo = QuantizationInfoSchema;
-export type TensorRole = SchemaTensorRole;
-
-// =============================================================================
-// Kernel Types
-// =============================================================================
-
-export type Q4KLayout = 'row' | 'col' | null;
-
-// =============================================================================
-// Manifest Types
-// =============================================================================
-
-export interface ShardInfo {
-  index: number;
-  filename: string;
-  size: number;
-  hash: string;
-  offset: number;
-  hashAlgorithm?: HashAlgorithm;
-}
-
-export interface MoEConfig extends MoEConfigSchema {
-  expertSize?: number;
-}
-
-export interface LayerConfig {
-  numLayers: number;
-  hiddenSize: number;
-  intermediateSize: number;
-  numAttentionHeads: number;
-  numKeyValueHeads?: number;
-  headDim?: number;
-  vocabSize: number;
-  maxSeqLen: number;
-}
-
-export interface ComponentGroup extends ComponentGroupSchema {}
-
-export interface TensorLocation {
-  shard: number;
-  shardIndex?: number;
-  offset: number;
-  size: number;
-  shape: number[];
-  dtype: string;
-  role: TensorRole;
-  group?: string;
-  spans?: Array<{ shard?: number; shardIndex?: number; offset: number; size: number }>;
-  layout?: WeightLayout;
-  originalShape?: number[];
-}
-
-export interface ConversionInfo {
-  source: string;
-  convertedAt: string;
-  converterVersion: string;
-  command?: string;
-  quantization: {
-    type: string;
-    layout?: Q4KLayout;
-    fuseGateUp?: boolean;
-    quantizeEmbeddings?: boolean;
-  };
-  originalDtype?: string;
-  notes?: string;
-}
-
-export interface RuntimeOptimizations {
-  /** Preferred kernel path override */
-  kernelPath?: KernelPathRef;
-}
-
-export interface RDRRManifest {
-  version: number;
-  modelId: string;
-  modelType: ModelType;
-  quantization: string;
-  quantizationInfo?: QuantizationInfo;
-  hashAlgorithm: HashAlgorithm;
-  eos_token_id: number | number[] | null;
-  architecture: LayerConfig | string;
-  groups?: Record<string, ComponentGroup>;
-  shards: ShardInfo[];
-  totalSize: number;
-  tensorsFile?: string;
-  tensorCount?: number;
-  tokenizer?: {
-    type: string;
-    file: string;
-    vocabSize: number;
-  };
-  moeConfig?: MoEConfig;
-  optimizations?: RuntimeOptimizations;
-  config?: Record<string, unknown>;
-  conversion?: ConversionInfo;
-
-  // Required inference configuration (populated by converter)
-  inference: ManifestInferenceSchema;
-  blake3Full?: string;
-  defaultWeightLayout?: WeightLayout;
-  metadata?: Record<string, unknown>;
-
-  // Adapter support (for LoRA/QLoRA)
-  /** Adapter type - present only for adapter manifests */
-  adapterType?: 'lora' | 'qlora';
-  /** Base model compatibility - required for adapter manifests */
-  baseCompatibility?: string[];
-  /** Merged adapter info - present when adapter is baked into weights */
-  mergedAdapter?: AdapterConfigSchema;
-  /** Adapter config - full config for standalone adapter manifests */
-  adapterConfig?: AdapterConfigSchema;
-
-  // Provenance (for merged/frankenstein models)
-  provenance?: ProvenanceSchema;
-
-  // LoRA adapter fields (used by adapter loading system)
-  baseModel?: string;
-  loraConfig?: {
-    rank: number;
-    alpha: number;
-    targetModules?: string[];
-    dropout?: number;
-  };
-
-  // Legacy inline tensors (use tensorsFile for new manifests)
-  tensors?: Record<string, TensorLocation>;
-}
-
-export type TensorMap = Record<string, TensorLocation>;
-
-export interface ValidationResult {
-  valid: boolean;
-  errors: string[];
-}
-
-export interface CreateManifestOptions {
-  modelId: string;
-  modelType: ModelType;
-  quantization: string;
-  quantizationInfo?: QuantizationInfo;
-  hashAlgorithm?: HashAlgorithm;
-  eos_token_id?: number | number[];
-  architecture: LayerConfig | string;
-  groups?: Record<string, ComponentGroup>;
-  shards: ShardInfo[];
-  totalSize: number;
-  tensorCount?: number;
-  tensorsFile?: string;
-  tensors?: Record<string, TensorLocation>;
-  tokenizer?: { type: string; file: string; vocabSize: number };
-  moeConfig?: MoEConfig;
-  config?: Record<string, unknown>;
-  conversion?: ConversionInfo;
-  blake3Full?: string;
-  metadata?: Record<string, unknown>;
-  // Required inference configuration
-  inference: ManifestInferenceSchema;
-  // Adapter support
-  adapterType?: 'lora' | 'qlora';
-  baseCompatibility?: string[];
-  mergedAdapter?: AdapterConfigSchema;
-  adapterConfig?: AdapterConfigSchema;
-  provenance?: ProvenanceSchema;
-}

package/src/formats/rdrr/types.js

@@ -1,16 +0,0 @@
-
-
-import {
-  RDRR_VERSION as SCHEMA_VERSION,
-  SHARD_SIZE as SCHEMA_SHARD_SIZE,
-  TENSORS_FILENAME as SCHEMA_TENSORS_FILENAME,
-} from '../../config/schema/index.js';
-
-// =============================================================================
-// Re-exports from Schema
-// =============================================================================
-
-export const RDRR_VERSION = SCHEMA_VERSION;
-export const SHARD_SIZE = SCHEMA_SHARD_SIZE;
-export const MANIFEST_FILENAME = 'manifest.json';
-export const TENSORS_FILENAME = SCHEMA_TENSORS_FILENAME;