@simulatte/doppler 0.1.0

package/src/debug/reference/hf_embed_check.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""
+Compare embeddings and layer outputs between HuggingFace and DOPPLER.
+
+Usage:
+    python hf_embed_check.py <config.json>
+
+Config (JSON):
+    {
+      "model": "google/gemma-2-2b-it",
+      "prompt": "The color of the sky is"
+    }
+
+Requires: pip install torch transformers
+"""
+
+import json
+import sys
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+
+def load_config():
+    if len(sys.argv) != 2:
+        raise SystemExit("Usage: python hf_embed_check.py <config.json>")
+    path = sys.argv[1]
+    with open(path, "r", encoding="utf-8") as handle:
+        config = json.load(handle)
+    if not isinstance(config, dict):
+        raise SystemExit("Config must be a JSON object")
+    return config
+
+
+def require_string(config, key):
+    value = config.get(key)
+    if not isinstance(value, str) or not value.strip():
+        raise SystemExit(f'Config "{key}" must be a non-empty string')
+    return value
+
+
+def main():
+    config = load_config()
+    model_id = require_string(config, "model")
+    prompt = require_string(config, "prompt")
+
+    print(f"Loading model: {model_id}")
+    model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float32, device_map="cpu")
+    tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+    inputs = tokenizer(prompt, return_tensors="pt")
+    input_ids = inputs['input_ids']
+
+    print(f"\nPrompt: {prompt}")
+    print(f"Token IDs: {input_ids[0].tolist()}")
+
+    # Get embeddings (raw, not scaled)
+    raw_embeddings = model.model.embed_tokens(input_ids)
+    print(f"\nRaw embeddings shape: {raw_embeddings.shape}")
+    print(f"Raw embeddings (last token first 5): {raw_embeddings[0, -1, :5].tolist()}")
+
+    # Gemma scales embeddings by sqrt(hidden_size)
+    hidden_size = model.config.hidden_size
+    scaled_embeddings = raw_embeddings * (hidden_size ** 0.5)
+    print(f"Scaled embeddings (last token first 5): {scaled_embeddings[0, -1, :5].tolist()}")
+
+    # Get layer 0 output via hooks
+    layer0_outputs = {}
+
+    def hook_layer0_output(module, input, output):
+        layer0_outputs['output'] = output[0].detach()
+
+    model.model.layers[0].register_forward_hook(hook_layer0_output)
+
+    # Run forward pass
+    with torch.no_grad():
+        outputs = model(input_ids, output_hidden_states=True)
+
+    print(f"\nLayer 0 output (last token first 5): {layer0_outputs['output'][0, -1, :5].tolist()}")
+
+    # Per-token comparison
+    print(f"\nPer-token maxAbs comparison:")
+    for t in range(inputs['input_ids'].shape[1]):
+        emb_max = scaled_embeddings[0, t].abs().max().item()
+        l0_max = layer0_outputs['output'][0, t].abs().max().item()
+        print(f"  Token {t}: emb_maxAbs={emb_max:.2f}, L0_maxAbs={l0_max:.2f}")
+
+
+if __name__ == "__main__":
+    main()

package/src/debug/reference/hf_layer_out.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+"""
+Compare layer outputs at each layer between HuggingFace and DOPPLER.
+
+Usage:
+    python hf_layer_out.py <config.json>
+
+Config (JSON):
+    {
+      "model": "google/gemma-2-2b-it",
+      "prompt": "The color of the sky is",
+      "layers": [0, 12, 25],
+      "token": -1
+    }
+
+Requires: pip install torch transformers
+"""
+
+import json
+import sys
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+
+def main():
+    if len(sys.argv) != 2:
+        raise SystemExit("Usage: python hf_layer_out.py <config.json>")
+    path = sys.argv[1]
+    with open(path, "r", encoding="utf-8") as handle:
+        config = json.load(handle)
+    if not isinstance(config, dict):
+        raise SystemExit("Config must be a JSON object")
+
+    model_id = config.get("model")
+    prompt = config.get("prompt")
+    layers = config.get("layers")
+    token = config.get("token")
+
+    if not isinstance(model_id, str) or not model_id.strip():
+        raise SystemExit('Config "model" must be a non-empty string')
+    if not isinstance(prompt, str) or not prompt.strip():
+        raise SystemExit('Config "prompt" must be a non-empty string')
+    if not isinstance(layers, list) or not layers:
+        raise SystemExit('Config "layers" must be a non-empty array')
+    if not all(isinstance(x, int) for x in layers):
+        raise SystemExit('Config "layers" entries must be integers')
+    if not isinstance(token, int):
+        raise SystemExit('Config "token" must be an integer')
+
+    layer_indices = layers
+
+    print(f"Loading model: {model_id}")
+    model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float32, device_map="cpu")
+    tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+    inputs = tokenizer(prompt, return_tensors="pt")
+    input_ids = inputs['input_ids']
+    num_tokens = input_ids.shape[1]
+    token_idx = token if token >= 0 else num_tokens + token
+
+    print(f"\nPrompt: {prompt}")
+    print(f"Token IDs: {input_ids[0].tolist()}")
+    print(f"Checking token index: {token_idx}")
+
+    # Run forward pass with hidden states
+    with torch.no_grad():
+        outputs = model(input_ids, output_hidden_states=True)
+
+    hidden_states = outputs.hidden_states  # Tuple of [batch, seq, hidden]
+    print(f"\nNum hidden states: {len(hidden_states)} (embed + {len(hidden_states)-1} layers)")
+
+    for layer_idx in layer_indices:
+        if layer_idx >= len(hidden_states) - 1:
+            print(f"\nLayer {layer_idx}: OUT OF RANGE")
+            continue
+
+        # hidden_states[0] = embeddings, hidden_states[1] = layer 0 output, etc.
+        hs = hidden_states[layer_idx + 1]  # +1 because index 0 is embeddings
+
+        vals = hs[0, token_idx, :8].tolist()
+        max_abs = hs[0, token_idx].abs().max().item()
+        mean_abs = hs[0, token_idx].abs().mean().item()
+
+        print(f"\nLayer {layer_idx} output (token {token_idx}):")
+        print(f"  First 8: {[f'{v:.4f}' for v in vals]}")
+        print(f"  maxAbs: {max_abs:.4f}, meanAbs: {mean_abs:.4f}")
+
+    # Also show BOS (token 0) for comparison
+    print(f"\n--- BOS Token (position 0) for reference ---")
+    for layer_idx in layer_indices:
+        if layer_idx >= len(hidden_states) - 1:
+            continue
+        hs = hidden_states[layer_idx + 1]
+        vals = hs[0, 0, :8].tolist()
+        max_abs = hs[0, 0].abs().max().item()
+        print(f"Layer {layer_idx} (BOS): first8={[f'{v:.4f}' for v in vals]}, maxAbs={max_abs:.4f}")
+
+
+if __name__ == "__main__":
+    main()

package/src/debug/reference/hf_rope_check.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+"""
+Verify RoPE (Rotary Position Embedding) values match between HF and DOPPLER.
+
+Usage:
+    python hf_rope_check.py <config.json>
+
+Config (JSON):
+    {
+      "model": "google/gemma-2-2b-it",
+      "pos": 6,
+      "dim": 256,
+      "theta": 10000.0
+    }
+
+Requires: pip install torch transformers
+"""
+
+import json
+import math
+import sys
+import torch
+from transformers import AutoModelForCausalLM
+
+
+def compute_rope_freqs(head_dim: int, theta: float = 10000.0):
+    """Compute RoPE frequencies for a given head dimension."""
+    freqs = 1.0 / (theta ** (torch.arange(0, head_dim, 2).float() / head_dim))
+    return freqs
+
+
+def apply_rope_to_vector(x: torch.Tensor, pos: int, freqs: torch.Tensor):
+    """Apply RoPE to a single vector at a given position."""
+    # x shape: [head_dim]
+    angles = pos * freqs
+    cos = torch.cos(angles)
+    sin = torch.sin(angles)
+
+    # Split into pairs
+    x_even = x[0::2]
+    x_odd = x[1::2]
+
+    # Apply rotation
+    out_even = x_even * cos - x_odd * sin
+    out_odd = x_even * sin + x_odd * cos
+
+    # Interleave back
+    out = torch.zeros_like(x)
+    out[0::2] = out_even
+    out[1::2] = out_odd
+
+    return out
+
+
+def main():
+    if len(sys.argv) != 2:
+        raise SystemExit("Usage: python hf_rope_check.py <config.json>")
+    path = sys.argv[1]
+    with open(path, "r", encoding="utf-8") as handle:
+        config = json.load(handle)
+    if not isinstance(config, dict):
+        raise SystemExit("Config must be a JSON object")
+
+    model_id = config.get("model")
+    pos = config.get("pos")
+    dim = config.get("dim")
+    theta = config.get("theta")
+
+    if not isinstance(model_id, str) or not model_id.strip():
+        raise SystemExit('Config "model" must be a non-empty string')
+    if not isinstance(pos, int):
+        raise SystemExit('Config "pos" must be an integer')
+    if not isinstance(dim, int):
+        raise SystemExit('Config "dim" must be an integer')
+    if not isinstance(theta, (int, float)):
+        raise SystemExit('Config "theta" must be a number')
+
+    print(f"RoPE Check for position {pos}, head_dim={dim}, theta={theta}")
+
+    # Compute frequencies
+    freqs = compute_rope_freqs(dim, theta)
+    print(f"\nFrequencies (first 8): {freqs[:8].tolist()}")
+
+    # Compute angles at position
+    angles = pos * freqs
+    print(f"Angles at pos {pos} (first 8): {angles[:8].tolist()}")
+
+    # Compute cos/sin
+    cos = torch.cos(angles)
+    sin = torch.sin(angles)
+    print(f"Cos at pos {pos} (first 8): {cos[:8].tolist()}")
+    print(f"Sin at pos {pos} (first 8): {sin[:8].tolist()}")
+
+    # Test with a sample vector
+    test_vec = torch.randn(dim)
+    test_vec[0] = 1.0
+    test_vec[1] = 0.0
+    print(f"\nTest vector (first 8): {test_vec[:8].tolist()}")
+
+    rotated = apply_rope_to_vector(test_vec, pos, freqs)
+    print(f"After RoPE (first 8): {rotated[:8].tolist()}")
+
+    # Also load model and check actual RoPE config
+    print(f"\nLoading model to verify config: {model_id}")
+    model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float32, device_map="cpu")
+    config = model.config
+
+    print(f"\nModel RoPE config:")
+    print(f"  rope_theta: {getattr(config, 'rope_theta', 'not set')}")
+    print(f"  head_dim: {config.head_dim}")
+    if hasattr(config, 'rope_scaling'):
+        print(f"  rope_scaling: {config.rope_scaling}")
+
+
+if __name__ == "__main__":
+    main()

package/src/debug/reference/hf_weights.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+"""
+Dump weight values for Q/K/V/O projections from HuggingFace model.
+
+Usage:
+    python hf_weights.py <config.json>
+
+Config (JSON):
+    {
+      "model": "google/gemma-2-2b-it",
+      "layer": 0,
+      "proj": "all"
+    }
+
+Requires: pip install torch transformers
+"""
+
+import json
+import sys
+import torch
+from transformers import AutoModelForCausalLM
+
+
+def main():
+    if len(sys.argv) != 2:
+        raise SystemExit("Usage: python hf_weights.py <config.json>")
+    path = sys.argv[1]
+    with open(path, "r", encoding="utf-8") as handle:
+        config = json.load(handle)
+    if not isinstance(config, dict):
+        raise SystemExit("Config must be a JSON object")
+
+    model_id = config.get("model")
+    layer_index = config.get("layer")
+    proj = config.get("proj")
+
+    if not isinstance(model_id, str) or not model_id.strip():
+        raise SystemExit('Config "model" must be a non-empty string')
+    if not isinstance(layer_index, int):
+        raise SystemExit('Config "layer" must be an integer')
+    if proj not in ["q", "k", "v", "o", "all"]:
+        raise SystemExit('Config "proj" must be one of: q, k, v, o, all')
+
+    print(f"Loading model: {model_id}")
+    model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float32, device_map="cpu")
+
+    layer = model.model.layers[layer_index]
+    attn = layer.self_attn
+
+    projections = {
+        "q": attn.q_proj,
+        "k": attn.k_proj,
+        "v": attn.v_proj,
+        "o": attn.o_proj,
+    }
+
+    projs_to_show = list(projections.keys()) if proj == "all" else [proj]
+
+    for name in projs_to_show:
+        proj = projections[name]
+        weight = proj.weight.data  # [out_features, in_features]
+
+        print(f"\n{name.upper()}_proj weight shape: {weight.shape}")
+        print(f"{name.upper()}_proj weight[0, :8] (first row, first 8 cols):")
+        print(f"  {weight[0, :8].tolist()}")
+        print(f"{name.upper()}_proj weight[:8, 0] (first 8 rows, first col):")
+        print(f"  {weight[:8, 0].tolist()}")
+
+        # Some specific indices for comparison
+        if weight.shape[1] > 100:
+            print(f"{name.upper()}_proj weight[0, 100]: {weight[0, 100].item():.6f}")
+
+
+if __name__ == "__main__":
+    main()

package/src/debug/signals.d.ts

@@ -0,0 +1,63 @@
+/**
+ * DOPPLER Debug Module - Completion Signals
+ *
+ * Standard completion signal prefixes for CLI/automation detection.
+ *
+ * Usage:
+ *   console.log(`${SIGNALS.DONE} ${JSON.stringify({status: 'success', elapsed: 1234})}`);
+ *   console.log(`${SIGNALS.RESULT} ${JSON.stringify(benchmarkData)}`);
+ *   console.log(`${SIGNALS.ERROR} ${JSON.stringify({error: 'message'})}`);
+ *
+ * Detection (CLI/Puppeteer):
+ *   if (text.startsWith('[DOPPLER:DONE]')) { ... }
+ *
+ * @module debug/signals
+ */
+
+/**
+ * Standard completion signal prefixes for CLI/automation detection.
+ */
+export declare const SIGNALS: {
+  /** Task completed (success or error) - always emitted at end */
+  readonly DONE: '[DOPPLER:DONE]';
+  /** Full result payload (JSON) - emitted before DONE for data extraction */
+  readonly RESULT: '[DOPPLER:RESULT]';
+  /** Error occurred - can be emitted before DONE */
+  readonly ERROR: '[DOPPLER:ERROR]';
+  /** Progress update (optional) */
+  readonly PROGRESS: '[DOPPLER:PROGRESS]';
+};
+
+export type SignalType = keyof typeof SIGNALS;
+
+/**
+ * Completion payload for DONE signal.
+ */
+export interface DonePayload {
+  status: 'success' | 'error';
+  elapsed: number;
+  tokens?: number;
+  tokensPerSecond?: number;
+  error?: string;
+}
+
+/**
+ * Emit a completion signal to console.
+ * This is the standard way to signal task completion for CLI detection.
+ */
+export declare function signalDone(payload: DonePayload): void;
+
+/**
+ * Emit a result signal with full data payload.
+ */
+export declare function signalResult(data: Record<string, unknown>): void;
+
+/**
+ * Emit an error signal.
+ */
+export declare function signalError(error: string, details?: Record<string, unknown>): void;
+
+/**
+ * Emit a progress signal.
+ */
+export declare function signalProgress(percent: number, message?: string): void;

package/src/debug/signals.js

@@ -0,0 +1,33 @@
+
+
+
+export const SIGNALS = {
+  
+  DONE: '[DOPPLER:DONE]',
+  
+  RESULT: '[DOPPLER:RESULT]',
+  
+  ERROR: '[DOPPLER:ERROR]',
+  
+  PROGRESS: '[DOPPLER:PROGRESS]',
+};
+
+
+export function signalDone(payload) {
+  console.log(`${SIGNALS.DONE} ${JSON.stringify(payload)}`);
+}
+
+
+export function signalResult(data) {
+  console.log(`${SIGNALS.RESULT} ${JSON.stringify(data)}`);
+}
+
+
+export function signalError(error, details) {
+  console.log(`${SIGNALS.ERROR} ${JSON.stringify({ error, ...details })}`);
+}
+
+
+export function signalProgress(percent, message) {
+  console.log(`${SIGNALS.PROGRESS} ${JSON.stringify({ percent, message })}`);
+}

package/src/debug/stats.d.ts

@@ -0,0 +1,47 @@
+export interface SampleStats {
+  mean: number;
+  median: number;
+  min: number;
+  max: number;
+  p95: number;
+  p99: number;
+  stdDev: number;
+  ci95: number;
+  samples: number;
+  samplesAfterOutlierRemoval: number;
+  outliersRemoved: number;
+}
+
+export interface ArrayStats {
+  min: number;
+  max: number;
+  mean: number;
+  std: number;
+  maxAbs: number;
+  nanCount: number;
+  infCount: number;
+  zeroCount: number;
+  validCount: number;
+}
+
+export interface BasicStats {
+  mean: number;
+  min: number;
+  max: number;
+  total: number;
+  count: number;
+}
+
+export declare function percentile(sorted: number[], p: number): number;
+export declare function median(sorted: number[]): number;
+export declare function removeOutliersIQR(values: number[], multiplier?: number): {
+  filtered: number[];
+  removed: number;
+  lower: number;
+  upper: number;
+};
+export declare function sampleStdDev(values: number[], meanValue: number): number;
+export declare function confidenceInterval95(stdDev: number, n: number): number;
+export declare function computeSampleStats(values: number[], options?: { outlierIqrMultiplier?: number }): SampleStats;
+export declare function computeArrayStats(values: ArrayLike<number>, limit?: number): ArrayStats;
+export declare function computeBasicStats(values: number[]): BasicStats;

package/src/debug/stats.js

@@ -0,0 +1,160 @@
+export function percentile(sorted, p) {
+  if (!sorted.length) return 0;
+  if (sorted.length === 1) return sorted[0];
+  const idx = (p / 100) * (sorted.length - 1);
+  const lo = Math.floor(idx);
+  const hi = Math.ceil(idx);
+  if (lo === hi) return sorted[lo];
+  return sorted[lo] + (sorted[hi] - sorted[lo]) * (idx - lo);
+}
+
+export function median(sorted) {
+  if (!sorted.length) return 0;
+  const mid = Math.floor(sorted.length / 2);
+  return sorted.length % 2 === 0
+    ? (sorted[mid - 1] + sorted[mid]) / 2
+    : sorted[mid];
+}
+
+export function removeOutliersIQR(values, multiplier = 1.5) {
+  if (values.length < 4) return { filtered: values, removed: 0, lower: -Infinity, upper: Infinity };
+  const sorted = [...values].sort((a, b) => a - b);
+  const q1 = percentile(sorted, 25);
+  const q3 = percentile(sorted, 75);
+  const iqr = q3 - q1;
+  const lower = q1 - multiplier * iqr;
+  const upper = q3 + multiplier * iqr;
+  const filtered = values.filter((v) => v >= lower && v <= upper);
+  return { filtered, removed: values.length - filtered.length, lower, upper };
+}
+
+export function sampleStdDev(values, meanValue) {
+  const n = values.length;
+  if (n < 2) return 0;
+  const variance = values.reduce((sum, v) => sum + (v - meanValue) ** 2, 0) / (n - 1);
+  return Math.sqrt(variance);
+}
+
+export function confidenceInterval95(stdDev, n) {
+  if (n < 2) return 0;
+  const tValue = n >= 30 ? 1.96 : 2.0 + 3.0 / n;
+  return tValue * (stdDev / Math.sqrt(n));
+}
+
+export function computeSampleStats(values, options = {}) {
+  const { outlierIqrMultiplier = 1.5 } = options;
+  const rawValues = Array.isArray(values) ? values : [];
+  const finiteValues = rawValues.filter((value) => Number.isFinite(value));
+  const { filtered, removed } = removeOutliersIQR(finiteValues, outlierIqrMultiplier);
+  const nonFiniteRemoved = rawValues.length - finiteValues.length;
+  const sorted = [...filtered].sort((a, b) => a - b);
+  const n = sorted.length;
+  const totalRemoved = removed + nonFiniteRemoved;
+
+  if (n === 0) {
+    return {
+      mean: 0,
+      median: 0,
+      min: 0,
+      max: 0,
+      p95: 0,
+      p99: 0,
+      stdDev: 0,
+      ci95: 0,
+      samples: rawValues.length,
+      samplesAfterOutlierRemoval: 0,
+      outliersRemoved: totalRemoved,
+    };
+  }
+
+  const meanValue = filtered.reduce((a, b) => a + b, 0) / n;
+  const stdDev = sampleStdDev(filtered, meanValue);
+
+  return {
+    mean: meanValue,
+    median: median(sorted),
+    min: sorted[0],
+    max: sorted[n - 1],
+    p95: percentile(sorted, 95),
+    p99: percentile(sorted, 99),
+    stdDev,
+    ci95: confidenceInterval95(stdDev, n),
+    samples: rawValues.length,
+    samplesAfterOutlierRemoval: n,
+    outliersRemoved: totalRemoved,
+  };
+}
+
+export function computeArrayStats(values, limit = values.length) {
+  const stats = {
+    min: Infinity,
+    max: -Infinity,
+    mean: 0,
+    std: 0,
+    maxAbs: 0,
+    nanCount: 0,
+    infCount: 0,
+    zeroCount: 0,
+    validCount: 0,
+  };
+
+  let sum = 0;
+  let sumSq = 0;
+  const count = Math.min(values.length, limit);
+
+  for (let i = 0; i < count; i++) {
+    const v = values[i];
+    if (Number.isNaN(v)) {
+      stats.nanCount++;
+      continue;
+    }
+    if (!Number.isFinite(v)) {
+      stats.infCount++;
+      continue;
+    }
+    if (v === 0) stats.zeroCount++;
+    stats.min = Math.min(stats.min, v);
+    stats.max = Math.max(stats.max, v);
+    sum += v;
+    sumSq += v * v;
+    stats.validCount++;
+  }
+
+  if (stats.validCount > 0) {
+    stats.mean = sum / stats.validCount;
+    const variance = sumSq / stats.validCount - stats.mean * stats.mean;
+    stats.std = Math.sqrt(Math.max(0, variance));
+    stats.maxAbs = Math.max(Math.abs(stats.min), Math.abs(stats.max));
+  } else {
+    stats.min = 0;
+    stats.max = 0;
+    stats.maxAbs = 0;
+  }
+
+  return stats;
+}
+
+export function computeBasicStats(values) {
+  const count = values.length;
+  if (!count) {
+    return { mean: 0, min: 0, max: 0, total: 0, count: 0 };
+  }
+
+  let min = Infinity;
+  let max = -Infinity;
+  let total = 0;
+
+  for (const v of values) {
+    min = Math.min(min, v);
+    max = Math.max(max, v);
+    total += v;
+  }
+
+  return {
+    mean: total / count,
+    min,
+    max,
+    total,
+    count,
+  };
+}