@simulatte/doppler 0.1.8 → 0.2.0

package/src/training/checkpoint.js

@@ -1,277 +0,0 @@
-import { sha256Hex } from '../utils/sha256.js';
-
-function isNodeRuntime() {
-  return typeof process !== 'undefined' && !!process.versions?.node;
-}
-
-function looksLikePath(value) {
-  const normalized = String(value || '').trim();
-  return normalized.includes('/') || normalized.includes('\\') || normalized.endsWith('.json');
-}
-
-function openCheckpointDB(options = {}) {
-  const {
-    dbName = 'doppler-training',
-    storeName = 'checkpoints',
-    version = 1,
-  } = options;
-
-  return new Promise((resolve, reject) => {
-    const request = indexedDB.open(dbName, version);
-
-    request.onupgradeneeded = () => {
-      const db = request.result;
-      if (!db.objectStoreNames.contains(storeName)) {
-        db.createObjectStore(storeName);
-      }
-    };
-
-    request.onerror = () => reject(request.error);
-    request.onsuccess = () => resolve({ db: request.result, storeName });
-  });
-}
-
-function closeCheckpointDB(db) {
-  if (!db || typeof db.close !== 'function') {
-    return;
-  }
-  db.close();
-}
-
-async function resolveNodeCheckpointPath(key, options = {}) {
-  const [{ resolve, join, dirname }, { mkdir }] = await Promise.all([
-    import('node:path'),
-    import('node:fs/promises'),
-  ]);
-  const configuredRoot = typeof options.nodeDir === 'string' && options.nodeDir.trim()
-    ? options.nodeDir.trim()
-    : '.doppler-checkpoints';
-  if (looksLikePath(key)) {
-    const direct = resolve(String(key));
-    await mkdir(dirname(direct), { recursive: true });
-    return direct;
-  }
-  const safeKey = String(key).replace(/[^a-zA-Z0-9._-]/g, '_');
-  const root = resolve(configuredRoot);
-  await mkdir(root, { recursive: true });
-  return join(root, `${safeKey}.json`);
-}
-
-async function readNodeCheckpointRecord(filePath) {
-  const { readFile } = await import('node:fs/promises');
-  try {
-    const raw = await readFile(filePath, 'utf8');
-    return JSON.parse(raw);
-  } catch (error) {
-    if (error?.code === 'ENOENT') {
-      return null;
-    }
-    throw error;
-  }
-}
-
-async function writeNodeCheckpointRecord(filePath, data) {
-  const [{ writeFile, mkdir }, { dirname }] = await Promise.all([
-    import('node:fs/promises'),
-    import('node:path'),
-  ]);
-  await mkdir(dirname(filePath), { recursive: true });
-  await writeFile(filePath, `${JSON.stringify(data, null, 2)}\n`, 'utf8');
-}
-
-function readCheckpointRecord(db, storeName, key) {
-  return new Promise((resolve, reject) => {
-    const tx = db.transaction(storeName, 'readonly');
-    tx.onerror = () => reject(tx.error);
-    const store = tx.objectStore(storeName);
-    const request = store.get(key);
-    request.onsuccess = () => resolve(request.result ?? null);
-    request.onerror = () => reject(request.error);
-  });
-}
-
-function stableSortObject(value) {
-  if (Array.isArray(value)) {
-    return value.map((entry) => stableSortObject(entry));
-  }
-  if (!value || typeof value !== 'object') {
-    return value;
-  }
-  const sorted = {};
-  for (const key of Object.keys(value).sort()) {
-    sorted[key] = stableSortObject(value[key]);
-  }
-  return sorted;
-}
-
-function stableJson(value) {
-  return JSON.stringify(stableSortObject(value));
-}
-
-function coalesceMetadataValue(...candidates) {
-  for (const value of candidates) {
-    if (value !== undefined && value !== null) {
-      return value;
-    }
-  }
-  return null;
-}
-
-function buildCheckpointHashPayload(data) {
-  const metadata = data?.metadata || {};
-  const lineage = metadata.lineage || {};
-  return {
-    payload: {
-      ...data,
-      metadata: undefined,
-    },
-    metadata: {
-      configHash: metadata.configHash ?? null,
-      datasetHash: metadata.datasetHash ?? null,
-      tokenizerHash: metadata.tokenizerHash ?? null,
-      optimizerHash: metadata.optimizerHash ?? null,
-      runtimePresetId: metadata.runtimePresetId ?? null,
-      kernelPathId: metadata.kernelPathId ?? null,
-      environmentMetadata: metadata.environmentMetadata ?? null,
-      buildProvenance: metadata.buildProvenance ?? null,
-      lineage: {
-        checkpointKey: lineage.checkpointKey ?? null,
-        sequence: Number.isInteger(lineage.sequence) ? lineage.sequence : 0,
-        previousCheckpointHash: lineage.previousCheckpointHash ?? null,
-      },
-    },
-  };
-}
-
-export async function saveCheckpoint(key, payload, options = {}) {
-  const useNodeStore = isNodeRuntime() && typeof indexedDB === 'undefined';
-  const nodePath = useNodeStore ? await resolveNodeCheckpointPath(key, options) : null;
-  const browserStore = useNodeStore ? null : await openCheckpointDB(options);
-  let previousData;
-  try {
-    previousData = useNodeStore
-      ? await readNodeCheckpointRecord(nodePath)
-      : await readCheckpointRecord(browserStore.db, browserStore.storeName, key);
-  } catch (error) {
-    closeCheckpointDB(browserStore?.db);
-    throw error;
-  }
-  const previousMetadata = previousData?.metadata || {};
-  const previousLineage = previousMetadata.lineage || {};
-  const previousCheckpointHash = options.priorCheckpointHash
-    || previousMetadata.checkpointHash
-    || previousLineage.previousCheckpointHash
-    || null;
-  const lineageSequence = Number.isInteger(previousLineage.sequence)
-    ? previousLineage.sequence + 1
-    : 1;
-
-  const data = { ...payload };
-  const payloadMetadata = data.metadata && typeof data.metadata === 'object'
-    ? data.metadata
-    : {};
-  data.metadata = {
-    ...payloadMetadata,
-    timestamp: Date.now(),
-    configHash: coalesceMetadataValue(options.configHash, payloadMetadata.configHash, previousMetadata.configHash),
-    datasetHash: coalesceMetadataValue(options.datasetHash, payloadMetadata.datasetHash, previousMetadata.datasetHash),
-    tokenizerHash: coalesceMetadataValue(options.tokenizerHash, payloadMetadata.tokenizerHash, previousMetadata.tokenizerHash),
-    optimizerHash: coalesceMetadataValue(options.optimizerHash, payloadMetadata.optimizerHash, previousMetadata.optimizerHash),
-    runtimePresetId: coalesceMetadataValue(options.runtimePresetId, payloadMetadata.runtimePresetId, previousMetadata.runtimePresetId),
-    kernelPathId: coalesceMetadataValue(options.kernelPathId, payloadMetadata.kernelPathId, previousMetadata.kernelPathId),
-    environmentMetadata: coalesceMetadataValue(
-      options.environmentMetadata,
-      payloadMetadata.environmentMetadata,
-      previousMetadata.environmentMetadata
-    ),
-    buildProvenance: coalesceMetadataValue(options.buildProvenance, payloadMetadata.buildProvenance, previousMetadata.buildProvenance),
-    lineage: {
-      checkpointKey: key,
-      sequence: lineageSequence,
-      previousCheckpointHash,
-    },
-  };
-  data.metadata.checkpointHash = sha256Hex(
-    stableJson(buildCheckpointHashPayload(data))
-  );
-
-  if (useNodeStore) {
-    await writeNodeCheckpointRecord(nodePath, data);
-    return {
-      key,
-      path: nodePath,
-      metadata: data.metadata,
-      data,
-    };
-  }
-
-  return new Promise((resolve, reject) => {
-    const tx = browserStore.db.transaction(browserStore.storeName, 'readwrite');
-    tx.oncomplete = () => {
-      closeCheckpointDB(browserStore.db);
-      resolve({
-        key,
-        path: null,
-        metadata: data.metadata,
-        data,
-      });
-    };
-    tx.onerror = () => {
-      const error = tx.error;
-      closeCheckpointDB(browserStore.db);
-      reject(error);
-    };
-    tx.onabort = () => {
-      const error = tx.error ?? new Error('Checkpoint transaction aborted');
-      closeCheckpointDB(browserStore.db);
-      reject(error);
-    };
-    const store = tx.objectStore(browserStore.storeName);
-    store.put(data, key);
-  });
-}
-
-export async function loadCheckpoint(key, options = {}) {
-  const useNodeStore = isNodeRuntime() && typeof indexedDB === 'undefined';
-  const nodePath = useNodeStore ? await resolveNodeCheckpointPath(key, options) : null;
-  const data = useNodeStore
-    ? await readNodeCheckpointRecord(nodePath)
-    : await (async () => {
-      const { db, storeName } = await openCheckpointDB(options);
-      try {
-        return await readCheckpointRecord(db, storeName, key);
-      } finally {
-        closeCheckpointDB(db);
-      }
-    })();
-
-  if (!data || !data.metadata || !options.expectedMetadata) {
-    return data;
-  }
-
-  const mismatches = [];
-  for (const [k, v] of Object.entries(options.expectedMetadata)) {
-    if (data.metadata[k] !== v) {
-      mismatches.push(k);
-    }
-  }
-
-  if (mismatches.length > 0) {
-    if (!options.forceResume) {
-      throw new Error(`Checkpoint mismatch on fields: ${mismatches.join(', ')}`);
-    }
-    const priorCheckpointMetadataHash = data.metadata?.checkpointHash ?? null;
-    data.metadata.resumeAudits = data.metadata.resumeAudits || [];
-    data.metadata.resumeAudits.push({
-      timestamp: Date.now(),
-      mismatchedFields: mismatches,
-      source: options.forceResumeSource || 'unspecified',
-      operator: options.forceResumeOperator || null,
-      reason: options.forceResumeReason || 'forced resume flag provided',
-      priorCheckpointMetadataHash,
-    });
-    await saveCheckpoint(key, data, options);
-  }
-
-  return data;
-}

package/src/training/clip.d.ts

@@ -1,9 +0,0 @@
-import type { TrainingConfigSchema } from '../config/training-defaults.d.ts';
-import type { Tensor } from '../gpu/tensor.js';
-
-import type { ClipMetrics } from './trainer.d.ts';
-
-export declare function clipGradients(
-  grads: Map<Tensor, Tensor>,
-  config: TrainingConfigSchema
-): Promise<ClipMetrics>;

package/src/training/clip.js

@@ -1,55 +0,0 @@
-
-import { readBuffer } from '../memory/buffer-pool.js';
-import { runScale } from '../gpu/kernels/index.js';
-import { f16ToF32Array } from '../inference/kv-cache/types.js';
-
-async function readGradData(grad) {
-  const buffer = await readBuffer(grad.buffer);
-  if (grad.dtype === 'f16') {
-    return f16ToF32Array(new Uint16Array(buffer));
-  }
-  return new Float32Array(buffer);
-}
-
-export async function clipGradients(grads, config) {
-  const maxNorm = config?.training?.gradientClipping?.maxNorm
-    ?? config?.training?.gradient?.maxNorm;
-  let sumSq = 0;
-  let totalParamCount = 0;
-
-  for (const grad of grads.values()) {
-    const data = await readGradData(grad);
-    totalParamCount += data.length;
-    for (let i = 0; i < data.length; i += 1) {
-      const value = data[i];
-      sumSq += value * value;
-    }
-  }
-
-  const globalNorm = Math.sqrt(sumSq);
-
-  if (!maxNorm || maxNorm <= 0 || !globalNorm || globalNorm <= maxNorm) {
-    return {
-      clippedGrads: grads,
-      gradient_norm_unclipped: globalNorm,
-      gradient_norm_clipped: globalNorm,
-      clipped_event_count: 0,
-      total_param_count: totalParamCount,
-    };
-  }
-
-  const scale = maxNorm / (globalNorm + 1e-6);
-  const clipped = new Map();
-  for (const [param, grad] of grads.entries()) {
-    const scaled = await runScale(grad, scale, { inplace: true });
-    clipped.set(param, scaled);
-  }
-
-  return {
-    clippedGrads: clipped,
-    gradient_norm_unclipped: globalNorm,
-    gradient_norm_clipped: maxNorm,
-    clipped_event_count: 1,
-    total_param_count: totalParamCount,
-  };
-}

package/src/training/dataloader.d.ts

@@ -1,8 +0,0 @@
-export declare class DataLoader<T> {
-  constructor(dataset: T[], batchSize: number, shuffle?: boolean);
-  dataset: T[];
-  batchSize: number;
-  shuffle: boolean;
-  collate(batch: T[]): T[] | unknown;
-  batches(): AsyncGenerator<T[] | unknown, void, void>;
-}

package/src/training/dataloader.js

@@ -1,44 +0,0 @@
-let fallbackRandomState = (Date.now() >>> 0) || 0x6d2b79f5;
-
-function unseededRandom() {
-  if (typeof crypto !== 'undefined' && typeof crypto.getRandomValues === 'function') {
-    const values = new Uint32Array(1);
-    crypto.getRandomValues(values);
-    return values[0] / 4294967296;
-  }
-  fallbackRandomState = (fallbackRandomState + 0x6d2b79f5) >>> 0;
-  return fallbackRandomState / 4294967296;
-}
-
-function shuffledIndices(length) {
-  const indices = Array.from({ length }, (_, i) => i);
-  for (let i = indices.length - 1; i > 0; i -= 1) {
-    const j = Math.floor(unseededRandom() * (i + 1));
-    const tmp = indices[i];
-    indices[i] = indices[j];
-    indices[j] = tmp;
-  }
-  return indices;
-}
-
-export class DataLoader {
-  constructor(dataset, batchSize, shuffle = true) {
-    this.dataset = dataset;
-    this.batchSize = batchSize;
-    this.shuffle = shuffle;
-  }
-
-  collate(batch) {
-    return batch;
-  }
-
-  async *batches() {
-    const length = this.dataset.length ?? 0;
-    const indices = this.shuffle ? shuffledIndices(length) : Array.from({ length }, (_, i) => i);
-    for (let i = 0; i < indices.length; i += this.batchSize) {
-      const batchIndices = indices.slice(i, i + this.batchSize);
-      const batch = batchIndices.map((idx) => this.dataset[idx]);
-      yield this.collate(batch);
-    }
-  }
-}

package/src/training/datasets/index.d.ts

@@ -1,12 +0,0 @@
-export { parseJsonl, loadJsonl, mapJsonl } from './jsonl.js';
-export { tokenizeTextPairs, buildCausalPair, type TextPair, type TokenizedSample } from './text-pairs.js';
-export {
-  mapTranslationPairs,
-  tokenizeTranslationPairs,
-  type TranslationPair,
-  type TranslationTokenizedSample,
-  type MapTranslationPairsOptions,
-  type TokenizeTranslationPairsOptions,
-} from './translation-pairs.js';
-export { buildTokenBatch, createTokenBatchTensors, type TokenBatch } from './token-batch.js';
-export { reploidTracesToTextPairs } from './reploid.js';

package/src/training/datasets/index.js

@@ -1,6 +0,0 @@
-
-export { parseJsonl, loadJsonl, mapJsonl } from './jsonl.js';
-export { tokenizeTextPairs, buildCausalPair } from './text-pairs.js';
-export { mapTranslationPairs, tokenizeTranslationPairs } from './translation-pairs.js';
-export { buildTokenBatch, createTokenBatchTensors } from './token-batch.js';
-export { reploidTracesToTextPairs } from './reploid.js';

package/src/training/datasets/jsonl.d.ts

@@ -1,11 +0,0 @@
-export declare function parseJsonl(text: string): unknown[];
-
-export declare function loadJsonl(
-  source: string,
-  options?: { fetch?: (url: string) => Promise<string> }
-): Promise<unknown[]>;
-
-export declare function mapJsonl<T, U>(
-  records: T[],
-  mapper?: (record: T) => U | null | undefined
-): U[];

package/src/training/datasets/jsonl.js

@@ -1,50 +0,0 @@
-
-function parseLine(line, lineNumber) {
-  if (!line.trim()) {
-    return null;
-  }
-  try {
-    return JSON.parse(line);
-  } catch (err) {
-    const message = err instanceof Error ? err.message : String(err);
-    throw new Error(`Invalid JSONL at line ${lineNumber}: ${message}`);
-  }
-}
-
-export function parseJsonl(text) {
-  const records = [];
-  const lines = text.split(/\r?\n/);
-  for (let i = 0; i < lines.length; i += 1) {
-    const value = parseLine(lines[i], i + 1);
-    if (value !== null) {
-      records.push(value);
-    }
-  }
-  return records;
-}
-
-export async function loadJsonl(source, options = {}) {
-  if (typeof source !== 'string') {
-    throw new Error('loadJsonl expects a string source');
-  }
-
-  const looksLikeUrl = /^https?:\/\//i.test(source);
-  const text = looksLikeUrl
-    ? await (options.fetch ? options.fetch(source) : fetch(source).then((res) => res.text()))
-    : source;
-  return parseJsonl(text);
-}
-
-export function mapJsonl(records, mapper) {
-  if (!mapper) {
-    return records;
-  }
-  const mapped = [];
-  for (const record of records) {
-    const next = mapper(record);
-    if (next !== null && next !== undefined) {
-      mapped.push(next);
-    }
-  }
-  return mapped;
-}

package/src/training/datasets/reploid.d.ts

@@ -1,3 +0,0 @@
-import type { TextPair } from './text-pairs.js';
-
-export declare function reploidTracesToTextPairs(traces: Array<Record<string, unknown>>): TextPair[];

package/src/training/datasets/reploid.js

@@ -1,36 +0,0 @@
-
-function formatToolCall(tool, args, result) {
-  const argText = args ? JSON.stringify(args) : '';
-  const resultText = result ? JSON.stringify(result) : '';
-  return `tool:${tool}\nargs:${argText}\nresult:${resultText}`;
-}
-
-function extractPair(event) {
-  if (event.prompt && event.completion) {
-    return { prompt: event.prompt, completion: event.completion };
-  }
-  if (event.query && event.response) {
-    return { prompt: event.query, completion: event.response };
-  }
-  if (event.type === 'tool:execute' && event.payload) {
-    const tool = event.payload.tool || 'tool';
-    const args = event.payload.args || null;
-    const result = event.payload.result || event.payload.output || null;
-    return {
-      prompt: event.payload.prompt || event.payload.query || `call ${tool}`,
-      completion: formatToolCall(tool, args, result),
-    };
-  }
-  return null;
-}
-
-export function reploidTracesToTextPairs(traces) {
-  const pairs = [];
-  for (const event of traces) {
-    const pair = extractPair(event);
-    if (pair) {
-      pairs.push(pair);
-    }
-  }
-  return pairs;
-}

package/src/training/datasets/text-pairs.d.ts

@@ -1,21 +0,0 @@
-export interface TextPair {
-  prompt: string;
-  completion: string;
-}
-
-export interface TokenizedSample {
-  inputIds: number[];
-  targetIds: number[];
-  text?: string;
-}
-
-export declare function buildCausalPair(tokens: number[]): {
-  inputIds: number[];
-  targetIds: number[];
-};
-
-export declare function tokenizeTextPairs(
-  tokenizer: { encode: (text: string) => number[] },
-  pairs: TextPair[],
-  options?: { maxLength?: number | null; joinWith?: string }
-): Promise<TokenizedSample[]>;

package/src/training/datasets/text-pairs.js

@@ -1,42 +0,0 @@
-
-function concatTokens(tokens, maxLength) {
-  if (!maxLength || tokens.length <= maxLength) {
-    return tokens;
-  }
-  return tokens.slice(0, maxLength);
-}
-
-export function buildCausalPair(tokens) {
-  if (tokens.length < 2) {
-    return { inputIds: [], targetIds: [] };
-  }
-  return {
-    inputIds: tokens.slice(0, tokens.length - 1),
-    targetIds: tokens.slice(1),
-  };
-}
-
-export async function tokenizeTextPairs(tokenizer, pairs, options = {}) {
-  if (!tokenizer || typeof tokenizer.encode !== 'function') {
-    throw new Error('tokenizeTextPairs requires a tokenizer with encode()');
-  }
-
-  const {
-    maxLength = null,
-    joinWith = '',
-  } = options;
-
-  const samples = [];
-  for (const pair of pairs) {
-    const prompt = pair.prompt ?? '';
-    const completion = pair.completion ?? '';
-    const fullText = `${prompt}${joinWith}${completion}`;
-    const tokens = tokenizer.encode(fullText);
-    const clipped = concatTokens(tokens, maxLength);
-    const { inputIds, targetIds } = buildCausalPair(clipped);
-    if (inputIds.length > 0) {
-      samples.push({ inputIds, targetIds, text: fullText });
-    }
-  }
-  return samples;
-}

package/src/training/datasets/token-batch.d.ts

@@ -1,21 +0,0 @@
-import type { Tensor } from '../../gpu/tensor.js';
-
-export interface TokenizedSample {
-  inputIds: number[];
-  targetIds: number[];
-  text?: string;
-}
-
-export interface TokenBatch {
-  inputFlat: Uint32Array;
-  targetFlat: Uint32Array;
-  offsets: number[];
-}
-
-export declare function buildTokenBatch(samples: TokenizedSample[]): TokenBatch;
-
-export declare function createTokenBatchTensors(batch: TokenBatch): {
-  input: Tensor;
-  targets: Tensor;
-  offsets: number[];
-};

package/src/training/datasets/token-batch.js

@@ -1,52 +0,0 @@
-
-import { acquireBuffer, uploadData, releaseBuffer } from '../../memory/buffer-pool.js';
-import { createTensor } from '../../gpu/tensor.js';
-
-function flattenTokenBatch(samples, key) {
-  const offsets = [];
-  let total = 0;
-  for (const sample of samples) {
-    offsets.push(total);
-    total += sample[key].length;
-  }
-  const flat = new Uint32Array(total);
-  for (let i = 0; i < samples.length; i += 1) {
-    const ids = samples[i][key];
-    flat.set(ids, offsets[i]);
-  }
-  return { flat, offsets };
-}
-
-export function buildTokenBatch(samples) {
-  if (!samples.length) {
-    throw new Error('buildTokenBatch requires at least one sample');
-  }
-  const { flat: inputFlat, offsets } = flattenTokenBatch(samples, 'inputIds');
-  const { flat: targetFlat } = flattenTokenBatch(samples, 'targetIds');
-  return { inputFlat, targetFlat, offsets };
-}
-
-export function createTokenBatchTensors(batch) {
-  let inputBuf = null;
-  let targetBuf = null;
-  try {
-    inputBuf = acquireBuffer(batch.inputFlat.byteLength, undefined, 'train_input_tokens');
-    uploadData(inputBuf, batch.inputFlat);
-
-    targetBuf = acquireBuffer(batch.targetFlat.byteLength, undefined, 'train_target_tokens');
-    uploadData(targetBuf, batch.targetFlat);
-
-    const input = createTensor(inputBuf, 'f32', [batch.inputFlat.length], 'train_input_tokens');
-    const targets = createTensor(targetBuf, 'f32', [batch.targetFlat.length], 'train_target_tokens');
-
-    return { input, targets, offsets: batch.offsets };
-  } catch (error) {
-    if (inputBuf) {
-      releaseBuffer(inputBuf);
-    }
-    if (targetBuf) {
-      releaseBuffer(targetBuf);
-    }
-    throw error;
-  }
-}