@seanhogg/builderforce-memory-engine 2026.7.12 → 2026.7.14

package/dist/index.d.ts

@@ -10,9 +10,13 @@ export { MambaBlock } from './model/mamba1_block.js';
 export { SharedExpertMoE, LoadBalanceAccumulator, DEFAULT_MOE_CONFIG, DEFAULT_MOE_SEED, MoETrainer, EvermindModelPackage, } from './moe/index.js';
 export type { MoEConfig, MoEParam, RouteResult, MoESample, MoETrainOptions, MoEEpochResult, EvermindModelManifest, EvermindModelCard, EvermindModelType, EvermindModality, PackageMeta, ValidationResult, } from './moe/index.js';
 export { EvermindLM, EvermindLMTrainer, DEFAULT_LM_CONFIG, DEFAULT_LM_SEED } from './lm/index.js';
-export type { EvermindLMConfig, LMGenerateOptions, TextCodec } from './lm/index.js';
+export type { EvermindLMConfig, LMGenerateOptions, TextCodec, EvermindLMTrainOptions } from './lm/index.js';
 export { AdamW } from './optim/adamw.js';
-export type { AdamWOptions, OptimTarget, OptimParam } from './optim/adamw.js';
+export type { AdamWOptions, OptimTarget, OptimParam, ShardSpec } from './optim/adamw.js';
+export { LoRAAdapter, EvermindLMLoRA, quantizeBase } from './training/lora.js';
+export type { LoRAConfig, LoRAFitOptions, BaseQuant } from './training/lora.js';
+export { DynamicLossScaler, roundFp16, fp16View } from './training/mixed_precision.js';
+export type { LossScalerOptions } from './training/mixed_precision.js';
 export { MambaTrainer } from './training/trainer.js';
 export { Tensor, backward, enableGrad, noGrad, clearTape, recordOperation, crossEntropyLoss, crossEntropyGrad, } from './training/autograd.js';
 export { BPETokenizer } from './tokenizer/bpe.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAGtE,OAAO,EAAE,WAAW,EAAE,MAAQ,yBAAyB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,MAAQ,yBAAyB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,MAAQ,yBAAyB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAG5D,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAGrD,OAAO,EACH,eAAe,EACf,sBAAsB,EACtB,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,EACV,oBAAoB,GACvB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACR,SAAS,EACT,QAAQ,EACR,WAAW,EACX,SAAS,EACT,eAAe,EACf,cAAc,EACd,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EACjB,gBAAgB,EAChB,WAAW,EACX,gBAAgB,GACnB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAClG,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AACpF,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9E,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EACH,MAAM,EACN,QAAQ,EACR,UAAU,EACV,MAAM,EACN,SAAS,EACT,eAAe,EACf,gBAAgB,EAChB,gBAAgB,GACnB,MAAM,wBAAwB,CAAC;AAIhC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,YAAY,EAAE,gBAAgB,EAAE,OAAO,EAAE,wBAAwB,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAIrH,OAAO,EACH,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,aAAa,EACb,aAAa,EACb,kBAAkB,EAClB,aAAa,EACb,aAAa,GAChB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACR,qBAAqB,EACrB,SAAS,EACT,cAAc,EACd,KAAK,EACL,KAAK,GACR,MAAM,kBAAkB,CAAC;AAI1B,OAAO,EACH,cAAc,EACd,cAAc,EACd,iBAAiB,EACjB,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,UAAU,EACV,oBAAoB,EACpB,aAAa,EACb,iBAAiB,EACjB,YAAY,EACZ,kBAAkB,EAClB,MAAM,EACN,UAAU,GACb,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EACR,YAAY,EACZ,UAAU,EACV,YAAY,EACZ,aAAa,EACb,MAAM,EACN,WAAW,EACX,YAAY,EACZ,UAAU,GACb,MAAM,mBAAmB,CAAC;AAI3B,OAAO,EACH,oBAAoB,EACpB,cAAc,EACd,qBAAqB,EACrB,oBAAoB,GACvB,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAIvD,OAAO,EACH,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,iBAAiB,EACjB,MAAM,IAAI,WAAW,EACrB,WAAW,EACX,UAAU,EACV,YAAY,EACZ,GAAG,GACN,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACR,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,EACf,gBAAgB,EAChB,wBAAwB,EACxB,uBAAuB,GAC1B,MAAM,kBAAkB,CAAC;AAI1B,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAChG,YAAY,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EACH,eAAe,EACf,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,EACf,mBAAmB,GACtB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAIjD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAI9E,YAAY,EACR,sBAAsB,EACtB,gBAAgB,EAChB,kBAAkB,EAClB,eAAe,EACf,SAAS,GACZ,MAAM,wBAAwB,CAAC;AAEhC,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC1G,YAAY,EAAE,iBAAiB,EAAE,UAAU,EAAE,UAAU,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC/H,YAAY,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAC9E,YAAY,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAO,yBAAyB,CAAC;AAC/E,YAAY,EAAE,oBAAoB,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAIvF,OAAO,EACH,UAAU,EACV,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,qBAAqB,EACrB,eAAe,EACf,cAAc,EACd,UAAU,EACV,YAAY,EACZ,IAAI,EACJ,UAAU,GACb,MAAM,sBAAsB,CAAC;AAI9B,OAAO,EACH,YAAY,EACZ,cAAc,EACd,WAAW,EACX,WAAW,EACX,YAAY,EACZ,cAAc,EACd,sBAAsB,EACtB,wBAAwB,EACxB,cAAc,GACjB,MAAM,yBAAyB,CAAC;AAKjC,OAAO,EAAE,2BAA2B,EAAE,4BAA4B,EAAE,MAC3D,6BAA6B,CAAC;AACvC,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAC3C,qBAAqB,CAAC;AAC/B,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAC3C,gCAAgC,CAAC;AAC1C,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MACpC,4BAA4B,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,yBAAyB,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAC1F,0BAA0B,CAAC;AAGpC,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MACrC,kBAAkB,CAAC;AAG5B,OAAO,EAAE,wBAAwB,EAAE,yBAAyB,EAAE,MACrD,0BAA0B,CAAC;AAGpC,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,YAAY,EAAE,MAC/D,wBAAwB,CAAC;AAIlC,OAAO,EACH,MAAM,EACN,UAAU,EACV,gBAAgB,EAChB,gBAAgB,EAChB,aAAa,EACb,aAAa,EACb,QAAQ,EACR,UAAU,EACV,YAAY,EACZ,aAAa,EACb,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,qBAAqB,EACrB,mBAAmB,EACnB,aAAa,GAChB,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EACR,MAAM,EACN,aAAa,EACb,iBAAiB,EACjB,iBAAiB,EACjB,aAAa,EACb,WAAW,EACX,YAAY,EACZ,kBAAkB,GACrB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAIhE,eAAO,MAAM,OAAO,UAAc,CAAC;AACnC,eAAO,MAAM,WAAW,wFAAwF,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAGtE,OAAO,EAAE,WAAW,EAAE,MAAQ,yBAAyB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,MAAQ,yBAAyB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,MAAQ,yBAAyB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAG5D,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAGrD,OAAO,EACH,eAAe,EACf,sBAAsB,EACtB,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,EACV,oBAAoB,GACvB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACR,SAAS,EACT,QAAQ,EACR,WAAW,EACX,SAAS,EACT,eAAe,EACf,cAAc,EACd,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EACjB,gBAAgB,EAChB,WAAW,EACX,gBAAgB,GACnB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAClG,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,SAAS,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAC5G,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAGzF,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAC/E,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,+BAA+B,CAAC;AACvF,YAAY,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAIvE,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EACH,MAAM,EACN,QAAQ,EACR,UAAU,EACV,MAAM,EACN,SAAS,EACT,eAAe,EACf,gBAAgB,EAChB,gBAAgB,GACnB,MAAM,wBAAwB,CAAC;AAIhC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,YAAY,EAAE,gBAAgB,EAAE,OAAO,EAAE,wBAAwB,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAIrH,OAAO,EACH,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,aAAa,EACb,aAAa,EACb,kBAAkB,EAClB,aAAa,EACb,aAAa,GAChB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACR,qBAAqB,EACrB,SAAS,EACT,cAAc,EACd,KAAK,EACL,KAAK,GACR,MAAM,kBAAkB,CAAC;AAI1B,OAAO,EACH,cAAc,EACd,cAAc,EACd,iBAAiB,EACjB,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,UAAU,EACV,oBAAoB,EACpB,aAAa,EACb,iBAAiB,EACjB,YAAY,EACZ,kBAAkB,EAClB,MAAM,EACN,UAAU,GACb,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EACR,YAAY,EACZ,UAAU,EACV,YAAY,EACZ,aAAa,EACb,MAAM,EACN,WAAW,EACX,YAAY,EACZ,UAAU,GACb,MAAM,mBAAmB,CAAC;AAI3B,OAAO,EACH,oBAAoB,EACpB,cAAc,EACd,qBAAqB,EACrB,oBAAoB,GACvB,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAIvD,OAAO,EACH,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,iBAAiB,EACjB,MAAM,IAAI,WAAW,EACrB,WAAW,EACX,UAAU,EACV,YAAY,EACZ,GAAG,GACN,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACR,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,EACf,gBAAgB,EAChB,wBAAwB,EACxB,uBAAuB,GAC1B,MAAM,kBAAkB,CAAC;AAI1B,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAChG,YAAY,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EACH,eAAe,EACf,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,EACf,mBAAmB,GACtB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAIjD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAI9E,YAAY,EACR,sBAAsB,EACtB,gBAAgB,EAChB,kBAAkB,EAClB,eAAe,EACf,SAAS,GACZ,MAAM,wBAAwB,CAAC;AAEhC,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC1G,YAAY,EAAE,iBAAiB,EAAE,UAAU,EAAE,UAAU,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC/H,YAAY,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAC9E,YAAY,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAO,yBAAyB,CAAC;AAC/E,YAAY,EAAE,oBAAoB,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAIvF,OAAO,EACH,UAAU,EACV,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,qBAAqB,EACrB,eAAe,EACf,cAAc,EACd,UAAU,EACV,YAAY,EACZ,IAAI,EACJ,UAAU,GACb,MAAM,sBAAsB,CAAC;AAI9B,OAAO,EACH,YAAY,EACZ,cAAc,EACd,WAAW,EACX,WAAW,EACX,YAAY,EACZ,cAAc,EACd,sBAAsB,EACtB,wBAAwB,EACxB,cAAc,GACjB,MAAM,yBAAyB,CAAC;AAKjC,OAAO,EAAE,2BAA2B,EAAE,4BAA4B,EAAE,MAC3D,6BAA6B,CAAC;AACvC,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAC3C,qBAAqB,CAAC;AAC/B,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAC3C,gCAAgC,CAAC;AAC1C,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MACpC,4BAA4B,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,yBAAyB,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAC1F,0BAA0B,CAAC;AAGpC,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MACrC,kBAAkB,CAAC;AAG5B,OAAO,EAAE,wBAAwB,EAAE,yBAAyB,EAAE,MACrD,0BAA0B,CAAC;AAGpC,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,YAAY,EAAE,MAC/D,wBAAwB,CAAC;AAIlC,OAAO,EACH,MAAM,EACN,UAAU,EACV,gBAAgB,EAChB,gBAAgB,EAChB,aAAa,EACb,aAAa,EACb,QAAQ,EACR,UAAU,EACV,YAAY,EACZ,aAAa,EACb,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,qBAAqB,EACrB,mBAAmB,EACnB,aAAa,GAChB,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EACR,MAAM,EACN,aAAa,EACb,iBAAiB,EACjB,iBAAiB,EACjB,aAAa,EACb,WAAW,EACX,YAAY,EACZ,kBAAkB,GACrB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAIhE,eAAO,MAAM,OAAO,UAAc,CAAC;AACnC,eAAO,MAAM,WAAW,wFAAwF,CAAC"}

package/dist/index.js

@@ -15,6 +15,9 @@ export { SharedExpertMoE, LoadBalanceAccumulator, DEFAULT_MOE_CONFIG, DEFAULT_MO
 // ── EvermindLM (the generative model) + AdamW ──────────────────────────────────
 export { EvermindLM, EvermindLMTrainer, DEFAULT_LM_CONFIG, DEFAULT_LM_SEED } from './lm/index.js';
 export { AdamW } from './optim/adamw.js';
+// PEFT / efficient-training toolkit (LoRA, QLoRA, mixed precision, checkpointing).
+export { LoRAAdapter, EvermindLMLoRA, quantizeBase } from './training/lora.js';
+export { DynamicLossScaler, roundFp16, fp16View } from './training/mixed_precision.js';
 // ── Training ──────────────────────────────────────────────────────────────────
 export { MambaTrainer } from './training/trainer.js';
 export { Tensor, backward, enableGrad, noGrad, clearTape, recordOperation, crossEntropyLoss, crossEntropyGrad, } from './training/autograd.js';

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,iFAAiF;AAEjF,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEtE,oBAAoB;AACpB,OAAO,EAAE,WAAW,EAAE,MAAQ,yBAAyB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,MAAQ,yBAAyB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,MAAQ,yBAAyB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAE5D,sCAAsC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAErD,kFAAkF;AAClF,OAAO,EACH,eAAe,EACf,sBAAsB,EACtB,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,EACV,oBAAoB,GACvB,MAAM,gBAAgB,CAAC;AAgBxB,kFAAkF;AAClF,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAElG,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAGzC,iFAAiF;AAEjF,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EACH,MAAM,EACN,QAAQ,EACR,UAAU,EACV,MAAM,EACN,SAAS,EACT,eAAe,EACf,gBAAgB,EAChB,gBAAgB,GACnB,MAAM,wBAAwB,CAAC;AAEhC,iFAAiF;AAEjF,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAGlD,mFAAmF;AAEnF,OAAO,EACH,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,aAAa,EACb,aAAa,EACb,kBAAkB,EAClB,aAAa,EACb,aAAa,GAChB,MAAM,kBAAkB,CAAC;AAS1B,mFAAmF;AAEnF,OAAO,EACH,cAAc,EACd,cAAc,EACd,iBAAiB,EACjB,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,UAAU,EACV,oBAAoB,EACpB,aAAa,EACb,iBAAiB,EACjB,YAAY,EACZ,kBAAkB,EAClB,MAAM,EACN,UAAU,GACb,MAAM,mBAAmB,CAAC;AAY3B,oFAAoF;AAEpF,OAAO,EACH,oBAAoB,EACpB,cAAc,EACd,qBAAqB,EACrB,oBAAoB,GACvB,MAAM,mBAAmB,CAAC;AAG3B,iFAAiF;AAEjF,OAAO,EACH,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,iBAAiB,EACjB,MAAM,IAAI,WAAW,EACrB,WAAW,EACX,UAAU,EACV,YAAY,EACZ,GAAG,GACN,MAAM,kBAAkB,CAAC;AAW1B,iFAAiF;AAEjF,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAEhG,OAAO,EACH,eAAe,EACf,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,EACf,mBAAmB,GACtB,MAAM,kBAAkB,CAAC;AAG1B,iFAAiF;AAEjF,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAkB9E,iFAAiF;AAEjF,OAAO,EACH,UAAU,EACV,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,qBAAqB,EACrB,eAAe,EACf,cAAc,EACd,UAAU,EACV,YAAY,EACZ,IAAI,EACJ,UAAU,GACb,MAAM,sBAAsB,CAAC;AAE9B,iFAAiF;AAEjF,OAAO,EACH,YAAY,EACZ,cAAc,EACd,WAAW,EACX,WAAW,EACX,YAAY,EACZ,cAAc,EACd,sBAAsB,EACtB,wBAAwB,EACxB,cAAc,GACjB,MAAM,yBAAyB,CAAC;AAEjC,iFAAiF;AAEjF,8BAA8B;AAC9B,OAAO,EAAE,2BAA2B,EAAE,4BAA4B,EAAE,MAC3D,6BAA6B,CAAC;AACvC,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAC3C,qBAAqB,CAAC;AAC/B,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAC3C,gCAAgC,CAAC;AAC1C,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MACpC,4BAA4B,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,yBAAyB,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAC1F,0BAA0B,CAAC;AAEpC,sBAAsB;AACtB,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MACrC,kBAAkB,CAAC;AAE5B,8BAA8B;AAC9B,OAAO,EAAE,wBAAwB,EAAE,yBAAyB,EAAE,MACrD,0BAA0B,CAAC;AAEpC,oBAAoB;AACpB,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,YAAY,EAAE,MAC/D,wBAAwB,CAAC;AAElC,iFAAiF;AAEjF,OAAO,EACH,MAAM,EACN,UAAU,EACV,gBAAgB,EAChB,gBAAgB,EAChB,aAAa,EACb,aAAa,EACb,QAAQ,EACR,UAAU,EACV,YAAY,EACZ,aAAa,EACb,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,qBAAqB,EACrB,mBAAmB,EACnB,aAAa,GAChB,MAAM,mBAAmB,CAAC;AAY3B,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAEhE,iFAAiF;AAEjF,MAAM,CAAC,MAAM,OAAO,GAAO,OAAO,CAAC;AACnC,MAAM,CAAC,MAAM,WAAW,GAAG,qFAAqF,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,iFAAiF;AAEjF,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEtE,oBAAoB;AACpB,OAAO,EAAE,WAAW,EAAE,MAAQ,yBAAyB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,MAAQ,yBAAyB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,MAAQ,yBAAyB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAE5D,sCAAsC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAErD,kFAAkF;AAClF,OAAO,EACH,eAAe,EACf,sBAAsB,EACtB,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,EACV,oBAAoB,GACvB,MAAM,gBAAgB,CAAC;AAgBxB,kFAAkF;AAClF,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAElG,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAGzC,mFAAmF;AACnF,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAE/E,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,+BAA+B,CAAC;AAGvF,iFAAiF;AAEjF,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EACH,MAAM,EACN,QAAQ,EACR,UAAU,EACV,MAAM,EACN,SAAS,EACT,eAAe,EACf,gBAAgB,EAChB,gBAAgB,GACnB,MAAM,wBAAwB,CAAC;AAEhC,iFAAiF;AAEjF,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAGlD,mFAAmF;AAEnF,OAAO,EACH,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,aAAa,EACb,aAAa,EACb,kBAAkB,EAClB,aAAa,EACb,aAAa,GAChB,MAAM,kBAAkB,CAAC;AAS1B,mFAAmF;AAEnF,OAAO,EACH,cAAc,EACd,cAAc,EACd,iBAAiB,EACjB,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,UAAU,EACV,oBAAoB,EACpB,aAAa,EACb,iBAAiB,EACjB,YAAY,EACZ,kBAAkB,EAClB,MAAM,EACN,UAAU,GACb,MAAM,mBAAmB,CAAC;AAY3B,oFAAoF;AAEpF,OAAO,EACH,oBAAoB,EACpB,cAAc,EACd,qBAAqB,EACrB,oBAAoB,GACvB,MAAM,mBAAmB,CAAC;AAG3B,iFAAiF;AAEjF,OAAO,EACH,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,iBAAiB,EACjB,MAAM,IAAI,WAAW,EACrB,WAAW,EACX,UAAU,EACV,YAAY,EACZ,GAAG,GACN,MAAM,kBAAkB,CAAC;AAW1B,iFAAiF;AAEjF,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAEhG,OAAO,EACH,eAAe,EACf,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,EACf,mBAAmB,GACtB,MAAM,kBAAkB,CAAC;AAG1B,iFAAiF;AAEjF,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAkB9E,iFAAiF;AAEjF,OAAO,EACH,UAAU,EACV,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,qBAAqB,EACrB,eAAe,EACf,cAAc,EACd,UAAU,EACV,YAAY,EACZ,IAAI,EACJ,UAAU,GACb,MAAM,sBAAsB,CAAC;AAE9B,iFAAiF;AAEjF,OAAO,EACH,YAAY,EACZ,cAAc,EACd,WAAW,EACX,WAAW,EACX,YAAY,EACZ,cAAc,EACd,sBAAsB,EACtB,wBAAwB,EACxB,cAAc,GACjB,MAAM,yBAAyB,CAAC;AAEjC,iFAAiF;AAEjF,8BAA8B;AAC9B,OAAO,EAAE,2BAA2B,EAAE,4BAA4B,EAAE,MAC3D,6BAA6B,CAAC;AACvC,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAC3C,qBAAqB,CAAC;AAC/B,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAC3C,gCAAgC,CAAC;AAC1C,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MACpC,4BAA4B,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,yBAAyB,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAC1F,0BAA0B,CAAC;AAEpC,sBAAsB;AACtB,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MACrC,kBAAkB,CAAC;AAE5B,8BAA8B;AAC9B,OAAO,EAAE,wBAAwB,EAAE,yBAAyB,EAAE,MACrD,0BAA0B,CAAC;AAEpC,oBAAoB;AACpB,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,YAAY,EAAE,MAC/D,wBAAwB,CAAC;AAElC,iFAAiF;AAEjF,OAAO,EACH,MAAM,EACN,UAAU,EACV,gBAAgB,EAChB,gBAAgB,EAChB,aAAa,EACb,aAAa,EACb,QAAQ,EACR,UAAU,EACV,YAAY,EACZ,aAAa,EACb,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,qBAAqB,EACrB,mBAAmB,EACnB,aAAa,GAChB,MAAM,mBAAmB,CAAC;AAY3B,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAEhE,iFAAiF;AAEjF,MAAM,CAAC,MAAM,OAAO,GAAO,OAAO,CAAC;AACnC,MAAM,CAAC,MAAM,WAAW,GAAG,qFAAqF,CAAC"}

package/dist/lm/evermind_lm.d.ts

@@ -21,6 +21,7 @@
  * acceleration with the same shapes.
  */
 import { type AdamWOptions } from "../optim/adamw.js";
+import { DynamicLossScaler, type LossScalerOptions } from "../training/mixed_precision.js";
 export interface EvermindLMConfig {
     /** Vocabulary size (the only required field; everything else has a default). */
     vocabSize: number;
@@ -97,17 +98,53 @@ export declare class EvermindLM {
     /** Per-layer channel mixer. */
     private readonly moe;
     constructor(config: EvermindLMConfig);
+    /** Embed a token sequence into per-position channel vectors. */
+    private _embed;
+    /**
+     * One (conv + MoE) block: pre-norm → depthwise causal conv → residual, then
+     * pre-norm → MoE channel mixer → residual. Returns the block output and the
+     * activation cache its backward needs. Isolating this is what lets
+     * {@link lossAndBackwardCheckpointed} recompute a layer's activations on demand
+     * instead of retaining every layer's cache at once.
+     */
+    private _forwardLayer;
+    /** Tied output head: logits_t[v] = x_t · emb[v]. */
+    private _head;
     /** Run the model over a token sequence; returns per-position logits + a cache. */
     forward(tokens: number[]): {
         logits: Float32Array[];
         cache: ForwardCache;
     };
+    /**
+     * Head + tied-embedding gradient. Accumulates dL/d(head→emb) into gEmb and
+     * returns the mean next-token loss plus dL/d(finalX). Shared by the full and
+     * checkpointed backward paths so the head maths lives in one place.
+     */
+    private _headBackward;
+    /**
+     * Backward through one (conv + MoE) block given dL/d(block output) and the
+     * block's activation cache. Accumulates conv/norm/MoE gradients and returns
+     * dL/d(block input). The inverse of {@link _forwardLayer}.
+     */
+    private _backwardLayer;
+    /** Embedding lookup: dL/d(layer-0 input) flows into the row for each token. */
+    private _embedBackward;
     /**
      * Next-token cross-entropy over the sequence (predict tokens[t+1] from
      * position t), accumulating exact gradients. Returns the mean loss. Call
      * {@link zeroGrad} before and an optimiser step after.
      */
     lossAndBackward(tokens: number[]): number;
+    /**
+     * Activation-checkpointed backward — numerically identical gradients to
+     * {@link lossAndBackward}, but retains only the per-LAYER inputs during the
+     * forward instead of every layer's full activation cache. Each layer's
+     * activations are RECOMPUTED (a cheap extra forward) when its backward runs,
+     * so peak activation memory is one layer's cache, not all of them. This is the
+     * memory-for-compute trade the cookbook pairs with FSDP to fit longer
+     * sequences / bigger models on a constrained device.
+     */
+    lossAndBackwardCheckpointed(tokens: number[]): number;
     /**
      * Text-level generation: encode the prompt, generate, decode. `codec` is any
      * tokenizer exposing encode/decode (the engine's `BPETokenizer` satisfies it),
@@ -149,14 +186,44 @@ export declare class EvermindLM {
     /** Reconstruct weights from a base EVL0 checkpoint + a delta (EVM-6). */
     loadDelta(baseCheckpoint: ArrayBuffer, delta: ArrayBuffer): void;
 }
+export interface EvermindLMTrainOptions extends AdamWOptions {
+    epochs?: number;
+    /**
+     * Gradient accumulation: average gradients over this many sequences (micro-
+     * batches) before each optimiser step, for a larger effective batch on a
+     * memory-constrained device. Default 1.
+     */
+    accumSteps?: number;
+    /**
+     * Use activation checkpointing (recompute layer activations in backward) to
+     * cap peak activation memory. Identical gradients, a little extra compute.
+     * Default false.
+     */
+    checkpoint?: boolean;
+    /**
+     * Mixed-precision training: fp16-rounded gradients with dynamic loss scaling
+     * over fp32 master weights (the model params). Overflowing steps are skipped
+     * and the scale backs off. Default false.
+     */
+    mixedPrecision?: boolean | LossScalerOptions;
+}
 /** Minimal sequence trainer: AdamW over next-token cross-entropy. */
 export declare class EvermindLMTrainer {
     private readonly model;
     private readonly opts;
     private readonly adam;
-    constructor(model: EvermindLM, opts?: AdamWOptions & {
-        epochs?: number;
-    });
+    private readonly scaler;
+    private readonly grads;
+    constructor(model: EvermindLM, opts?: EvermindLMTrainOptions);
+    /** The dynamic loss scaler (mixed-precision mode only) — exposes scale/overflow stats. */
+    get lossScaler(): DynamicLossScaler | null;
+    private _backward;
+    /** Divide accumulated gradients by `n` (accumulation averaging), in place. */
+    private _scaleGrads;
+    /** Round accumulated gradients to fp16 precision (mixed-precision simulation), in place. */
+    private _fp16Grads;
+    /** Multiply accumulated gradients by `s`, in place. */
+    private _mulGrads;
     /** Train on a set of token sequences; returns per-epoch mean loss. */
     fit(sequences: number[][]): number[];
 }

package/dist/lm/evermind_lm.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evermind_lm.d.ts","sourceRoot":"","sources":["../../src/lm/evermind_lm.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAIH,OAAO,EAAS,KAAK,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAM7D,MAAM,WAAW,gBAAgB;IAC/B,gFAAgF;IAChF,SAAS,EAAE,MAAM,CAAC;IAClB,6CAA6C;IAC7C,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gDAAgD;IAChD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2CAA2C;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+CAA+C;IAC/C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,8CAA8C;IAC9C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,eAAO,MAAM,iBAAiB,EAAE,QAAQ,CAAC,IAAI,CAAC,gBAAgB,EAAE,MAAM,GAAG,WAAW,CAAC,CAOpF,CAAC;AAEF,eAAO,MAAM,eAAe,aAAa,CAAC;AAG1C,UAAU,YAAY;IACpB,CAAC,EAAE,YAAY,CAAC;IAChB,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,EAAE,YAAY,CAAA;KAAE,CAAC;IACnE,SAAS,EAAE,YAAY,CAAC;IACxB,OAAO,EAAE,YAAY,CAAC;IACtB,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,OAAO,EAAE,YAAY,EAAE,CAAC;CACzB;AAED,UAAU,UAAU;IAClB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,UAAU,EAAE,YAAY,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,YAAY,EAAE,CAAC;CAC1B;AAED,UAAU,YAAY;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED,yFAAyF;AACzF,MAAM,WAAW,SAAS;IACxB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC/B,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,iBAAiB;IAChC,YAAY,EAAE,MAAM,CAAC;IACrB,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mEAAmE;IACnE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,UAAU;IACrB,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC,CAAC;IAE1D,0EAA0E;IAC1E,GAAG,EAAE,YAAY,CAAC;IAClB,OAAO,CAAC,IAAI,CAAe;IAC3B,oEAAoE;IACpE,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiB;IACtC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,gEAAgE;IAChE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiB;IACtC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,+BAA+B;IAC/B,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAoB;gBAE5B,MAAM,EAAE,gBAAgB;IA+DpC,kFAAkF;IAClF,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG;QAAE,MAAM,EAAE,YAAY,EAAE,CAAC;QAAC,KAAK,EAAE,YAAY,CAAA;KAAE;IA4E1E;;;;OAIG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM;IAqFzC;;;;;OAKG;IACH,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,iBAAiB,GAAG,MAAM;IAI/E,qFAAqF;IACrF,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,iBAAiB,GAAG,MAAM,EAAE;IAkB7D,8EAA8E;IAC9E,UAAU,IAAI;QAAE,IAAI,EAAE,YAAY,CAAA;KAAE,EAAE;IAStC,+DAA+D;IAC/D,SAAS,IAAI;QAAE,IAAI,EAAE,YAAY,CAAA;KAAE,EAAE;IASrC,QAAQ,IAAI,IAAI;IAUhB,uFAAuF;IACvF,aAAa,CAAC,IAAI,GAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,WAAW;IAiCzD,sFAAsF;IACtF,WAAW,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI;IAItC,oFAAoF;IACpF,OAAO,CAAC,SAAS;IA0BjB,qFAAqF;IACrF,OAAO,CAAC,KAAK;IASb,uEAAuE;IACvE,OAAO,CAAC,QAAQ;IAQhB;;;;OAIG;IACH,WAAW,CAAC,cAAc,EAAE,WAAW,EAAE,IAAI,GAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,WAAW;IASlF,yEAAyE;IACzE,SAAS,CAAC,cAAc,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,GAAG,IAAI;CAIjE;AAED,qEAAqE;AACrE,qBAAa,iBAAiB;IAG1B,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAHvB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAQ;gBAEV,KAAK,EAAE,UAAU,EACjB,IAAI,GAAE,YAAY,GAAG;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAO;IAIhE,sEAAsE;IACtE,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE;CAiBrC"}
+{"version":3,"file":"evermind_lm.d.ts","sourceRoot":"","sources":["../../src/lm/evermind_lm.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAIH,OAAO,EAAS,KAAK,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAa,KAAK,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AAMtG,MAAM,WAAW,gBAAgB;IAC/B,gFAAgF;IAChF,SAAS,EAAE,MAAM,CAAC;IAClB,6CAA6C;IAC7C,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gDAAgD;IAChD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2CAA2C;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+CAA+C;IAC/C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,8CAA8C;IAC9C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,eAAO,MAAM,iBAAiB,EAAE,QAAQ,CAAC,IAAI,CAAC,gBAAgB,EAAE,MAAM,GAAG,WAAW,CAAC,CAOpF,CAAC;AAEF,eAAO,MAAM,eAAe,aAAa,CAAC;AAG1C,UAAU,YAAY;IACpB,CAAC,EAAE,YAAY,CAAC;IAChB,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,EAAE,YAAY,CAAA;KAAE,CAAC;IACnE,SAAS,EAAE,YAAY,CAAC;IACxB,OAAO,EAAE,YAAY,CAAC;IACtB,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,OAAO,EAAE,YAAY,EAAE,CAAC;CACzB;AAED,UAAU,UAAU;IAClB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,UAAU,EAAE,YAAY,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,YAAY,EAAE,CAAC;CAC1B;AAED,UAAU,YAAY;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED,yFAAyF;AACzF,MAAM,WAAW,SAAS;IACxB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC/B,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,iBAAiB;IAChC,YAAY,EAAE,MAAM,CAAC;IACrB,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mEAAmE;IACnE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,UAAU;IACrB,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC,CAAC;IAE1D,0EAA0E;IAC1E,GAAG,EAAE,YAAY,CAAC;IAClB,OAAO,CAAC,IAAI,CAAe;IAC3B,oEAAoE;IACpE,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiB;IACtC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,gEAAgE;IAChE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiB;IACtC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,+BAA+B;IAC/B,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAoB;gBAE5B,MAAM,EAAE,gBAAgB;IA+DpC,gEAAgE;IAChE,OAAO,CAAC,MAAM;IAUd;;;;;;OAMG;IACH,OAAO,CAAC,aAAa;IA2CrB,oDAAoD;IACpD,OAAO,CAAC,KAAK;IAcb,kFAAkF;IAClF,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG;QAAE,MAAM,EAAE,YAAY,EAAE,CAAC;QAAC,KAAK,EAAE,YAAY,CAAA;KAAE;IAa1E;;;;OAIG;IACH,OAAO,CAAC,aAAa;IAyBrB;;;;OAIG;IACH,OAAO,CAAC,cAAc;IA8CtB,+EAA+E;IAC/E,OAAO,CAAC,cAAc;IAQtB;;;;OAIG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM;IAUzC;;;;;;;;OAQG;IACH,2BAA2B,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM;IAqBrD;;;;;OAKG;IACH,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,iBAAiB,GAAG,MAAM;IAI/E,qFAAqF;IACrF,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,iBAAiB,GAAG,MAAM,EAAE;IAkB7D,8EAA8E;IAC9E,UAAU,IAAI;QAAE,IAAI,EAAE,YAAY,CAAA;KAAE,EAAE;IAStC,+DAA+D;IAC/D,SAAS,IAAI;QAAE,IAAI,EAAE,YAAY,CAAA;KAAE,EAAE;IASrC,QAAQ,IAAI,IAAI;IAUhB,uFAAuF;IACvF,aAAa,CAAC,IAAI,GAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,WAAW;IAiCzD,sFAAsF;IACtF,WAAW,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI;IAItC,oFAAoF;IACpF,OAAO,CAAC,SAAS;IA0BjB,qFAAqF;IACrF,OAAO,CAAC,KAAK;IASb,uEAAuE;IACvE,OAAO,CAAC,QAAQ;IAQhB;;;;OAIG;IACH,WAAW,CAAC,cAAc,EAAE,WAAW,EAAE,IAAI,GAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,WAAW;IASlF,yEAAyE;IACzE,SAAS,CAAC,cAAc,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,GAAG,IAAI;CAIjE;AAED,MAAM,WAAW,sBAAuB,SAAQ,YAAY;IAC1D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,GAAG,iBAAiB,CAAC;CAC9C;AAED,qEAAqE;AACrE,qBAAa,iBAAiB;IAK1B,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,IAAI;IALvB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAQ;IAC7B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAClD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA2B;gBAE9B,KAAK,EAAE,UAAU,EACjB,IAAI,GAAE,sBAA2B;IASpD,0FAA0F;IAC1F,IAAI,UAAU,IAAI,iBAAiB,GAAG,IAAI,CAEzC;IAED,OAAO,CAAC,SAAS;IAIjB,8EAA8E;IAC9E,OAAO,CAAC,WAAW;IAMnB,4FAA4F;IAC5F,OAAO,CAAC,UAAU;IAIlB,uDAAuD;IACvD,OAAO,CAAC,SAAS;IAKjB,sEAAsE;IACtE,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE;CA8CrC"}

package/dist/lm/evermind_lm.js

@@ -23,6 +23,7 @@
 import { SharedExpertMoE } from "../moe/moe_model.js";
 import { crossEntropyLoss, crossEntropyGrad } from "../training/autograd.js";
 import { AdamW } from "../optim/adamw.js";
+import { DynamicLossScaler, roundFp16 } from "../training/mixed_precision.js";
 import { SeededRng } from "../utils/rng.js";
 import { quantizeFp16, dequantizeFp16 } from "../utils/quantization.js";
 import { appendCrcTrailer, verifyCrcTrailer } from "../utils/crc32.js";
@@ -112,65 +113,70 @@ export class EvermindLM {
         }
     }
     // ── Forward ────────────────────────────────────────────────────────────────
-    /** Run the model over a token sequence; returns per-position logits + a cache. */
-    forward(tokens) {
-        const { dModel, convKernel, numLayers, vocabSize } = this.config;
-        const T = tokens.length;
-        // Embed.
-        let x = tokens.map((tok) => {
+    /** Embed a token sequence into per-position channel vectors. */
+    _embed(tokens) {
+        const { dModel } = this.config;
+        return tokens.map((tok) => {
             const row = new Float32Array(dModel);
             const off = tok * dModel;
             for (let c = 0; c < dModel; c++)
                 row[c] = this.emb[off + c];
             return row;
         });
-        const layers = [];
-        for (let l = 0; l < numLayers; l++) {
-            const layerIn = x;
-            const ker = this.conv[l];
-            const nConv = this.nConv[l];
-            const nMoe = this.nMoe[l];
-            // Pre-norm → depthwise causal conv → residual.
-            const normedConv = [];
-            const rmsConv = [];
-            for (let t = 0; t < T; t++) {
-                const { y, r } = rmsNorm(layerIn[t], nConv);
-                normedConv.push(y);
-                rmsConv.push(r);
-            }
-            const afterConv = [];
-            for (let t = 0; t < T; t++) {
-                const out = Float32Array.from(layerIn[t]); // residual base
-                for (let c = 0; c < dModel; c++) {
-                    let acc = 0;
-                    for (let j = 0; j < convKernel; j++) {
-                        const ti = t - j;
-                        if (ti >= 0)
-                            acc += ker[c * convKernel + j] * normedConv[ti][c];
-                    }
-                    out[c] = out[c] + acc;
+    }
+    /**
+     * One (conv + MoE) block: pre-norm → depthwise causal conv → residual, then
+     * pre-norm → MoE channel mixer → residual. Returns the block output and the
+     * activation cache its backward needs. Isolating this is what lets
+     * {@link lossAndBackwardCheckpointed} recompute a layer's activations on demand
+     * instead of retaining every layer's cache at once.
+     */
+    _forwardLayer(l, layerIn) {
+        const { dModel, convKernel } = this.config;
+        const T = layerIn.length;
+        const ker = this.conv[l];
+        const nConv = this.nConv[l];
+        const nMoe = this.nMoe[l];
+        const normedConv = [];
+        const rmsConv = [];
+        for (let t = 0; t < T; t++) {
+            const { y, r } = rmsNorm(layerIn[t], nConv);
+            normedConv.push(y);
+            rmsConv.push(r);
+        }
+        const afterConv = [];
+        for (let t = 0; t < T; t++) {
+            const out = Float32Array.from(layerIn[t]); // residual base
+            for (let c = 0; c < dModel; c++) {
+                let acc = 0;
+                for (let j = 0; j < convKernel; j++) {
+                    const ti = t - j;
+                    if (ti >= 0)
+                        acc += ker[c * convKernel + j] * normedConv[ti][c];
                 }
-                afterConv.push(out);
-            }
-            // Pre-norm → MoE channel mixer → residual.
-            const rmsMoe = [];
-            const moeCache = [];
-            const afterMoe = [];
-            for (let t = 0; t < T; t++) {
-                const { y, r } = rmsNorm(afterConv[t], nMoe);
-                rmsMoe.push(r);
-                const out = Float32Array.from(afterConv[t]); // residual base
-                const mr = this.moe[l].forward(y);
-                for (let c = 0; c < dModel; c++)
-                    out[c] = out[c] + mr.output[c];
-                afterMoe.push(out);
-                moeCache.push(mr.cache);
+                out[c] = out[c] + acc;
             }
-            layers.push({ layerIn, normedConv, rmsConv, afterConv, rmsMoe, moeCache });
-            x = afterMoe;
+            afterConv.push(out);
         }
-        // Tied head: logits_t[v] = x_t · emb[v].
-        const logits = x.map((xt) => {
+        const rmsMoe = [];
+        const moeCache = [];
+        const afterMoe = [];
+        for (let t = 0; t < T; t++) {
+            const { y, r } = rmsNorm(afterConv[t], nMoe);
+            rmsMoe.push(r);
+            const out = Float32Array.from(afterConv[t]); // residual base
+            const mr = this.moe[l].forward(y);
+            for (let c = 0; c < dModel; c++)
+                out[c] = out[c] + mr.output[c];
+            afterMoe.push(out);
+            moeCache.push(mr.cache);
+        }
+        return { afterMoe, cache: { layerIn, normedConv, rmsConv, afterConv, rmsMoe, moeCache } };
+    }
+    /** Tied output head: logits_t[v] = x_t · emb[v]. */
+    _head(x) {
+        const { dModel, vocabSize } = this.config;
+        return x.map((xt) => {
             const lg = new Float32Array(vocabSize);
             for (let v = 0; v < vocabSize; v++) {
                 let acc = 0;
@@ -181,30 +187,36 @@ export class EvermindLM {
             }
             return lg;
         });
-        return { logits, cache: { tokens, layers, finalX: x } };
+    }
+    /** Run the model over a token sequence; returns per-position logits + a cache. */
+    forward(tokens) {
+        let x = this._embed(tokens);
+        const layers = [];
+        for (let l = 0; l < this.config.numLayers; l++) {
+            const { afterMoe, cache } = this._forwardLayer(l, x);
+            layers.push(cache);
+            x = afterMoe;
+        }
+        return { logits: this._head(x), cache: { tokens, layers, finalX: x } };
     }
     // ── Loss + backward ──────────────────────────────────────────────────────────
     /**
-     * Next-token cross-entropy over the sequence (predict tokens[t+1] from
-     * position t), accumulating exact gradients. Returns the mean loss. Call
-     * {@link zeroGrad} before and an optimiser step after.
+     * Head + tied-embedding gradient. Accumulates dL/d(head→emb) into gEmb and
+     * returns the mean next-token loss plus dL/d(finalX). Shared by the full and
+     * checkpointed backward paths so the head maths lives in one place.
      */
-    lossAndBackward(tokens) {
-        const { dModel, convKernel, numLayers, vocabSize } = this.config;
+    _headBackward(tokens, logits, finalX) {
+        const { dModel, vocabSize } = this.config;
         const T = tokens.length;
-        if (T < 2)
-            return 0;
-        const { logits, cache } = this.forward(tokens);
         const predPositions = T - 1; // positions 0..T-2 predict the next token
         const inv = 1 / predPositions;
-        // dL/d(finalX_t) and head gradient into the tied embedding.
         const dX = Array.from({ length: T }, () => new Float32Array(dModel));
         let loss = 0;
         for (let t = 0; t < predPositions; t++) {
             const target = tokens[t + 1];
             loss += crossEntropyLoss(logits[t], target) * inv;
             const dLogit = crossEntropyGrad(logits[t], target); // probs - onehot
-            const xt = cache.finalX[t];
+            const xt = finalX[t];
             for (let v = 0; v < vocabSize; v++) {
                 const g = dLogit[v] * inv;
                 if (g === 0)
@@ -216,60 +228,111 @@ export class EvermindLM {
                 }
             }
         }
-        // Backprop through layers in reverse.
-        for (let l = numLayers - 1; l >= 0; l--) {
-            const lc = cache.layers[l];
-            const ker = this.conv[l];
-            const gker = this.gConv[l];
-            const nConv = this.nConv[l];
-            const gNConv = this.gNConv[l];
-            const nMoe = this.nMoe[l];
-            const gNMoe = this.gNMoe[l];
-            // MoE sub-block: afterMoe = afterConv + MoE(RMSNorm(afterConv, nMoe)).
-            const dAfterConv = [];
-            for (let t = 0; t < T; t++) {
-                const dMoeNormed = this.moe[l].backward(dX[t], lc.moeCache[t]);
-                const { dx, dgain } = rmsNormBackward(dMoeNormed, lc.afterConv[t], lc.rmsMoe[t], nMoe);
-                for (let c = 0; c < dModel; c++)
-                    gNMoe[c] = gNMoe[c] + dgain[c];
-                const d = Float32Array.from(dX[t]); // residual passthrough
-                for (let c = 0; c < dModel; c++)
-                    d[c] = d[c] + dx[c];
-                dAfterConv.push(d);
-            }
-            // Conv sub-block: afterConv = layerIn + conv(RMSNorm(layerIn, nConv)).
-            const dNormedConv = Array.from({ length: T }, () => new Float32Array(dModel));
-            const dLayerIn = dAfterConv.map((v) => Float32Array.from(v)); // residual passthrough
-            for (let t = 0; t < T; t++) {
-                for (let c = 0; c < dModel; c++) {
-                    const dmix = dAfterConv[t][c];
-                    if (dmix === 0)
+        return { loss, dX };
+    }
+    /**
+     * Backward through one (conv + MoE) block given dL/d(block output) and the
+     * block's activation cache. Accumulates conv/norm/MoE gradients and returns
+     * dL/d(block input). The inverse of {@link _forwardLayer}.
+     */
+    _backwardLayer(l, dOut, lc) {
+        const { dModel, convKernel } = this.config;
+        const T = dOut.length;
+        const ker = this.conv[l];
+        const gker = this.gConv[l];
+        const nConv = this.nConv[l];
+        const gNConv = this.gNConv[l];
+        const nMoe = this.nMoe[l];
+        const gNMoe = this.gNMoe[l];
+        // MoE sub-block: afterMoe = afterConv + MoE(RMSNorm(afterConv, nMoe)).
+        const dAfterConv = [];
+        for (let t = 0; t < T; t++) {
+            const dMoeNormed = this.moe[l].backward(dOut[t], lc.moeCache[t]);
+            const { dx, dgain } = rmsNormBackward(dMoeNormed, lc.afterConv[t], lc.rmsMoe[t], nMoe);
+            for (let c = 0; c < dModel; c++)
+                gNMoe[c] = gNMoe[c] + dgain[c];
+            const d = Float32Array.from(dOut[t]); // residual passthrough
+            for (let c = 0; c < dModel; c++)
+                d[c] = d[c] + dx[c];
+            dAfterConv.push(d);
+        }
+        // Conv sub-block: afterConv = layerIn + conv(RMSNorm(layerIn, nConv)).
+        const dNormedConv = Array.from({ length: T }, () => new Float32Array(dModel));
+        const dLayerIn = dAfterConv.map((v) => Float32Array.from(v)); // residual passthrough
+        for (let t = 0; t < T; t++) {
+            for (let c = 0; c < dModel; c++) {
+                const dmix = dAfterConv[t][c];
+                if (dmix === 0)
+                    continue;
+                for (let j = 0; j < convKernel; j++) {
+                    const ti = t - j;
+                    if (ti < 0)
                         continue;
-                    for (let j = 0; j < convKernel; j++) {
-                        const ti = t - j;
-                        if (ti < 0)
-                            continue;
-                        gker[c * convKernel + j] = gker[c * convKernel + j] + dmix * lc.normedConv[ti][c];
-                        dNormedConv[ti][c] = dNormedConv[ti][c] + dmix * ker[c * convKernel + j];
-                    }
-                }
-            }
-            for (let t = 0; t < T; t++) {
-                const { dx, dgain } = rmsNormBackward(dNormedConv[t], lc.layerIn[t], lc.rmsConv[t], nConv);
-                for (let c = 0; c < dModel; c++) {
-                    gNConv[c] = gNConv[c] + dgain[c];
-                    dLayerIn[t][c] = dLayerIn[t][c] + dx[c];
+                    gker[c * convKernel + j] = gker[c * convKernel + j] + dmix * lc.normedConv[ti][c];
+                    dNormedConv[ti][c] = dNormedConv[ti][c] + dmix * ker[c * convKernel + j];
                 }
             }
-            for (let t = 0; t < T; t++)
-                dX[t] = dLayerIn[t];
         }
-        // Embedding lookup: dX at layer-0 input flows into the row for token_t.
         for (let t = 0; t < T; t++) {
+            const { dx, dgain } = rmsNormBackward(dNormedConv[t], lc.layerIn[t], lc.rmsConv[t], nConv);
+            for (let c = 0; c < dModel; c++) {
+                gNConv[c] = gNConv[c] + dgain[c];
+                dLayerIn[t][c] = dLayerIn[t][c] + dx[c];
+            }
+        }
+        return dLayerIn;
+    }
+    /** Embedding lookup: dL/d(layer-0 input) flows into the row for each token. */
+    _embedBackward(tokens, dIn) {
+        const { dModel } = this.config;
+        for (let t = 0; t < tokens.length; t++) {
             const off = tokens[t] * dModel;
             for (let c = 0; c < dModel; c++)
-                this.gEmb[off + c] = this.gEmb[off + c] + dX[t][c];
+                this.gEmb[off + c] = this.gEmb[off + c] + dIn[t][c];
         }
+    }
+    /**
+     * Next-token cross-entropy over the sequence (predict tokens[t+1] from
+     * position t), accumulating exact gradients. Returns the mean loss. Call
+     * {@link zeroGrad} before and an optimiser step after.
+     */
+    lossAndBackward(tokens) {
+        if (tokens.length < 2)
+            return 0;
+        const { logits, cache } = this.forward(tokens);
+        const { loss, dX } = this._headBackward(tokens, logits, cache.finalX);
+        let d = dX;
+        for (let l = this.config.numLayers - 1; l >= 0; l--)
+            d = this._backwardLayer(l, d, cache.layers[l]);
+        this._embedBackward(tokens, d);
+        return loss;
+    }
+    /**
+     * Activation-checkpointed backward — numerically identical gradients to
+     * {@link lossAndBackward}, but retains only the per-LAYER inputs during the
+     * forward instead of every layer's full activation cache. Each layer's
+     * activations are RECOMPUTED (a cheap extra forward) when its backward runs,
+     * so peak activation memory is one layer's cache, not all of them. This is the
+     * memory-for-compute trade the cookbook pairs with FSDP to fit longer
+     * sequences / bigger models on a constrained device.
+     */
+    lossAndBackwardCheckpointed(tokens) {
+        if (tokens.length < 2)
+            return 0;
+        let x = this._embed(tokens);
+        const layerInputs = [];
+        for (let l = 0; l < this.config.numLayers; l++) {
+            layerInputs.push(x); // keep only the input; drop the cache
+            x = this._forwardLayer(l, x).afterMoe;
+        }
+        const logits = this._head(x);
+        const { loss, dX } = this._headBackward(tokens, logits, x);
+        let d = dX;
+        for (let l = this.config.numLayers - 1; l >= 0; l--) {
+            const { cache } = this._forwardLayer(l, layerInputs[l]); // recompute this layer's activations
+            d = this._backwardLayer(l, d, cache);
+        }
+        this._embedBackward(tokens, d);
         return loss;
     }
     // ── Generation ───────────────────────────────────────────────────────────────
@@ -437,26 +500,95 @@ export class EvermindLMTrainer {
     model;
     opts;
     adam;
+    scaler;
+    grads;
     constructor(model, opts = {}) {
         this.model = model;
         this.opts = opts;
         this.adam = new AdamW(model, opts);
+        this.grads = model.gradients();
+        this.scaler = opts.mixedPrecision
+            ? new DynamicLossScaler(typeof opts.mixedPrecision === "object" ? opts.mixedPrecision : {})
+            : null;
+    }
+    /** The dynamic loss scaler (mixed-precision mode only) — exposes scale/overflow stats. */
+    get lossScaler() {
+        return this.scaler;
+    }
+    _backward(seq) {
+        return this.opts.checkpoint ? this.model.lossAndBackwardCheckpointed(seq) : this.model.lossAndBackward(seq);
+    }
+    /** Divide accumulated gradients by `n` (accumulation averaging), in place. */
+    _scaleGrads(n) {
+        if (n === 1)
+            return;
+        const inv = 1 / n;
+        for (const g of this.grads)
+            for (let i = 0; i < g.data.length; i++)
+                g.data[i] = g.data[i] * inv;
+    }
+    /** Round accumulated gradients to fp16 precision (mixed-precision simulation), in place. */
+    _fp16Grads() {
+        for (const g of this.grads)
+            for (let i = 0; i < g.data.length; i++)
+                g.data[i] = roundFp16(g.data[i]);
+    }
+    /** Multiply accumulated gradients by `s`, in place. */
+    _mulGrads(s) {
+        if (s === 1)
+            return;
+        for (const g of this.grads)
+            for (let i = 0; i < g.data.length; i++)
+                g.data[i] = g.data[i] * s;
     }
     /** Train on a set of token sequences; returns per-epoch mean loss. */
     fit(sequences) {
         const epochs = this.opts.epochs ?? 1;
+        const accum = Math.max(1, this.opts.accumSteps ?? 1);
         const history = [];
         for (let e = 0; e < epochs; e++) {
             let total = 0;
             let n = 0;
+            let pending = 0;
+            // The scale is fixed for the duration of one accumulation window, so scale-up
+            // (per sequence) and unscale (at flush) always use the SAME factor even as the
+            // controller grows/backs it off between windows.
+            let windowScale = this.scaler ? this.scaler.scale : 1;
+            this.model.zeroGrad();
+            const flush = () => {
+                if (pending === 0)
+                    return;
+                this._scaleGrads(pending); // accumulation averaging
+                if (this.scaler) {
+                    this._fp16Grads(); // half-precision gradients
+                    const overflow = this.scaler.check(this.grads);
+                    if (this.scaler.update(overflow)) {
+                        this._mulGrads(1 / windowScale); // unscale by the SAME factor we scaled by
+                        this.adam.step();
+                    } // else: skip the step, scale has backed off
+                    windowScale = this.scaler.scale; // next window uses the updated scale
+                }
+                else {
+                    this.adam.step();
+                }
+                this.model.zeroGrad();
+                pending = 0;
+            };
             for (const seq of sequences) {
                 if (seq.length < 2)
                     continue;
-                this.model.zeroGrad();
-                total += this.model.lossAndBackward(seq);
-                this.adam.step();
+                if (pending === 0 && this.scaler)
+                    windowScale = this.scaler.scale;
+                const loss = this._backward(seq);
+                // Emulate scaling the loss before backward: the gradient is linear in the
+                // loss, so scaling the just-produced gradients by the loss scale is equivalent.
+                this._mulGrads(windowScale);
+                total += loss;
                 n++;
+                if (++pending >= accum)
+                    flush();
             }
+            flush();
             history.push(n > 0 ? total / n : 0);
         }
         return history;