@infernetprotocol/training 0.1.90 → 0.1.91

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@infernetprotocol/training",
3
3
  "private": false,
4
- "version": "0.1.90",
4
+ "version": "0.1.91",
5
5
  "description": "Training orchestration for Infernet — pluggable backends (DeepSpeed/FSDP for trusted clusters, OpenDiLoCo for cross-provider async, Petals for trustless P2P fine-tunes) over a stable async-iterator API.",
6
6
  "license": "MIT",
7
7
  "type": "module",
@@ -19,8 +19,8 @@
19
19
  "node": ">=18"
20
20
  },
21
21
  "dependencies": {
22
- "@infernetprotocol/engine": "0.1.90",
23
- "@infernetprotocol/liquid-agent": "0.1.90"
22
+ "@infernetprotocol/engine": "0.1.91",
23
+ "@infernetprotocol/liquid-agent": "0.1.91"
24
24
  },
25
25
  "repository": {
26
26
  "type": "git",
@@ -13,7 +13,8 @@ infernet.text.train). `job.json` is that package's job config:
13
13
  "base_model": "HuggingFaceTB/SmolLM2-135M-Instruct",
14
14
  "system_prompt": "You are ...", optional
15
15
  "dataset": { "url": "corpus.jsonl", "sha256": "<hex>"? }, jsonl, see below
16
- "hyperparams": { "lr", "batch_size", "epochs", "max_steps", "seq_len", "lora_r", "threads", "seed" },
16
+ "hyperparams": { "lr", "batch_size", "epochs", "max_steps", "seq_len", "lora_r", "threads", "seed",
17
+ "dtype" }, float32 | bfloat16 | float16 | auto (default)
17
18
  "output": { "destination_url": "out/adapter" }
18
19
  }
19
20
 
@@ -27,6 +28,12 @@ Dataset records (one JSON object per line, mixing allowed):
27
28
  system prompt is added when the
28
29
  conversation has none
29
30
 
31
+ `dtype` is what the base model's weights load in; the LoRA weights always
32
+ train in float32. `auto` picks bfloat16 where the hardware does it natively
33
+ (CUDA with bf16, CPUs with avx512_bf16 / amx_bf16 / Arm bf16) and float32
34
+ elsewhere: bf16 is several times faster and needs half the RAM on those CPUs,
35
+ but emulated bf16 on an older CPU is slower than float32.
36
+
30
37
  Relative paths resolve against the job file's directory. Progress goes to
31
38
  stdout as training-protocol v1 NDJSON (meta / step / eval / checkpoint /
32
39
  done / error); human-readable logs go to stderr. The output dir holds a PEFT
@@ -75,6 +82,37 @@ ROLES = ("system", "user", "assistant")
75
82
  IGNORE = -100
76
83
 
77
84
 
85
+ DTYPES = ("float32", "bfloat16", "float16")
86
+ DTYPE_ALIASES = {"fp32": "float32", "bf16": "bfloat16", "fp16": "float16", "half": "float16"}
87
+
88
+
89
+ def cpu_has_bf16(cpuinfo=None):
90
+ """Whether this CPU does bfloat16 natively (x86 avx512_bf16 / amx_bf16, Arm bf16)."""
91
+ if cpuinfo is None:
92
+ try:
93
+ cpuinfo = Path("/proc/cpuinfo").read_text()
94
+ except OSError:
95
+ return False
96
+ flags = set()
97
+ for line in cpuinfo.splitlines():
98
+ key, _, val = line.partition(":")
99
+ if key.strip().lower() in ("flags", "features"):
100
+ flags.update(val.split())
101
+ return bool(flags & {"avx512_bf16", "amx_bf16", "bf16"})
102
+
103
+
104
+ def resolve_dtype(requested, device, cuda_bf16=False, cpu_bf16=False):
105
+ """The dtype name to load the base model in: `hyperparams.dtype` if set,
106
+ else (auto) bfloat16 where it is native and float32 elsewhere."""
107
+ want = DTYPE_ALIASES.get(str(requested or "auto").lower(), str(requested or "auto").lower())
108
+ if want == "auto":
109
+ native = cuda_bf16 if device == "cuda" else cpu_bf16
110
+ return "bfloat16" if native else "float32"
111
+ if want not in DTYPES:
112
+ raise ValueError(f"hyperparams.dtype must be one of {', '.join(DTYPES)} or auto, not {requested!r}")
113
+ return want
114
+
115
+
78
116
  def parse_record(d):
79
117
  """One dataset line as a doc, or None to skip it. Raises on a malformed
80
118
  conversation so a bad dataset fails loudly instead of training on less."""
@@ -204,6 +242,14 @@ def main():
204
242
  system = job.get("system_prompt") or DEFAULT_SYSTEM
205
243
  out_dir = resolve(base_dir, job["output"]["destination_url"])
206
244
  device = "cuda" if torch.cuda.is_available() else "cpu"
245
+ dtype = resolve_dtype(
246
+ hp.get("dtype"), device,
247
+ cuda_bf16=device == "cuda" and torch.cuda.is_bf16_supported(),
248
+ cpu_bf16=cpu_has_bf16(),
249
+ )
250
+ hp["dtype"] = dtype
251
+ if dtype == "float16" and device == "cpu":
252
+ log("warning: float16 on CPU is slow and can overflow; bfloat16 or float32 is the better choice")
207
253
 
208
254
  random.seed(hp["seed"])
209
255
  torch.manual_seed(hp["seed"])
@@ -240,11 +286,11 @@ def main():
240
286
  total = int(hp["max_steps"] or steps_per_epoch * int(hp["epochs"]))
241
287
 
242
288
  emit(job_id, "meta", started_at=now(), backend="peft", world_size=1, kind=job.get("kind", "lora"),
243
- base_model=job["base_model"], device=device, total_steps=total, documents=len(docs),
289
+ base_model=job["base_model"], device=device, dtype=dtype, total_steps=total, documents=len(docs),
244
290
  dataset_sha256=dataset_sha)
245
- log(f"{len(docs)} docs -> {len(train_blocks[0])} train / {len(eval_blocks[0])} eval blocks of {hp['seq_len']} tokens; {total} steps on {device}")
291
+ log(f"{len(docs)} docs -> {len(train_blocks[0])} train / {len(eval_blocks[0])} eval blocks of {hp['seq_len']} tokens; {total} steps on {device} in {dtype}")
246
292
 
247
- model = AutoModelForCausalLM.from_pretrained(job["base_model"], dtype=torch.float32).to(device)
293
+ model = AutoModelForCausalLM.from_pretrained(job["base_model"], dtype=getattr(torch, dtype)).to(device)
248
294
  # Recompute activations in the backward pass: roughly halves peak RAM, so
249
295
  # the job fits beside other work on a shared box.
250
296
  model.gradient_checkpointing_enable()
@@ -254,6 +300,11 @@ def main():
254
300
  target_modules=hp.get("target_modules") or ["q_proj", "k_proj", "v_proj", "o_proj"],
255
301
  task_type="CAUSAL_LM",
256
302
  ))
303
+ # The adapter trains in float32 whatever the base is in: bf16 optimizer
304
+ # updates on small LoRA weights lose most of their precision.
305
+ for p in model.parameters():
306
+ if p.requires_grad and p.dtype != torch.float32:
307
+ p.data = p.data.float()
257
308
  log(f"trainable params: {sum(p.numel() for p in model.parameters() if p.requires_grad):,}")
258
309
 
259
310
  opt = torch.optim.AdamW([p for p in model.parameters() if p.requires_grad], lr=float(hp["lr"]))
@@ -103,5 +103,28 @@ class DatasetTests(unittest.TestCase):
103
103
  self.assertEqual(pt.DATASET_FORMATS, ("text", "messages"))
104
104
 
105
105
 
106
+
107
+ class DtypeTests(unittest.TestCase):
108
+ def test_explicit_dtype_is_honoured(self):
109
+ for want in ("float32", "bfloat16", "float16"):
110
+ self.assertEqual(pt.resolve_dtype(want, "cpu"), want)
111
+ self.assertEqual(pt.resolve_dtype("bf16", "cpu"), "bfloat16")
112
+ self.assertEqual(pt.resolve_dtype("FP32", "cuda", cuda_bf16=True), "float32")
113
+
114
+ def test_auto_picks_bf16_only_where_native(self):
115
+ self.assertEqual(pt.resolve_dtype(None, "cpu", cpu_bf16=True), "bfloat16")
116
+ self.assertEqual(pt.resolve_dtype("auto", "cpu", cpu_bf16=False), "float32")
117
+ self.assertEqual(pt.resolve_dtype(None, "cuda", cuda_bf16=True), "bfloat16")
118
+ self.assertEqual(pt.resolve_dtype(None, "cuda", cuda_bf16=False, cpu_bf16=True), "float32")
119
+
120
+ def test_unknown_dtype_fails_loudly(self):
121
+ with self.assertRaises(ValueError):
122
+ pt.resolve_dtype("int8", "cpu")
123
+
124
+ def test_cpu_bf16_detection(self):
125
+ self.assertTrue(pt.cpu_has_bf16("flags\t\t: fpu sse avx2 avx512f avx512_bf16\n"))
126
+ self.assertTrue(pt.cpu_has_bf16("Features\t: fp asimd bf16 i8mm\n"))
127
+ self.assertFalse(pt.cpu_has_bf16("flags\t\t: fpu sse avx2 avx512f\n"))
128
+
106
129
  if __name__ == "__main__":
107
130
  unittest.main()