@infernetprotocol/training 0.1.90 → 0.1.92
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/runners/peft_train.py +55 -4
- package/runners/test_peft_train.py +23 -0
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@infernetprotocol/training",
|
|
3
3
|
"private": false,
|
|
4
|
-
"version": "0.1.
|
|
4
|
+
"version": "0.1.92",
|
|
5
5
|
"description": "Training orchestration for Infernet — pluggable backends (DeepSpeed/FSDP for trusted clusters, OpenDiLoCo for cross-provider async, Petals for trustless P2P fine-tunes) over a stable async-iterator API.",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"type": "module",
|
|
@@ -19,8 +19,8 @@
|
|
|
19
19
|
"node": ">=18"
|
|
20
20
|
},
|
|
21
21
|
"dependencies": {
|
|
22
|
-
"@infernetprotocol/engine": "0.1.
|
|
23
|
-
"@infernetprotocol/liquid-agent": "0.1.
|
|
22
|
+
"@infernetprotocol/engine": "0.1.92",
|
|
23
|
+
"@infernetprotocol/liquid-agent": "0.1.92"
|
|
24
24
|
},
|
|
25
25
|
"repository": {
|
|
26
26
|
"type": "git",
|
package/runners/peft_train.py
CHANGED
|
@@ -13,7 +13,8 @@ infernet.text.train). `job.json` is that package's job config:
|
|
|
13
13
|
"base_model": "HuggingFaceTB/SmolLM2-135M-Instruct",
|
|
14
14
|
"system_prompt": "You are ...", optional
|
|
15
15
|
"dataset": { "url": "corpus.jsonl", "sha256": "<hex>"? }, jsonl, see below
|
|
16
|
-
"hyperparams": { "lr", "batch_size", "epochs", "max_steps", "seq_len", "lora_r", "threads", "seed"
|
|
16
|
+
"hyperparams": { "lr", "batch_size", "epochs", "max_steps", "seq_len", "lora_r", "threads", "seed",
|
|
17
|
+
"dtype" }, float32 | bfloat16 | float16 | auto (default)
|
|
17
18
|
"output": { "destination_url": "out/adapter" }
|
|
18
19
|
}
|
|
19
20
|
|
|
@@ -27,6 +28,12 @@ Dataset records (one JSON object per line, mixing allowed):
|
|
|
27
28
|
system prompt is added when the
|
|
28
29
|
conversation has none
|
|
29
30
|
|
|
31
|
+
`dtype` is what the base model's weights load in; the LoRA weights always
|
|
32
|
+
train in float32. `auto` picks bfloat16 where the hardware does it natively
|
|
33
|
+
(CUDA with bf16, CPUs with avx512_bf16 / amx_bf16 / Arm bf16) and float32
|
|
34
|
+
elsewhere: bf16 is several times faster and needs half the RAM on those CPUs,
|
|
35
|
+
but emulated bf16 on an older CPU is slower than float32.
|
|
36
|
+
|
|
30
37
|
Relative paths resolve against the job file's directory. Progress goes to
|
|
31
38
|
stdout as training-protocol v1 NDJSON (meta / step / eval / checkpoint /
|
|
32
39
|
done / error); human-readable logs go to stderr. The output dir holds a PEFT
|
|
@@ -75,6 +82,37 @@ ROLES = ("system", "user", "assistant")
|
|
|
75
82
|
IGNORE = -100
|
|
76
83
|
|
|
77
84
|
|
|
85
|
+
DTYPES = ("float32", "bfloat16", "float16")
|
|
86
|
+
DTYPE_ALIASES = {"fp32": "float32", "bf16": "bfloat16", "fp16": "float16", "half": "float16"}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def cpu_has_bf16(cpuinfo=None):
|
|
90
|
+
"""Whether this CPU does bfloat16 natively (x86 avx512_bf16 / amx_bf16, Arm bf16)."""
|
|
91
|
+
if cpuinfo is None:
|
|
92
|
+
try:
|
|
93
|
+
cpuinfo = Path("/proc/cpuinfo").read_text()
|
|
94
|
+
except OSError:
|
|
95
|
+
return False
|
|
96
|
+
flags = set()
|
|
97
|
+
for line in cpuinfo.splitlines():
|
|
98
|
+
key, _, val = line.partition(":")
|
|
99
|
+
if key.strip().lower() in ("flags", "features"):
|
|
100
|
+
flags.update(val.split())
|
|
101
|
+
return bool(flags & {"avx512_bf16", "amx_bf16", "bf16"})
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def resolve_dtype(requested, device, cuda_bf16=False, cpu_bf16=False):
|
|
105
|
+
"""The dtype name to load the base model in: `hyperparams.dtype` if set,
|
|
106
|
+
else (auto) bfloat16 where it is native and float32 elsewhere."""
|
|
107
|
+
want = DTYPE_ALIASES.get(str(requested or "auto").lower(), str(requested or "auto").lower())
|
|
108
|
+
if want == "auto":
|
|
109
|
+
native = cuda_bf16 if device == "cuda" else cpu_bf16
|
|
110
|
+
return "bfloat16" if native else "float32"
|
|
111
|
+
if want not in DTYPES:
|
|
112
|
+
raise ValueError(f"hyperparams.dtype must be one of {', '.join(DTYPES)} or auto, not {requested!r}")
|
|
113
|
+
return want
|
|
114
|
+
|
|
115
|
+
|
|
78
116
|
def parse_record(d):
|
|
79
117
|
"""One dataset line as a doc, or None to skip it. Raises on a malformed
|
|
80
118
|
conversation so a bad dataset fails loudly instead of training on less."""
|
|
@@ -204,6 +242,14 @@ def main():
|
|
|
204
242
|
system = job.get("system_prompt") or DEFAULT_SYSTEM
|
|
205
243
|
out_dir = resolve(base_dir, job["output"]["destination_url"])
|
|
206
244
|
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
245
|
+
dtype = resolve_dtype(
|
|
246
|
+
hp.get("dtype"), device,
|
|
247
|
+
cuda_bf16=device == "cuda" and torch.cuda.is_bf16_supported(),
|
|
248
|
+
cpu_bf16=cpu_has_bf16(),
|
|
249
|
+
)
|
|
250
|
+
hp["dtype"] = dtype
|
|
251
|
+
if dtype == "float16" and device == "cpu":
|
|
252
|
+
log("warning: float16 on CPU is slow and can overflow; bfloat16 or float32 is the better choice")
|
|
207
253
|
|
|
208
254
|
random.seed(hp["seed"])
|
|
209
255
|
torch.manual_seed(hp["seed"])
|
|
@@ -240,11 +286,11 @@ def main():
|
|
|
240
286
|
total = int(hp["max_steps"] or steps_per_epoch * int(hp["epochs"]))
|
|
241
287
|
|
|
242
288
|
emit(job_id, "meta", started_at=now(), backend="peft", world_size=1, kind=job.get("kind", "lora"),
|
|
243
|
-
base_model=job["base_model"], device=device, total_steps=total, documents=len(docs),
|
|
289
|
+
base_model=job["base_model"], device=device, dtype=dtype, total_steps=total, documents=len(docs),
|
|
244
290
|
dataset_sha256=dataset_sha)
|
|
245
|
-
log(f"{len(docs)} docs -> {len(train_blocks[0])} train / {len(eval_blocks[0])} eval blocks of {hp['seq_len']} tokens; {total} steps on {device}")
|
|
291
|
+
log(f"{len(docs)} docs -> {len(train_blocks[0])} train / {len(eval_blocks[0])} eval blocks of {hp['seq_len']} tokens; {total} steps on {device} in {dtype}")
|
|
246
292
|
|
|
247
|
-
model = AutoModelForCausalLM.from_pretrained(job["base_model"], dtype=torch
|
|
293
|
+
model = AutoModelForCausalLM.from_pretrained(job["base_model"], dtype=getattr(torch, dtype)).to(device)
|
|
248
294
|
# Recompute activations in the backward pass: roughly halves peak RAM, so
|
|
249
295
|
# the job fits beside other work on a shared box.
|
|
250
296
|
model.gradient_checkpointing_enable()
|
|
@@ -254,6 +300,11 @@ def main():
|
|
|
254
300
|
target_modules=hp.get("target_modules") or ["q_proj", "k_proj", "v_proj", "o_proj"],
|
|
255
301
|
task_type="CAUSAL_LM",
|
|
256
302
|
))
|
|
303
|
+
# The adapter trains in float32 whatever the base is in: bf16 optimizer
|
|
304
|
+
# updates on small LoRA weights lose most of their precision.
|
|
305
|
+
for p in model.parameters():
|
|
306
|
+
if p.requires_grad and p.dtype != torch.float32:
|
|
307
|
+
p.data = p.data.float()
|
|
257
308
|
log(f"trainable params: {sum(p.numel() for p in model.parameters() if p.requires_grad):,}")
|
|
258
309
|
|
|
259
310
|
opt = torch.optim.AdamW([p for p in model.parameters() if p.requires_grad], lr=float(hp["lr"]))
|
|
@@ -103,5 +103,28 @@ class DatasetTests(unittest.TestCase):
|
|
|
103
103
|
self.assertEqual(pt.DATASET_FORMATS, ("text", "messages"))
|
|
104
104
|
|
|
105
105
|
|
|
106
|
+
|
|
107
|
+
class DtypeTests(unittest.TestCase):
|
|
108
|
+
def test_explicit_dtype_is_honoured(self):
|
|
109
|
+
for want in ("float32", "bfloat16", "float16"):
|
|
110
|
+
self.assertEqual(pt.resolve_dtype(want, "cpu"), want)
|
|
111
|
+
self.assertEqual(pt.resolve_dtype("bf16", "cpu"), "bfloat16")
|
|
112
|
+
self.assertEqual(pt.resolve_dtype("FP32", "cuda", cuda_bf16=True), "float32")
|
|
113
|
+
|
|
114
|
+
def test_auto_picks_bf16_only_where_native(self):
|
|
115
|
+
self.assertEqual(pt.resolve_dtype(None, "cpu", cpu_bf16=True), "bfloat16")
|
|
116
|
+
self.assertEqual(pt.resolve_dtype("auto", "cpu", cpu_bf16=False), "float32")
|
|
117
|
+
self.assertEqual(pt.resolve_dtype(None, "cuda", cuda_bf16=True), "bfloat16")
|
|
118
|
+
self.assertEqual(pt.resolve_dtype(None, "cuda", cuda_bf16=False, cpu_bf16=True), "float32")
|
|
119
|
+
|
|
120
|
+
def test_unknown_dtype_fails_loudly(self):
|
|
121
|
+
with self.assertRaises(ValueError):
|
|
122
|
+
pt.resolve_dtype("int8", "cpu")
|
|
123
|
+
|
|
124
|
+
def test_cpu_bf16_detection(self):
|
|
125
|
+
self.assertTrue(pt.cpu_has_bf16("flags\t\t: fpu sse avx2 avx512f avx512_bf16\n"))
|
|
126
|
+
self.assertTrue(pt.cpu_has_bf16("Features\t: fp asimd bf16 i8mm\n"))
|
|
127
|
+
self.assertFalse(pt.cpu_has_bf16("flags\t\t: fpu sse avx2 avx512f\n"))
|
|
128
|
+
|
|
106
129
|
if __name__ == "__main__":
|
|
107
130
|
unittest.main()
|