@infernetprotocol/training 0.1.79 → 0.1.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/runners/peft_generate.py +83 -0
- package/src/backends/peft.js +45 -0
- package/src/index.js +2 -0
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@infernetprotocol/training",
|
|
3
3
|
"private": false,
|
|
4
|
-
"version": "0.1.
|
|
4
|
+
"version": "0.1.80",
|
|
5
5
|
"description": "Training orchestration for Infernet — pluggable backends (DeepSpeed/FSDP for trusted clusters, OpenDiLoCo for cross-provider async, Petals for trustless P2P fine-tunes) over a stable async-iterator API.",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"type": "module",
|
|
@@ -19,8 +19,8 @@
|
|
|
19
19
|
"node": ">=18"
|
|
20
20
|
},
|
|
21
21
|
"dependencies": {
|
|
22
|
-
"@infernetprotocol/engine": "0.1.
|
|
23
|
-
"@infernetprotocol/liquid-agent": "0.1.
|
|
22
|
+
"@infernetprotocol/engine": "0.1.80",
|
|
23
|
+
"@infernetprotocol/liquid-agent": "0.1.80"
|
|
24
24
|
},
|
|
25
25
|
"repository": {
|
|
26
26
|
"type": "git",
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Answer chat messages with a base model plus a LoRA adapter (workload infernet.text.infer).
|
|
3
|
+
|
|
4
|
+
python peft_generate.py request.json
|
|
5
|
+
|
|
6
|
+
request.json:
|
|
7
|
+
{
|
|
8
|
+
"adapter_dir": "adapter", local dir: adapter_config.json, adapter_model.safetensors, manifest.json
|
|
9
|
+
"base_model": "HuggingFaceTB/SmolLM2-135M-Instruct", optional; else the adapter manifest's
|
|
10
|
+
"messages": [{"role": "user", "content": "..."}], a system message is added from the manifest if absent
|
|
11
|
+
"max_new_tokens": 350,
|
|
12
|
+
"threads": 4
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
Relative paths resolve against the request file's directory. Local files only.
|
|
16
|
+
Prints one JSON object on stdout:
|
|
17
|
+
{"v": 1, "type": "answer", "text": "...", "model": "<base>", "adapter": "sha256:<hex>", "tokens": N, "seconds": S}
|
|
18
|
+
or {"v": 1, "type": "error", "message": "..."} and exits 1. Logs go to stderr.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
import hashlib
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import sys
|
|
25
|
+
import time
|
|
26
|
+
import traceback
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def resolve(base_dir, p):
|
|
31
|
+
p = str(p).removeprefix("file://")
|
|
32
|
+
if "://" in p:
|
|
33
|
+
raise ValueError("adapter_dir must be a local path")
|
|
34
|
+
return Path(p) if os.path.isabs(p) else base_dir / p
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def main():
|
|
38
|
+
import torch
|
|
39
|
+
from peft import PeftModel
|
|
40
|
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
41
|
+
|
|
42
|
+
req_path = Path(sys.argv[1] if len(sys.argv) > 1 else "request.json").resolve()
|
|
43
|
+
req = json.loads(req_path.read_text())
|
|
44
|
+
adapter = resolve(req_path.parent, req["adapter_dir"])
|
|
45
|
+
manifest = {}
|
|
46
|
+
if (adapter / "manifest.json").exists():
|
|
47
|
+
manifest = json.loads((adapter / "manifest.json").read_text())
|
|
48
|
+
base = req.get("base_model") or manifest.get("base_model")
|
|
49
|
+
if not base:
|
|
50
|
+
raise ValueError("base_model is missing (not in the request or the adapter manifest)")
|
|
51
|
+
weights = adapter / "adapter_model.safetensors"
|
|
52
|
+
digest = "sha256:" + hashlib.sha256(weights.read_bytes()).hexdigest()
|
|
53
|
+
|
|
54
|
+
messages = [m for m in req.get("messages") or [] if isinstance(m, dict) and isinstance(m.get("content"), str)]
|
|
55
|
+
if not messages:
|
|
56
|
+
raise ValueError("messages[] is required")
|
|
57
|
+
if messages[0].get("role") != "system" and manifest.get("system_prompt"):
|
|
58
|
+
messages = [{"role": "system", "content": manifest["system_prompt"]}] + messages
|
|
59
|
+
|
|
60
|
+
torch.set_num_threads(int(req.get("threads") or min(4, os.cpu_count() or 1)))
|
|
61
|
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
62
|
+
t0 = time.time()
|
|
63
|
+
tok = AutoTokenizer.from_pretrained(adapter if (adapter / "tokenizer_config.json").exists() else base)
|
|
64
|
+
model = AutoModelForCausalLM.from_pretrained(base, dtype=torch.float32).to(device)
|
|
65
|
+
model = PeftModel.from_pretrained(model, adapter).eval()
|
|
66
|
+
prompt = tok.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
|
67
|
+
ids = tok(prompt, return_tensors="pt").to(device)
|
|
68
|
+
with torch.no_grad():
|
|
69
|
+
# Greedy: sampling on a small model echoes the question and rambles.
|
|
70
|
+
out = model.generate(**ids, max_new_tokens=int(req.get("max_new_tokens") or 350), do_sample=False, repetition_penalty=1.1)
|
|
71
|
+
new = out[0][ids["input_ids"].shape[1]:]
|
|
72
|
+
text = tok.decode(new, skip_special_tokens=True).strip()
|
|
73
|
+
print(json.dumps({"v": 1, "type": "answer", "text": text, "model": base, "adapter": digest,
|
|
74
|
+
"tokens": int(new.shape[0]), "seconds": round(time.time() - t0, 2)}), flush=True)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
if __name__ == "__main__":
|
|
78
|
+
try:
|
|
79
|
+
main()
|
|
80
|
+
except Exception as e:
|
|
81
|
+
traceback.print_exc(file=sys.stderr)
|
|
82
|
+
print(json.dumps({"v": 1, "type": "error", "message": str(e)}), flush=True)
|
|
83
|
+
sys.exit(1)
|
package/src/backends/peft.js
CHANGED
|
@@ -87,6 +87,51 @@ export async function peftRuntimeStatus() {
|
|
|
87
87
|
});
|
|
88
88
|
}
|
|
89
89
|
|
|
90
|
+
export const GENERATE_RUNNER_PATH = fileURLToPath(new URL("../../runners/peft_generate.py", import.meta.url));
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* One answer from a base model + LoRA adapter (workload infernet.text.infer).
|
|
94
|
+
* `request` is the runner's request object; relative paths resolve against
|
|
95
|
+
* `requestPath`'s directory when given. Resolves to the runner's answer
|
|
96
|
+
* object; rejects with its error message.
|
|
97
|
+
*/
|
|
98
|
+
export async function peftGenerate({ request, requestPath = null, python = null, runner = GENERATE_RUNNER_PATH, timeoutMs = 600000, spawnImpl = spawn } = {}) {
|
|
99
|
+
const py = python ?? resolvePeftPython();
|
|
100
|
+
if (!py) throw Object.assign(new Error("the peft runtime is not installed; run `infernet model runtime setup --peft --yes`"), { code: "E_NO_RUNTIME" });
|
|
101
|
+
let file = requestPath;
|
|
102
|
+
if (!file) {
|
|
103
|
+
const dir = await mkdtemp(path.join(os.tmpdir(), "infernet-generate-"));
|
|
104
|
+
file = path.join(dir, "request.json");
|
|
105
|
+
await writeFile(file, JSON.stringify(request));
|
|
106
|
+
}
|
|
107
|
+
return new Promise((resolve, reject) => {
|
|
108
|
+
const child = spawnImpl(py, [runner, file], { stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, PYTHONUNBUFFERED: "1" } });
|
|
109
|
+
let out = "";
|
|
110
|
+
const tail = [];
|
|
111
|
+
child.stdout.on("data", (b) => (out += b));
|
|
112
|
+
child.stderr.on("data", (b) => {
|
|
113
|
+
for (const l of b.toString().split("\n").filter(Boolean)) {
|
|
114
|
+
tail.push(l);
|
|
115
|
+
if (tail.length > 20) tail.shift();
|
|
116
|
+
}
|
|
117
|
+
});
|
|
118
|
+
const timer = setTimeout(() => child.kill("SIGTERM"), timeoutMs);
|
|
119
|
+
child.on("error", reject);
|
|
120
|
+
child.on("close", (code) => {
|
|
121
|
+
clearTimeout(timer);
|
|
122
|
+
const line = out.split("\n").map((l) => l.trim()).filter((l) => l.startsWith("{")).at(-1);
|
|
123
|
+
let res = null;
|
|
124
|
+
try {
|
|
125
|
+
res = line ? JSON.parse(line) : null;
|
|
126
|
+
} catch {
|
|
127
|
+
res = null;
|
|
128
|
+
}
|
|
129
|
+
if (code === 0 && res?.type === "answer") return resolve(res);
|
|
130
|
+
reject(Object.assign(new Error(res?.message ?? `generate exited ${code}: ${tail.at(-1) ?? "no output"}`), { code: "E_RUNTIME" }));
|
|
131
|
+
});
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
|
|
90
135
|
export function createPeftBackend({ python = null, runner = RUNNER_PATH, spawnImpl = spawn } = {}) {
|
|
91
136
|
const children = new Set();
|
|
92
137
|
return {
|
package/src/index.js
CHANGED