@infernetprotocol/training 0.1.79 → 0.1.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@infernetprotocol/training",
3
3
  "private": false,
4
- "version": "0.1.79",
4
+ "version": "0.1.80",
5
5
  "description": "Training orchestration for Infernet — pluggable backends (DeepSpeed/FSDP for trusted clusters, OpenDiLoCo for cross-provider async, Petals for trustless P2P fine-tunes) over a stable async-iterator API.",
6
6
  "license": "MIT",
7
7
  "type": "module",
@@ -19,8 +19,8 @@
19
19
  "node": ">=18"
20
20
  },
21
21
  "dependencies": {
22
- "@infernetprotocol/engine": "0.1.79",
23
- "@infernetprotocol/liquid-agent": "0.1.79"
22
+ "@infernetprotocol/engine": "0.1.80",
23
+ "@infernetprotocol/liquid-agent": "0.1.80"
24
24
  },
25
25
  "repository": {
26
26
  "type": "git",
@@ -0,0 +1,83 @@
1
+ #!/usr/bin/env python3
2
+ """Answer chat messages with a base model plus a LoRA adapter (workload infernet.text.infer).
3
+
4
+ python peft_generate.py request.json
5
+
6
+ request.json:
7
+ {
8
+ "adapter_dir": "adapter", local dir: adapter_config.json, adapter_model.safetensors, manifest.json
9
+ "base_model": "HuggingFaceTB/SmolLM2-135M-Instruct", optional; else the adapter manifest's
10
+ "messages": [{"role": "user", "content": "..."}], a system message is added from the manifest if absent
11
+ "max_new_tokens": 350,
12
+ "threads": 4
13
+ }
14
+
15
+ Relative paths resolve against the request file's directory. Local files only.
16
+ Prints one JSON object on stdout:
17
+ {"v": 1, "type": "answer", "text": "...", "model": "<base>", "adapter": "sha256:<hex>", "tokens": N, "seconds": S}
18
+ or {"v": 1, "type": "error", "message": "..."} and exits 1. Logs go to stderr.
19
+ """
20
+
21
+ import hashlib
22
+ import json
23
+ import os
24
+ import sys
25
+ import time
26
+ import traceback
27
+ from pathlib import Path
28
+
29
+
30
+ def resolve(base_dir, p):
31
+ p = str(p).removeprefix("file://")
32
+ if "://" in p:
33
+ raise ValueError("adapter_dir must be a local path")
34
+ return Path(p) if os.path.isabs(p) else base_dir / p
35
+
36
+
37
+ def main():
38
+ import torch
39
+ from peft import PeftModel
40
+ from transformers import AutoModelForCausalLM, AutoTokenizer
41
+
42
+ req_path = Path(sys.argv[1] if len(sys.argv) > 1 else "request.json").resolve()
43
+ req = json.loads(req_path.read_text())
44
+ adapter = resolve(req_path.parent, req["adapter_dir"])
45
+ manifest = {}
46
+ if (adapter / "manifest.json").exists():
47
+ manifest = json.loads((adapter / "manifest.json").read_text())
48
+ base = req.get("base_model") or manifest.get("base_model")
49
+ if not base:
50
+ raise ValueError("base_model is missing (not in the request or the adapter manifest)")
51
+ weights = adapter / "adapter_model.safetensors"
52
+ digest = "sha256:" + hashlib.sha256(weights.read_bytes()).hexdigest()
53
+
54
+ messages = [m for m in req.get("messages") or [] if isinstance(m, dict) and isinstance(m.get("content"), str)]
55
+ if not messages:
56
+ raise ValueError("messages[] is required")
57
+ if messages[0].get("role") != "system" and manifest.get("system_prompt"):
58
+ messages = [{"role": "system", "content": manifest["system_prompt"]}] + messages
59
+
60
+ torch.set_num_threads(int(req.get("threads") or min(4, os.cpu_count() or 1)))
61
+ device = "cuda" if torch.cuda.is_available() else "cpu"
62
+ t0 = time.time()
63
+ tok = AutoTokenizer.from_pretrained(adapter if (adapter / "tokenizer_config.json").exists() else base)
64
+ model = AutoModelForCausalLM.from_pretrained(base, dtype=torch.float32).to(device)
65
+ model = PeftModel.from_pretrained(model, adapter).eval()
66
+ prompt = tok.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
67
+ ids = tok(prompt, return_tensors="pt").to(device)
68
+ with torch.no_grad():
69
+ # Greedy: sampling on a small model echoes the question and rambles.
70
+ out = model.generate(**ids, max_new_tokens=int(req.get("max_new_tokens") or 350), do_sample=False, repetition_penalty=1.1)
71
+ new = out[0][ids["input_ids"].shape[1]:]
72
+ text = tok.decode(new, skip_special_tokens=True).strip()
73
+ print(json.dumps({"v": 1, "type": "answer", "text": text, "model": base, "adapter": digest,
74
+ "tokens": int(new.shape[0]), "seconds": round(time.time() - t0, 2)}), flush=True)
75
+
76
+
77
+ if __name__ == "__main__":
78
+ try:
79
+ main()
80
+ except Exception as e:
81
+ traceback.print_exc(file=sys.stderr)
82
+ print(json.dumps({"v": 1, "type": "error", "message": str(e)}), flush=True)
83
+ sys.exit(1)
@@ -87,6 +87,51 @@ export async function peftRuntimeStatus() {
87
87
  });
88
88
  }
89
89
 
90
+ export const GENERATE_RUNNER_PATH = fileURLToPath(new URL("../../runners/peft_generate.py", import.meta.url));
91
+
92
+ /**
93
+ * One answer from a base model + LoRA adapter (workload infernet.text.infer).
94
+ * `request` is the runner's request object; relative paths resolve against
95
+ * `requestPath`'s directory when given. Resolves to the runner's answer
96
+ * object; rejects with its error message.
97
+ */
98
+ export async function peftGenerate({ request, requestPath = null, python = null, runner = GENERATE_RUNNER_PATH, timeoutMs = 600000, spawnImpl = spawn } = {}) {
99
+ const py = python ?? resolvePeftPython();
100
+ if (!py) throw Object.assign(new Error("the peft runtime is not installed; run `infernet model runtime setup --peft --yes`"), { code: "E_NO_RUNTIME" });
101
+ let file = requestPath;
102
+ if (!file) {
103
+ const dir = await mkdtemp(path.join(os.tmpdir(), "infernet-generate-"));
104
+ file = path.join(dir, "request.json");
105
+ await writeFile(file, JSON.stringify(request));
106
+ }
107
+ return new Promise((resolve, reject) => {
108
+ const child = spawnImpl(py, [runner, file], { stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, PYTHONUNBUFFERED: "1" } });
109
+ let out = "";
110
+ const tail = [];
111
+ child.stdout.on("data", (b) => (out += b));
112
+ child.stderr.on("data", (b) => {
113
+ for (const l of b.toString().split("\n").filter(Boolean)) {
114
+ tail.push(l);
115
+ if (tail.length > 20) tail.shift();
116
+ }
117
+ });
118
+ const timer = setTimeout(() => child.kill("SIGTERM"), timeoutMs);
119
+ child.on("error", reject);
120
+ child.on("close", (code) => {
121
+ clearTimeout(timer);
122
+ const line = out.split("\n").map((l) => l.trim()).filter((l) => l.startsWith("{")).at(-1);
123
+ let res = null;
124
+ try {
125
+ res = line ? JSON.parse(line) : null;
126
+ } catch {
127
+ res = null;
128
+ }
129
+ if (code === 0 && res?.type === "answer") return resolve(res);
130
+ reject(Object.assign(new Error(res?.message ?? `generate exited ${code}: ${tail.at(-1) ?? "no output"}`), { code: "E_RUNTIME" }));
131
+ });
132
+ });
133
+ }
134
+
90
135
  export function createPeftBackend({ python = null, runner = RUNNER_PATH, spawnImpl = spawn } = {}) {
91
136
  const children = new Set();
92
137
  return {
package/src/index.js CHANGED
@@ -43,7 +43,9 @@ export * from "./protocol.js";
43
43
  export {
44
44
  createPeftBackend,
45
45
  peftRuntimeDir,
46
+ peftGenerate,
46
47
  peftRuntimeStatus,
48
+ GENERATE_RUNNER_PATH as PEFT_GENERATE_RUNNER_PATH,
47
49
  PEFT_KINDS,
48
50
  resolvePeftPython,
49
51
  RUNNER_PATH as PEFT_RUNNER_PATH,