@infernetprotocol/training 0.1.79 → 0.1.81

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@infernetprotocol/training",
3
3
  "private": false,
4
- "version": "0.1.79",
4
+ "version": "0.1.81",
5
5
  "description": "Training orchestration for Infernet — pluggable backends (DeepSpeed/FSDP for trusted clusters, OpenDiLoCo for cross-provider async, Petals for trustless P2P fine-tunes) over a stable async-iterator API.",
6
6
  "license": "MIT",
7
7
  "type": "module",
@@ -19,8 +19,8 @@
19
19
  "node": ">=18"
20
20
  },
21
21
  "dependencies": {
22
- "@infernetprotocol/engine": "0.1.79",
23
- "@infernetprotocol/liquid-agent": "0.1.79"
22
+ "@infernetprotocol/engine": "0.1.81",
23
+ "@infernetprotocol/liquid-agent": "0.1.81"
24
24
  },
25
25
  "repository": {
26
26
  "type": "git",
@@ -0,0 +1,83 @@
1
+ #!/usr/bin/env python3
2
+ """Answer chat messages with a base model plus a LoRA adapter (workload infernet.text.infer).
3
+
4
+ python peft_generate.py request.json
5
+
6
+ request.json:
7
+ {
8
+ "adapter_dir": "adapter", local dir: adapter_config.json, adapter_model.safetensors, manifest.json
9
+ "base_model": "HuggingFaceTB/SmolLM2-135M-Instruct", optional; else the adapter manifest's
10
+ "messages": [{"role": "user", "content": "..."}], a system message is added from the manifest if absent
11
+ "max_new_tokens": 350,
12
+ "threads": 4
13
+ }
14
+
15
+ Relative paths resolve against the request file's directory. Local files only.
16
+ Prints one JSON object on stdout:
17
+ {"v": 1, "type": "answer", "text": "...", "model": "<base>", "adapter": "sha256:<hex>", "tokens": N, "seconds": S}
18
+ or {"v": 1, "type": "error", "message": "..."} and exits 1. Logs go to stderr.
19
+ """
20
+
21
+ import hashlib
22
+ import json
23
+ import os
24
+ import sys
25
+ import time
26
+ import traceback
27
+ from pathlib import Path
28
+
29
+
30
+ def resolve(base_dir, p):
31
+ p = str(p).removeprefix("file://")
32
+ if "://" in p:
33
+ raise ValueError("adapter_dir must be a local path")
34
+ return Path(p) if os.path.isabs(p) else base_dir / p
35
+
36
+
37
+ def main():
38
+ import torch
39
+ from peft import PeftModel
40
+ from transformers import AutoModelForCausalLM, AutoTokenizer
41
+
42
+ req_path = Path(sys.argv[1] if len(sys.argv) > 1 else "request.json").resolve()
43
+ req = json.loads(req_path.read_text())
44
+ adapter = resolve(req_path.parent, req["adapter_dir"])
45
+ manifest = {}
46
+ if (adapter / "manifest.json").exists():
47
+ manifest = json.loads((adapter / "manifest.json").read_text())
48
+ base = req.get("base_model") or manifest.get("base_model")
49
+ if not base:
50
+ raise ValueError("base_model is missing (not in the request or the adapter manifest)")
51
+ weights = adapter / "adapter_model.safetensors"
52
+ digest = "sha256:" + hashlib.sha256(weights.read_bytes()).hexdigest()
53
+
54
+ messages = [m for m in req.get("messages") or [] if isinstance(m, dict) and isinstance(m.get("content"), str)]
55
+ if not messages:
56
+ raise ValueError("messages[] is required")
57
+ if messages[0].get("role") != "system" and manifest.get("system_prompt"):
58
+ messages = [{"role": "system", "content": manifest["system_prompt"]}] + messages
59
+
60
+ torch.set_num_threads(int(req.get("threads") or min(4, os.cpu_count() or 1)))
61
+ device = "cuda" if torch.cuda.is_available() else "cpu"
62
+ t0 = time.time()
63
+ tok = AutoTokenizer.from_pretrained(adapter if (adapter / "tokenizer_config.json").exists() else base)
64
+ model = AutoModelForCausalLM.from_pretrained(base, dtype=torch.float32).to(device)
65
+ model = PeftModel.from_pretrained(model, adapter).eval()
66
+ prompt = tok.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
67
+ ids = tok(prompt, return_tensors="pt").to(device)
68
+ with torch.no_grad():
69
+ # Greedy: sampling on a small model echoes the question and rambles.
70
+ out = model.generate(**ids, max_new_tokens=int(req.get("max_new_tokens") or 350), do_sample=False, repetition_penalty=1.1)
71
+ new = out[0][ids["input_ids"].shape[1]:]
72
+ text = tok.decode(new, skip_special_tokens=True).strip()
73
+ print(json.dumps({"v": 1, "type": "answer", "text": text, "model": base, "adapter": digest,
74
+ "tokens": int(new.shape[0]), "seconds": round(time.time() - t0, 2)}), flush=True)
75
+
76
+
77
+ if __name__ == "__main__":
78
+ try:
79
+ main()
80
+ except Exception as e:
81
+ traceback.print_exc(file=sys.stderr)
82
+ print(json.dumps({"v": 1, "type": "error", "message": str(e)}), flush=True)
83
+ sys.exit(1)
@@ -18,7 +18,7 @@
18
18
  import { randomUUID } from "node:crypto";
19
19
  import { spawn } from "node:child_process";
20
20
  import { existsSync } from "node:fs";
21
- import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
21
+ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
22
22
  import os from "node:os";
23
23
  import path from "node:path";
24
24
  import { fileURLToPath } from "node:url";
@@ -58,8 +58,26 @@ export async function setupPeftRuntime({ accelerator = "cpu", python = "python3"
58
58
  const dir = peftRuntimeDir();
59
59
  await mkdir(dir, { recursive: true });
60
60
  const venv = path.join(dir, "venv");
61
- if (!existsSync(path.join(venv, "bin", "python"))) await run(python, ["-m", "venv", venv], log);
62
61
  const py = path.join(venv, "bin", "python");
62
+ if (!existsSync(py)) {
63
+ try {
64
+ await run(python, ["-m", "venv", venv], log);
65
+ } catch (err) {
66
+ // Debian/Ubuntu ship venv without ensurepip unless python3-venv is
67
+ // installed (needs root). A pip-less venv plus get-pip.py works anyway.
68
+ log?.(`python -m venv failed (${err.message}); retrying without pip`);
69
+ await rm(venv, { recursive: true, force: true });
70
+ await run(python, ["-m", "venv", "--without-pip", venv], log);
71
+ }
72
+ }
73
+ const hasPip = await run(py, ["-m", "pip", "--version"]).then(() => true, () => false);
74
+ if (!hasPip) {
75
+ const getPip = path.join(dir, "get-pip.py");
76
+ const res = await fetch("https://bootstrap.pypa.io/get-pip.py");
77
+ if (!res.ok) throw new Error(`could not download get-pip.py: HTTP ${res.status}`);
78
+ await writeFile(getPip, Buffer.from(await res.arrayBuffer()));
79
+ await run(py, [getPip, "--quiet"], log);
80
+ }
63
81
  await run(py, ["-m", "pip", "install", "--upgrade", "--quiet", "pip"], log);
64
82
  const torchIndex = accelerator === "cuda" ? [] : ["--index-url", "https://download.pytorch.org/whl/cpu"];
65
83
  await run(py, ["-m", "pip", "install", "--quiet", "torch", ...torchIndex], log);
@@ -87,6 +105,51 @@ export async function peftRuntimeStatus() {
87
105
  });
88
106
  }
89
107
 
108
+ export const GENERATE_RUNNER_PATH = fileURLToPath(new URL("../../runners/peft_generate.py", import.meta.url));
109
+
110
+ /**
111
+ * One answer from a base model + LoRA adapter (workload infernet.text.infer).
112
+ * `request` is the runner's request object; relative paths resolve against
113
+ * `requestPath`'s directory when given. Resolves to the runner's answer
114
+ * object; rejects with its error message.
115
+ */
116
+ export async function peftGenerate({ request, requestPath = null, python = null, runner = GENERATE_RUNNER_PATH, timeoutMs = 600000, spawnImpl = spawn } = {}) {
117
+ const py = python ?? resolvePeftPython();
118
+ if (!py) throw Object.assign(new Error("the peft runtime is not installed; run `infernet model runtime setup --peft --yes`"), { code: "E_NO_RUNTIME" });
119
+ let file = requestPath;
120
+ if (!file) {
121
+ const dir = await mkdtemp(path.join(os.tmpdir(), "infernet-generate-"));
122
+ file = path.join(dir, "request.json");
123
+ await writeFile(file, JSON.stringify(request));
124
+ }
125
+ return new Promise((resolve, reject) => {
126
+ const child = spawnImpl(py, [runner, file], { stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, PYTHONUNBUFFERED: "1" } });
127
+ let out = "";
128
+ const tail = [];
129
+ child.stdout.on("data", (b) => (out += b));
130
+ child.stderr.on("data", (b) => {
131
+ for (const l of b.toString().split("\n").filter(Boolean)) {
132
+ tail.push(l);
133
+ if (tail.length > 20) tail.shift();
134
+ }
135
+ });
136
+ const timer = setTimeout(() => child.kill("SIGTERM"), timeoutMs);
137
+ child.on("error", reject);
138
+ child.on("close", (code) => {
139
+ clearTimeout(timer);
140
+ const line = out.split("\n").map((l) => l.trim()).filter((l) => l.startsWith("{")).at(-1);
141
+ let res = null;
142
+ try {
143
+ res = line ? JSON.parse(line) : null;
144
+ } catch {
145
+ res = null;
146
+ }
147
+ if (code === 0 && res?.type === "answer") return resolve(res);
148
+ reject(Object.assign(new Error(res?.message ?? `generate exited ${code}: ${tail.at(-1) ?? "no output"}`), { code: "E_RUNTIME" }));
149
+ });
150
+ });
151
+ }
152
+
90
153
  export function createPeftBackend({ python = null, runner = RUNNER_PATH, spawnImpl = spawn } = {}) {
91
154
  const children = new Set();
92
155
  return {
package/src/index.js CHANGED
@@ -43,7 +43,9 @@ export * from "./protocol.js";
43
43
  export {
44
44
  createPeftBackend,
45
45
  peftRuntimeDir,
46
+ peftGenerate,
46
47
  peftRuntimeStatus,
48
+ GENERATE_RUNNER_PATH as PEFT_GENERATE_RUNNER_PATH,
47
49
  PEFT_KINDS,
48
50
  resolvePeftPython,
49
51
  RUNNER_PATH as PEFT_RUNNER_PATH,