@infernetprotocol/cli 0.1.31 → 0.1.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,239 @@
1
+ /**
2
+ * `infernet publish` — push a trained model to HuggingFace and/or Ollama.
3
+ *
4
+ * The Ollama path is what makes a model show up at e.g.
5
+ * https://ollama.com/<your-org>/<model>
6
+ * (similar to ollama.com/rockypod/svelte-coder). We:
7
+ * 1. Locate the safetensors checkpoint in <dir>.
8
+ * 2. Optionally push it to huggingface.co/<HF_ORG>/<name> via huggingface-cli.
9
+ * 3. Convert to GGUF (llama.cpp) — best with the user's local llama.cpp install.
10
+ * 4. Generate a Modelfile + `ollama create` + `ollama push`.
11
+ *
12
+ * Requires:
13
+ * HUGGINGFACE_TOKEN for the HF push
14
+ * OLLAMA_USERNAME for the Ollama push (ollama signin must already be done)
15
+ * llama.cpp cloned to ~/llama.cpp OR override with --llama-cpp-path
16
+ */
17
+
18
+ import fs from "node:fs";
19
+ import fsp from "node:fs/promises";
20
+ import path from "node:path";
21
+ import { spawn } from "node:child_process";
22
+ import { loadConfig } from "../lib/config.js";
23
+
24
+ const HELP = `infernet publish — push a trained model to HuggingFace + Ollama
25
+
26
+ Usage:
27
+ infernet publish <dir> [flags]
28
+
29
+ Required:
30
+ <dir> Path to the trained model directory
31
+ (must contain config.json + *.safetensors)
32
+
33
+ Flags:
34
+ --hf <org>/<name> Push to huggingface.co/<org>/<name>
35
+ --ollama <user>/<name> Push to ollama.com/<user>/<name>
36
+ --base <hf-id> Base model id for the GGUF Modelfile FROM
37
+ (default: read from <dir>/config.json)
38
+ --quant <q> GGUF quantization (default: q4_k_m)
39
+ q2_k | q3_k_m | q4_0 | q4_k_m | q5_k_m | q8_0
40
+ --llama-cpp-path <dir> llama.cpp checkout (default: \$HOME/llama.cpp)
41
+ --skip-hf Skip the HuggingFace push (Ollama only)
42
+ --skip-ollama Skip the Ollama push (HF only)
43
+ --modelfile-only Just generate the Modelfile + GGUF, don't push
44
+ --help
45
+
46
+ Examples:
47
+ infernet publish ./run/checkpoint-final \\
48
+ --hf InfernetProtocol/svelte5-coder \\
49
+ --ollama infernet/svelte5-coder
50
+
51
+ infernet publish ./run --modelfile-only --quant q5_k_m
52
+
53
+ Prereqs:
54
+ HUGGINGFACE_TOKEN env or .env (write scope on the org)
55
+ ollama signin done once, prior to first push
56
+ llama.cpp cloned + built at \$HOME/llama.cpp (for GGUF convert)
57
+ `;
58
+
59
+ async function run(cmd, args, opts = {}) {
60
+ return new Promise((resolve, reject) => {
61
+ const child = spawn(cmd, args, { stdio: "inherit", ...opts });
62
+ child.on("exit", (code) => code === 0 ? resolve() : reject(new Error(`${cmd} ${args.join(" ")} exited ${code}`)));
63
+ child.on("error", reject);
64
+ });
65
+ }
66
+
67
+ async function which(bin) {
68
+ return new Promise((resolve) => {
69
+ const child = spawn("which", [bin], { stdio: ["ignore", "pipe", "ignore"] });
70
+ let out = "";
71
+ child.stdout.on("data", (b) => { out += b.toString(); });
72
+ child.on("exit", (code) => resolve(code === 0 ? out.trim() : null));
73
+ child.on("error", () => resolve(null));
74
+ });
75
+ }
76
+
77
+ async function detectBase(dir) {
78
+ try {
79
+ const cfg = JSON.parse(await fsp.readFile(path.join(dir, "config.json"), "utf8"));
80
+ return cfg?._name_or_path
81
+ ?? cfg?.model_name_or_path
82
+ ?? cfg?.base_model
83
+ ?? null;
84
+ } catch {
85
+ return null;
86
+ }
87
+ }
88
+
89
+ async function pushToHuggingFace({ dir, hfRepo, token }) {
90
+ if (!token) throw new Error("HUGGINGFACE_TOKEN required for HF push");
91
+ const hfCli = (await which("huggingface-cli")) ?? (await which("hf"));
92
+ if (!hfCli) {
93
+ throw new Error("huggingface-cli not found. Install: pip install -U huggingface_hub");
94
+ }
95
+ process.stdout.write(`\n→ Pushing ${dir} → huggingface.co/${hfRepo}\n`);
96
+ // Create the repo (idempotent — won't fail if it exists)
97
+ await run(hfCli, ["repo", "create", hfRepo, "--type", "model", "-y"], {
98
+ env: { ...process.env, HF_TOKEN: token, HUGGINGFACE_TOKEN: token }
99
+ }).catch(() => { /* may already exist */ });
100
+ await run(hfCli, ["upload", hfRepo, dir, ".", "--repo-type", "model"], {
101
+ env: { ...process.env, HF_TOKEN: token, HUGGINGFACE_TOKEN: token }
102
+ });
103
+ process.stdout.write(`✓ HF: https://huggingface.co/${hfRepo}\n`);
104
+ }
105
+
106
+ async function convertToGgufAndPush({ dir, ollamaRepo, base, quant = "q4_k_m", llamaCppPath, modelfileOnly }) {
107
+ const llamaCpp = llamaCppPath ?? path.join(process.env.HOME ?? "", "llama.cpp");
108
+ const convertScript = path.join(llamaCpp, "convert_hf_to_gguf.py");
109
+ if (!fs.existsSync(convertScript)) {
110
+ throw new Error(
111
+ `llama.cpp convert_hf_to_gguf.py not found at ${convertScript}.\n` +
112
+ `Clone + build llama.cpp first:\n` +
113
+ ` git clone https://github.com/ggml-org/llama.cpp ~/llama.cpp\n` +
114
+ ` cd ~/llama.cpp && cmake -B build && cmake --build build -j`
115
+ );
116
+ }
117
+
118
+ const ggufRaw = path.join(dir, "model.f16.gguf");
119
+ const ggufQuant = path.join(dir, `model.${quant}.gguf`);
120
+
121
+ process.stdout.write(`\n→ Converting ${dir} → ${ggufRaw}\n`);
122
+ await run("python3", [convertScript, dir, "--outfile", ggufRaw, "--outtype", "f16"]);
123
+
124
+ const quantBin = path.join(llamaCpp, "build", "bin", "llama-quantize");
125
+ if (fs.existsSync(quantBin)) {
126
+ process.stdout.write(`\n→ Quantizing → ${ggufQuant} (${quant})\n`);
127
+ await run(quantBin, [ggufRaw, ggufQuant, quant.toUpperCase()]);
128
+ } else {
129
+ process.stdout.write(`(skipping quantization — llama-quantize not built; using f16 GGUF)\n`);
130
+ }
131
+
132
+ const finalGguf = fs.existsSync(ggufQuant) ? ggufQuant : ggufRaw;
133
+
134
+ // Generate a Modelfile.
135
+ const modelfilePath = path.join(dir, "Modelfile");
136
+ const modelfile = [
137
+ `# Auto-generated by \`infernet publish\``,
138
+ `FROM ${finalGguf}`,
139
+ ``,
140
+ `# Inherit conversation template from base if provided.`,
141
+ base ? `# Base: ${base}` : `# Base: (unknown)`,
142
+ ``,
143
+ `PARAMETER temperature 0.7`,
144
+ `PARAMETER top_p 0.9`,
145
+ `PARAMETER repeat_penalty 1.1`,
146
+ ``,
147
+ `# ChatML template (works for Qwen / Llama-3 chat / most modern fine-tunes)`,
148
+ `TEMPLATE """{{ if .System }}<|im_start|>system`,
149
+ `{{ .System }}<|im_end|>`,
150
+ `{{ end }}{{ if .Prompt }}<|im_start|>user`,
151
+ `{{ .Prompt }}<|im_end|>`,
152
+ `{{ end }}<|im_start|>assistant`,
153
+ `{{ .Response }}<|im_end|>"""`,
154
+ ``,
155
+ `SYSTEM """You are a helpful assistant fine-tuned by Infernet Protocol."""`
156
+ ].join("\n");
157
+ await fsp.writeFile(modelfilePath, modelfile);
158
+ process.stdout.write(`\n✓ Modelfile: ${modelfilePath}\n`);
159
+
160
+ if (modelfileOnly) {
161
+ process.stdout.write(`\n(--modelfile-only set; skipping ollama create + push)\n`);
162
+ return;
163
+ }
164
+
165
+ if (!ollamaRepo) {
166
+ process.stdout.write(`(no --ollama target; skipping ollama push)\n`);
167
+ return;
168
+ }
169
+
170
+ process.stdout.write(`\n→ ollama create ${ollamaRepo} -f ${modelfilePath}\n`);
171
+ await run("ollama", ["create", ollamaRepo, "-f", modelfilePath]);
172
+
173
+ process.stdout.write(`\n→ ollama push ${ollamaRepo}\n`);
174
+ await run("ollama", ["push", ollamaRepo]);
175
+ process.stdout.write(`✓ Ollama: https://ollama.com/${ollamaRepo}\n`);
176
+ }
177
+
178
+ export default async function publish(args) {
179
+ if (args.has("help") || args.has("h")) {
180
+ process.stdout.write(HELP);
181
+ return 0;
182
+ }
183
+ const dir = args.positional?.[0];
184
+ if (!dir) {
185
+ process.stderr.write("error: model directory required\n\n");
186
+ process.stderr.write(HELP);
187
+ return 2;
188
+ }
189
+ if (!fs.existsSync(dir)) {
190
+ process.stderr.write(`error: ${dir} does not exist\n`);
191
+ return 1;
192
+ }
193
+
194
+ const hfRepo = args.get("hf") ?? null;
195
+ const ollamaRepo = args.get("ollama") ?? null;
196
+ const skipHf = args.has("skip-hf") || !hfRepo;
197
+ const skipOllama = args.has("skip-ollama") || (!ollamaRepo && !args.has("modelfile-only"));
198
+ const modelfileOnly = args.has("modelfile-only");
199
+ const quant = args.get("quant") ?? "q4_k_m";
200
+ const llamaCppPath = args.get("llama-cpp-path") ?? null;
201
+
202
+ if (skipHf && skipOllama && !modelfileOnly) {
203
+ process.stderr.write("error: nothing to do (set --hf <org>/<name> and/or --ollama <user>/<name>, or --modelfile-only)\n");
204
+ return 2;
205
+ }
206
+
207
+ const cfg = await loadConfig().catch(() => null);
208
+ const hfToken = process.env.HUGGINGFACE_TOKEN ?? cfg?.huggingface?.token ?? null;
209
+
210
+ const base = args.get("base") ?? await detectBase(dir);
211
+ if (!base && !skipOllama) {
212
+ process.stdout.write("(could not detect base model from config.json — Modelfile FROM line will reference the GGUF only)\n");
213
+ }
214
+
215
+ if (!skipHf) {
216
+ try {
217
+ await pushToHuggingFace({ dir, hfRepo, token: hfToken });
218
+ } catch (err) {
219
+ process.stderr.write(`HF push failed: ${err?.message ?? err}\n`);
220
+ return 1;
221
+ }
222
+ }
223
+
224
+ if (!skipOllama || modelfileOnly) {
225
+ try {
226
+ await convertToGgufAndPush({
227
+ dir, ollamaRepo, base, quant, llamaCppPath, modelfileOnly
228
+ });
229
+ } catch (err) {
230
+ process.stderr.write(`Ollama path failed: ${err?.message ?? err}\n`);
231
+ return 1;
232
+ }
233
+ }
234
+
235
+ process.stdout.write(`\n✓ publish complete\n`);
236
+ return 0;
237
+ }
238
+
239
+ export { HELP };
package/commands/train.js CHANGED
@@ -19,10 +19,15 @@ import os from "node:os";
19
19
  import { spawn } from "node:child_process";
20
20
  import { createWriteStream } from "node:fs";
21
21
  import { loadConfig } from "../lib/config.js";
22
+ import { resolveValueSerpKey, buildTrainingDataset } from "../lib/data-crawler.js";
22
23
 
23
24
  const HELP = `infernet train — fine-tune or train models on the P2P GPU network
24
25
 
25
26
  Usage:
27
+ infernet train data --query <q> Crawl ValueSerp + emit JSONL
28
+ [--out <file>] (default: ./data/train.jsonl)
29
+ [--num <n>] (default: 20 results)
30
+ [--domains <a,b,c>] (whitelist filter)
26
31
  infernet train init [--output <dir>] Scaffold infernet.train.yml
27
32
  infernet train run [--config <file>] Run training (local or P2P)
28
33
  [--local] Force local execution
@@ -32,6 +37,7 @@ Usage:
32
37
  infernet train logs [<run-id>] Tail run logs
33
38
 
34
39
  Examples:
40
+ infernet train data --query "svelte 5 framework docs" --domains svelte.dev,github.com
35
41
  infernet train init --output ./my-run
36
42
  infernet train run --local --output ./my-run
37
43
  infernet train run --config infernet.train.yml
@@ -524,6 +530,47 @@ async function cmdLogs(args) {
524
530
  return 0;
525
531
  }
526
532
 
533
+ async function cmdData(args) {
534
+ const query = args.get("query") ?? args.get("q");
535
+ if (!query) {
536
+ process.stderr.write("error: --query <text> is required\n");
537
+ process.stderr.write(`example: infernet train data --query "svelte 5 framework docs"\n`);
538
+ return 2;
539
+ }
540
+ const apiKey = await resolveValueSerpKey();
541
+ if (!apiKey) {
542
+ process.stderr.write(
543
+ "error: VALUESERP_API_KEY is not set.\n" +
544
+ "Get a key at https://valueserp.com → set in env or in\n" +
545
+ "~/.config/infernet/config.json under integrations.valueserp.api_key\n"
546
+ );
547
+ return 1;
548
+ }
549
+ const out = args.get("out") ?? args.get("output") ?? "./data/train.jsonl";
550
+ const num = Number.parseInt(args.get("num") ?? "20", 10) || 20;
551
+ const domains = args.get("domains")?.split(",").map((s) => s.trim()).filter(Boolean) ?? null;
552
+ const minChars = Number.parseInt(args.get("min-chars") ?? "200", 10) || 200;
553
+ const maxChars = Number.parseInt(args.get("max-chars") ?? "4000", 10) || 4000;
554
+
555
+ process.stdout.write(`Searching ValueSerp for: "${query}"\n`);
556
+ if (domains) process.stdout.write(` domain whitelist: ${domains.join(", ")}\n`);
557
+
558
+ const result = await buildTrainingDataset({
559
+ query, apiKey, outPath: out, num, domains, minChars, maxChars,
560
+ onProgress: ({ stage, index, total, url }) => {
561
+ if (stage === "fetch") {
562
+ process.stdout.write(` [${index}/${total}] ${url}\n`);
563
+ }
564
+ }
565
+ });
566
+
567
+ process.stdout.write(`\n✓ Wrote ${result.examples} examples from ${result.urls} URLs to ${result.outPath}\n`);
568
+ process.stdout.write(`\nNext steps:\n`);
569
+ process.stdout.write(` infernet train init --output ./run # scaffold infernet.train.yml\n`);
570
+ process.stdout.write(` infernet train run --local --output ./run # train against ${out}\n`);
571
+ return 0;
572
+ }
573
+
527
574
  export default async function train(args) {
528
575
  if (args.has("help") || args.has("h")) {
529
576
  process.stdout.write(HELP);
@@ -535,6 +582,9 @@ export default async function train(args) {
535
582
 
536
583
  try {
537
584
  switch (sub) {
585
+ case "data":
586
+ case "crawl":
587
+ return await cmdData(args);
538
588
  case "init":
539
589
  return await cmdInit(args);
540
590
  case "run":
package/index.js CHANGED
@@ -44,6 +44,7 @@ import debug from './commands/debug.js';
44
44
  import deploy from './commands/deploy.js';
45
45
  import consoleCmd from './commands/console.js';
46
46
  import train from './commands/train.js';
47
+ import publish from './commands/publish.js';
47
48
  import uncensored from './commands/uncensored.js';
48
49
 
49
50
  function parseArgs(argv) {
@@ -106,11 +107,11 @@ const COMMANDS = {
106
107
  // aliases
107
108
  uninstall: remove,
108
109
  start, status, stop, restart, stats, logs,
109
- payout, payments, gpu, firewall, chat, setup, model, train, uncensored, tui, doctor, service, pubkey, debug, deploy, console: consoleCmd, help
110
+ payout, payments, gpu, firewall, chat, setup, model, train, publish, uncensored, tui, doctor, service, pubkey, debug, deploy, console: consoleCmd, help
110
111
  };
111
112
 
112
113
  // Commands that can run without a loaded config.
113
- const NO_CONFIG = new Set(['init', 'login', 'help', 'stats', 'logs', 'stop', 'restart', 'gpu', 'firewall', 'chat', 'setup', 'model', 'train', 'uncensored', 'tui', 'doctor', 'service', 'pubkey', 'debug', 'deploy', 'console', 'upgrade', 'update', 'remove', 'uninstall']);
114
+ const NO_CONFIG = new Set(['init', 'login', 'help', 'stats', 'logs', 'stop', 'restart', 'gpu', 'firewall', 'chat', 'setup', 'model', 'train', 'publish', 'uncensored', 'tui', 'doctor', 'service', 'pubkey', 'debug', 'deploy', 'console', 'upgrade', 'update', 'remove', 'uninstall']);
114
115
  // Commands that need a config but not a control-plane client (none today
115
116
  // — kept as a future escape hatch).
116
117
  const NO_CLIENT = new Set();
@@ -0,0 +1,183 @@
1
+ /**
2
+ * Search-driven training-data collector.
3
+ *
4
+ * Pipeline:
5
+ * 1. Hit valueserp.com /search with a query. Returns ranked URLs.
6
+ * 2. For each URL: fetch HTML, strip nav/script/style, extract main text.
7
+ * 3. For each page: chunk into Q/A pairs (heading-based) and emit JSONL.
8
+ *
9
+ * The output JSONL uses the chatml `messages` shape, ready to feed into
10
+ * `infernet train run` or any HF/Unsloth/Axolotl pipeline.
11
+ *
12
+ * VALUESERP_API_KEY must be set in env or `~/.config/infernet/config.json`
13
+ * under integrations.valueserp.api_key.
14
+ */
15
+
16
+ import fs from "node:fs/promises";
17
+ import path from "node:path";
18
+ import { loadConfig } from "./config.js";
19
+
20
+ export async function resolveValueSerpKey() {
21
+ if (process.env.VALUESERP_API_KEY) return process.env.VALUESERP_API_KEY;
22
+ const cfg = await loadConfig().catch(() => null);
23
+ return cfg?.integrations?.valueserp?.api_key
24
+ ?? cfg?.valueserp?.api_key
25
+ ?? null;
26
+ }
27
+
28
+ /**
29
+ * Search the web via ValueSerp and return the ranked list of result URLs.
30
+ * @param {object} opts
31
+ * @param {string} opts.query
32
+ * @param {string} opts.apiKey
33
+ * @param {number} [opts.num=20]
34
+ * @param {string} [opts.location]
35
+ */
36
+ export async function valueSerpSearch({ query, apiKey, num = 20, location } = {}) {
37
+ if (!query) throw new Error("valueserp: query required");
38
+ if (!apiKey) throw new Error("valueserp: apiKey required (set VALUESERP_API_KEY)");
39
+
40
+ const params = new URLSearchParams({
41
+ api_key: apiKey,
42
+ q: query,
43
+ num: String(num),
44
+ output: "json"
45
+ });
46
+ if (location) params.set("location", location);
47
+
48
+ const res = await fetch(`https://api.valueserp.com/search?${params}`);
49
+ if (!res.ok) {
50
+ const txt = await res.text().catch(() => res.statusText);
51
+ throw new Error(`ValueSerp ${res.status}: ${txt.slice(0, 200)}`);
52
+ }
53
+ const json = await res.json();
54
+ return (json?.organic_results ?? []).map((r) => ({
55
+ rank: r.position,
56
+ title: r.title,
57
+ url: r.link,
58
+ snippet: r.snippet,
59
+ domain: r.domain
60
+ }));
61
+ }
62
+
63
+ /** Cheap HTML → readable text. Strips scripts/styles/nav, collapses whitespace. */
64
+ export function extractTextFromHtml(html) {
65
+ if (!html) return "";
66
+ // Drop script + style content entirely.
67
+ let s = html
68
+ .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, "")
69
+ .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, "")
70
+ .replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi, "")
71
+ .replace(/<head\b[^>]*>[\s\S]*?<\/head>/gi, "");
72
+ // Drop common non-content elements.
73
+ s = s.replace(/<(nav|footer|aside|header|form|iframe)\b[^>]*>[\s\S]*?<\/\1>/gi, "");
74
+ // Convert headings/paragraphs/list items to text with line breaks.
75
+ s = s.replace(/<\/?(h[1-6]|p|li|br|div|tr)[^>]*>/gi, "\n");
76
+ // Strip remaining tags.
77
+ s = s.replace(/<[^>]+>/g, " ");
78
+ // Decode common entities.
79
+ s = s
80
+ .replace(/&nbsp;/g, " ")
81
+ .replace(/&amp;/g, "&")
82
+ .replace(/&lt;/g, "<")
83
+ .replace(/&gt;/g, ">")
84
+ .replace(/&quot;/g, '"')
85
+ .replace(/&#39;/g, "'")
86
+ .replace(/&#x27;/g, "'");
87
+ // Collapse whitespace.
88
+ s = s.replace(/[ \t]+/g, " ").replace(/\n[ \t]*/g, "\n").replace(/\n{3,}/g, "\n\n");
89
+ return s.trim();
90
+ }
91
+
92
+ /** Fetch a URL and extract its main text content. Returns "" on failure. */
93
+ export async function fetchAndExtract(url, { timeoutMs = 15_000 } = {}) {
94
+ try {
95
+ const ctrl = new AbortController();
96
+ const t = setTimeout(() => ctrl.abort(), timeoutMs);
97
+ const res = await fetch(url, {
98
+ signal: ctrl.signal,
99
+ redirect: "follow",
100
+ headers: {
101
+ "User-Agent": "Mozilla/5.0 (compatible; InfernetTrainCrawler/0.1; +https://infernetprotocol.com)"
102
+ }
103
+ });
104
+ clearTimeout(t);
105
+ if (!res.ok) return "";
106
+ const ct = res.headers.get("content-type") ?? "";
107
+ if (!ct.includes("text/html") && !ct.includes("text/plain")) return "";
108
+ const html = await res.text();
109
+ return extractTextFromHtml(html);
110
+ } catch {
111
+ return "";
112
+ }
113
+ }
114
+
115
+ /**
116
+ * Convert a long body of text into chatml-style JSONL training examples.
117
+ * Heuristic: split on blank lines into paragraphs; treat the first
118
+ * paragraph as a "question" prompt and following paragraphs as the
119
+ * model's answer. Skips paragraphs shorter than minChars or longer than
120
+ * maxChars (caps very long pages).
121
+ */
122
+ export function paragraphsToChatml({ url, title, text, query, minChars = 200, maxChars = 4000 }) {
123
+ const paras = text.split(/\n{2,}/).map((p) => p.trim()).filter(Boolean);
124
+ const examples = [];
125
+ for (let i = 0; i < paras.length; i += 1) {
126
+ const para = paras[i];
127
+ if (para.length < minChars) continue;
128
+ const truncated = para.length > maxChars ? para.slice(0, maxChars) + "…" : para;
129
+ examples.push({
130
+ messages: [
131
+ { role: "system", content: `You are an expert assistant trained on ${title ?? url}.` },
132
+ { role: "user", content: `${query} — section ${i + 1}` },
133
+ { role: "assistant", content: truncated }
134
+ ],
135
+ meta: { source_url: url, source_title: title, query, paragraph_index: i }
136
+ });
137
+ }
138
+ return examples;
139
+ }
140
+
141
+ /**
142
+ * Top-level helper: search + crawl + chunk → write training JSONL.
143
+ * Used by the `infernet train data` CLI surface.
144
+ *
145
+ * @returns {{ urls: number, examples: number, outPath: string }}
146
+ */
147
+ export async function buildTrainingDataset({
148
+ query,
149
+ apiKey,
150
+ outPath,
151
+ num = 20,
152
+ minChars = 200,
153
+ maxChars = 4000,
154
+ domains = null, // optional whitelist (e.g. ["svelte.dev", "github.com"])
155
+ onProgress = () => {}
156
+ }) {
157
+ const results = await valueSerpSearch({ query, apiKey, num });
158
+ const filtered = domains
159
+ ? results.filter((r) => domains.some((d) => r.domain?.includes(d)))
160
+ : results;
161
+
162
+ await fs.mkdir(path.dirname(outPath), { recursive: true });
163
+ const out = await fs.open(outPath, "w");
164
+
165
+ let totalExamples = 0;
166
+ let urlsUsed = 0;
167
+ for (let i = 0; i < filtered.length; i += 1) {
168
+ const r = filtered[i];
169
+ onProgress({ stage: "fetch", index: i + 1, total: filtered.length, url: r.url });
170
+ const text = await fetchAndExtract(r.url);
171
+ if (!text || text.length < minChars) continue;
172
+ urlsUsed += 1;
173
+ const examples = paragraphsToChatml({
174
+ url: r.url, title: r.title, text, query, minChars, maxChars
175
+ });
176
+ for (const ex of examples) {
177
+ await out.write(JSON.stringify(ex) + "\n");
178
+ totalExamples += 1;
179
+ }
180
+ }
181
+ await out.close();
182
+ return { urls: urlsUsed, examples: totalExamples, outPath };
183
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@infernetprotocol/cli",
3
- "version": "0.1.31",
3
+ "version": "0.1.33",
4
4
  "description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -30,11 +30,11 @@
30
30
  "ora": "^9.4.0",
31
31
  "react": "^17.0.2",
32
32
  "react-blessed": "^0.7.2",
33
- "@infernetprotocol/auth": "0.1.31",
34
- "@infernetprotocol/engine": "0.1.31",
35
- "@infernetprotocol/deploy-providers": "0.1.31",
36
- "@infernetprotocol/gpu": "0.1.31",
37
- "@infernetprotocol/config": "0.1.31"
33
+ "@infernetprotocol/auth": "0.1.33",
34
+ "@infernetprotocol/deploy-providers": "0.1.33",
35
+ "@infernetprotocol/config": "0.1.33",
36
+ "@infernetprotocol/gpu": "0.1.33",
37
+ "@infernetprotocol/engine": "0.1.33"
38
38
  },
39
39
  "repository": {
40
40
  "type": "git",