@infernetprotocol/cli 0.1.40 → 0.1.45

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,115 +0,0 @@
1
- /**
2
- * Bundled Python petals client. Spawned by the daemon when it accepts
3
- * an /v1/petals/inference POST. Fans out across the public Petals
4
- * swarm using AutoDistributedModelForCausalLM and streams JSON lines
5
- * (one per token) to stdout. The daemon converts those to SSE frames.
6
- */
7
-
8
- export const PETALS_CLIENT_PY = `#!/usr/bin/env python3
9
- """Petals client — fans out across the swarm + streams tokens.
10
-
11
- Reads: JSON request from stdin: { model, messages, max_tokens, temperature }
12
- Writes: JSON lines to stdout: { event: 'token' | 'done' | 'error', ... }
13
-
14
- Each output line is a single JSON object terminated with \\n so the
15
- parent process can parse them line-by-line.
16
- """
17
- import json
18
- import sys
19
- import time
20
-
21
- def emit(obj):
22
- sys.stdout.write(json.dumps(obj) + "\\n")
23
- sys.stdout.flush()
24
-
25
- def main():
26
- try:
27
- req = json.loads(sys.stdin.read())
28
- except Exception as e:
29
- emit({"event": "error", "message": f"bad request: {e}"})
30
- sys.exit(2)
31
-
32
- model = req.get("model")
33
- messages = req.get("messages", [])
34
- max_tokens = int(req.get("max_tokens") or 256)
35
- temperature = float(req.get("temperature") or 0.7)
36
- if not model:
37
- emit({"event": "error", "message": "model required"})
38
- sys.exit(2)
39
-
40
- try:
41
- import torch
42
- from transformers import AutoTokenizer
43
- from petals import AutoDistributedModelForCausalLM
44
- except ImportError as e:
45
- emit({"event": "error", "message": f"petals not installed: {e}. Install: pip install -U petals"})
46
- sys.exit(2)
47
-
48
- emit({"event": "loading", "model": model})
49
- tokenizer = AutoTokenizer.from_pretrained(model)
50
- model_obj = AutoDistributedModelForCausalLM.from_pretrained(model, torch_dtype=torch.float16)
51
-
52
- # Compose a single prompt from messages using the tokenizer's chat template.
53
- prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
54
- input_ids = tokenizer.encode(prompt, return_tensors="pt")
55
-
56
- emit({"event": "ready", "input_tokens": int(input_ids.shape[-1])})
57
-
58
- # Petals streams via inference_session — emit one JSON line per generated token.
59
- full_text = ""
60
- t0 = time.time()
61
- with model_obj.inference_session(max_length=int(input_ids.shape[-1]) + max_tokens) as session:
62
- # Prefill
63
- outputs = model_obj.generate(
64
- input_ids,
65
- max_new_tokens=1,
66
- do_sample=temperature > 0,
67
- temperature=max(temperature, 1e-3),
68
- session=session,
69
- )
70
- new_id = int(outputs[0, -1])
71
- token_text = tokenizer.decode([new_id], skip_special_tokens=True)
72
- full_text += token_text
73
- emit({"event": "token", "text": token_text})
74
-
75
- # IPIP-0031 per-layer attribution: after the first generate the
76
- # session has resolved which DHT peers serve which transformer
77
- # blocks. Emit that mapping so the control plane can split the
78
- # CPR receipt across layer-contributing operators by block share.
79
- try:
80
- chosen = []
81
- for s in getattr(session, "chosen_servers", []) or []:
82
- chosen.append({
83
- "peer_id": str(getattr(s, "peer_id", "")),
84
- "start_block": int(getattr(s, "start", getattr(s, "start_block", 0))),
85
- "end_block": int(getattr(s, "end", getattr(s, "end_block", 0))),
86
- })
87
- if chosen:
88
- emit({"event": "routing", "peers": chosen})
89
- except Exception as e:
90
- emit({"event": "log", "warn": f"chosen_servers extract failed: {e}"})
91
-
92
- for _ in range(max_tokens - 1):
93
- outputs = model_obj.generate(
94
- outputs,
95
- max_new_tokens=1,
96
- do_sample=temperature > 0,
97
- temperature=max(temperature, 1e-3),
98
- session=session,
99
- )
100
- new_id = int(outputs[0, -1])
101
- if new_id == tokenizer.eos_token_id:
102
- break
103
- token_text = tokenizer.decode([new_id], skip_special_tokens=True)
104
- full_text += token_text
105
- emit({"event": "token", "text": token_text})
106
-
107
- emit({"event": "done", "text": full_text, "elapsed_s": round(time.time() - t0, 2)})
108
-
109
- if __name__ == "__main__":
110
- try:
111
- main()
112
- except Exception as e:
113
- emit({"event": "error", "message": str(e)})
114
- sys.exit(1)
115
- `;
@@ -1,75 +0,0 @@
1
- /**
2
- * Petals engine adapter — DISTRIBUTED inference across the volunteer
3
- * swarm (IPIP-0031, follow-up PR).
4
- *
5
- * Petals (https://github.com/bigscience-workshop/petals) splits a
6
- * Llama/Mixtral-style transformer by layer across DHT-discovered nodes.
7
- * Each node holds a few transformer blocks; activations flow through.
8
- * Designed for residential bandwidth — exactly the Infernet model.
9
- *
10
- * STATUS: scaffolding only. The flow:
11
- *
12
- * serve() — operator runs `infernet inference serve --backend petals
13
- * --model meta-llama/Llama-3.1-70B`. We:
14
- * 1. spawn `python -m petals.cli.run_server <model>`
15
- * 2. register the operator as a Petals server in our
16
- * own registry: row in providers.specs.petals_models
17
- * 3. heartbeat that the server is alive
18
- *
19
- * client() — control plane's chat router, when input_spec.distributed
20
- * is true, runs a Petals CLIENT (Python subprocess or
21
- * a thin TS wrapper) that fans out across the swarm.
22
- * Tokens stream back to the SSE handler exactly like
23
- * Ollama's stream.
24
- *
25
- * The SSE handler in apps/web/app/api/chat/stream/[jobId]/route.js
26
- * already has the typed events plumbing — when distributed===true,
27
- * it would call into routePetalsRequest below instead of subscribing
28
- * to job_events.
29
- *
30
- * To-do for the follow-up PR:
31
- * - bundle a thin Python venv with Petals at infernet setup time
32
- * - add `infernet inference serve --backend petals` as a top-level command
33
- * - register petals_models on heartbeat so the dashboard can show
34
- * "this model is served by 8 nodes via Petals · ~12 tok/s" badges
35
- * - wire the SSE handler's distributed branch through routePetalsRequest
36
- * - per-token receipts to operators contributing layers (CPR / IPIP-0007)
37
- */
38
-
39
- import { spawn } from "node:child_process";
40
-
41
- const DEFAULT_DHT_PREFIX = "/petals/v3-public";
42
-
43
- /**
44
- * Start a Petals server contributing this node's VRAM to the swarm.
45
- * Returns a child handle the daemon can monitor + restart.
46
- */
47
- export function startPetalsServer({ model, numBlocks, port = 31330, prefix = DEFAULT_DHT_PREFIX }) {
48
- const args = ["-m", "petals.cli.run_server", model];
49
- if (numBlocks) args.push("--num_blocks", String(numBlocks));
50
- if (port) args.push("--port", String(port));
51
- if (prefix) args.push("--initial_peers", prefix);
52
- const child = spawn("python3", args, { stdio: ["ignore", "pipe", "pipe"] });
53
- return child;
54
- }
55
-
56
- /**
57
- * Run a single inference through a Petals client and stream tokens.
58
- * Returns an async iterable of { type: 'token' | 'done' | 'error', data }.
59
- *
60
- * Currently a stub — wires up the subprocess but doesn't translate the
61
- * Python output into typed events. Follow-up PR: pip install
62
- * `petals` in our control-plane container and call from a Python
63
- * worker, OR use a JS-native bridge once one exists.
64
- */
65
- export async function* routePetalsRequest({ model, messages, maxTokens = 256, temperature = 0.7 }) {
66
- yield {
67
- type: "error",
68
- data: {
69
- message:
70
- `Distributed inference via Petals is in scaffolding (IPIP-0031). ` +
71
- `For now, this request needs to fall back to a single-node provider. ` +
72
- `Track progress at https://github.com/infernetprotocol/infernet-protocol/issues`
73
- }
74
- };
75
- }
@@ -1,130 +0,0 @@
1
- /**
2
- * Configuration settings for Infernet Protocol
3
- */
4
-
5
- // Load environment variables from .env file
6
- import dotenv from 'dotenv';
7
- dotenv.config();
8
-
9
- export default {
10
- // Supabase configuration (self-hosted via `supabase start` or cloud)
11
- supabase: {
12
- url: process.env.SUPABASE_URL || 'http://127.0.0.1:54321',
13
- serviceRoleKey: process.env.SUPABASE_SERVICE_ROLE_KEY,
14
- schema: process.env.SUPABASE_SCHEMA || 'public'
15
- },
16
-
17
- // Server configuration
18
- server: {
19
- port: parseInt(process.env.PORT || '3000', 10),
20
- host: process.env.HOST || '0.0.0.0',
21
- apiPrefix: process.env.API_PREFIX || '/api/v1',
22
- corsOrigins: (process.env.CORS_ORIGINS || '*').split(',')
23
- },
24
-
25
- // Node configuration
26
- node: {
27
- type: process.env.NODE_TYPE || 'provider', // provider, client, or aggregator
28
- id: process.env.NODE_ID || `node_${Date.now()}`,
29
- discoveryInterval: parseInt(process.env.DISCOVERY_INTERVAL || '60000', 10), // milliseconds
30
- bootstrapNodes: (process.env.BOOTSTRAP_NODES || '').split(',').filter(Boolean),
31
- maxConnections: parseInt(process.env.MAX_CONNECTIONS || '100', 10),
32
- heartbeatInterval: parseInt(process.env.HEARTBEAT_INTERVAL || '30000', 10), // milliseconds
33
- enableRemoteSeeding: process.env.ENABLE_REMOTE_SEEDING === 'true' || true, // Enable seeding from remote P2P instance
34
- remoteNodeUrl: process.env.REMOTE_NODE_URL || 'https://infernetprotocol.com/nodes' // URL to fetch remote nodes from
35
- },
36
-
37
- // WebSocket configuration
38
- websocket: {
39
- port: parseInt(process.env.WS_PORT || '3001', 10),
40
- host: process.env.WS_HOST || '0.0.0.0',
41
- path: process.env.WS_PATH || '/ws',
42
- heartbeatInterval: parseInt(process.env.WS_HEARTBEAT_INTERVAL || '30000', 10),
43
- reconnectInterval: parseInt(process.env.WS_RECONNECT_INTERVAL || '5000', 10),
44
- maxReconnectAttempts: parseInt(process.env.WS_MAX_RECONNECT_ATTEMPTS || '10', 10)
45
- },
46
-
47
- // DHT configuration
48
- dht: {
49
- bootstrapNodes: (process.env.DHT_BOOTSTRAP_NODES || '').split(',').filter(Boolean),
50
- kBucketSize: parseInt(process.env.DHT_K_BUCKET_SIZE || '20', 10),
51
- refreshInterval: parseInt(process.env.DHT_REFRESH_INTERVAL || '3600000', 10), // 1 hour
52
- staleTimeout: parseInt(process.env.DHT_STALE_TIMEOUT || '900000', 10) // 15 minutes
53
- },
54
-
55
- // Docker configuration for task execution
56
- docker: {
57
- socketPath: process.env.DOCKER_SOCKET || '/var/run/docker.sock',
58
- defaultImage: process.env.DEFAULT_DOCKER_IMAGE || 'ubuntu:latest',
59
- networkMode: process.env.DOCKER_NETWORK_MODE || 'bridge',
60
- cpuLimit: process.env.DOCKER_CPU_LIMIT,
61
- memoryLimit: process.env.DOCKER_MEMORY_LIMIT,
62
- gpuOptions: process.env.DOCKER_GPU_OPTIONS
63
- },
64
-
65
- // NOSTR configuration for decentralized identity
66
- nostr: {
67
- privateKey: process.env.NOSTR_PRIVATE_KEY,
68
- publicKey: process.env.NOSTR_PUBLIC_KEY,
69
- relays: (process.env.NOSTR_RELAYS || 'wss://relay.damus.io,wss://relay.nostr.info').split(','),
70
- enableNIP05: process.env.NOSTR_ENABLE_NIP05 === 'true' || false,
71
- nip05Domain: process.env.NOSTR_NIP05_DOMAIN,
72
- nip05Identifier: process.env.NOSTR_NIP05_IDENTIFIER
73
- },
74
-
75
- // Payment configuration
76
- payment: {
77
- methods: (process.env.PAYMENT_METHODS || 'lightning,polygon').split(','),
78
- minimumAmount: parseFloat(process.env.PAYMENT_MINIMUM_AMOUNT || '0.0001'),
79
- escrowEnabled: process.env.PAYMENT_ESCROW_ENABLED === 'true' || false,
80
- lightning: {
81
- nodeUrl: process.env.LIGHTNING_NODE_URL,
82
- macaroon: process.env.LIGHTNING_MACAROON,
83
- certificatePath: process.env.LIGHTNING_CERTIFICATE_PATH,
84
- invoiceExpirySeconds: parseInt(process.env.LIGHTNING_INVOICE_EXPIRY || '3600', 10)
85
- },
86
- polygon: {
87
- rpcUrl: process.env.POLYGON_RPC_URL,
88
- privateKey: process.env.POLYGON_PRIVATE_KEY,
89
- contractAddress: process.env.POLYGON_CONTRACT_ADDRESS,
90
- gasLimit: parseInt(process.env.POLYGON_GAS_LIMIT || '250000', 10)
91
- }
92
- },
93
-
94
- // Security configuration
95
- security: {
96
- encryptionKey: process.env.ENCRYPTION_KEY,
97
- jwtSecret: process.env.JWT_SECRET,
98
- jwtExpiryHours: parseInt(process.env.JWT_EXPIRY_HOURS || '24', 10),
99
- useSecureEnclave: process.env.USE_SECURE_ENCLAVE === 'true' || false,
100
- rateLimitRequests: parseInt(process.env.RATE_LIMIT_REQUESTS || '100', 10),
101
- rateLimitWindowMs: parseInt(process.env.RATE_LIMIT_WINDOW_MS || '900000', 10) // 15 minutes
102
- },
103
-
104
- // Logging configuration
105
- logging: {
106
- level: process.env.LOG_LEVEL || 'info',
107
- format: process.env.LOG_FORMAT || 'json',
108
- file: process.env.LOG_FILE || './logs/infernet.log',
109
- maxSize: parseInt(process.env.LOG_MAX_SIZE || '10485760', 10), // 10MB
110
- maxFiles: parseInt(process.env.LOG_MAX_FILES || '5', 10),
111
- console: process.env.LOG_TO_CONSOLE !== 'false'
112
- },
113
-
114
- // Content delivery configuration
115
- contentDelivery: {
116
- storageDir: process.env.CONTENT_STORAGE_DIR || './storage',
117
- maxContentSize: parseInt(process.env.MAX_CONTENT_SIZE || '10737418240', 10), // 10GB default
118
- tempDir: process.env.TEMP_DIR || './tmp',
119
- cleanupInterval: parseInt(process.env.CLEANUP_INTERVAL || '3600000', 10), // 1 hour
120
- contentExpiryHours: parseInt(process.env.CONTENT_EXPIRY_HOURS || '24', 10)
121
- },
122
-
123
- // Job execution configuration
124
- jobs: {
125
- maxConcurrent: parseInt(process.env.MAX_CONCURRENT_JOBS || '5', 10),
126
- timeout: parseInt(process.env.JOB_TIMEOUT || '3600000', 10), // 1 hour default
127
- retryLimit: parseInt(process.env.JOB_RETRY_LIMIT || '3', 10),
128
- retryDelay: parseInt(process.env.JOB_RETRY_DELAY || '60000', 10) // 1 minute
129
- }
130
- };