@whatcanirun/cli 0.1.0

package/README.md

@@ -0,0 +1,91 @@
+# @whatcanirun/cli
+
+Standardized local LLM inference benchmarks.
+
+## Install
+
+```bash
+bun install
+```
+
+## Usage
+
+```bash
+# Run a benchmark
+whatcanirun run --model ./models/llama-3.2-1b.gguf --runtime llama.cpp
+
+# Run with options
+whatcanirun run \
+  --model ./models/llama-3.2-1b.gguf \
+  --runtime llama.cpp \
+  --scenario chat_long_v1 \
+  --quant q4_k_m \
+  --trials 10 \
+  --warmups 3
+
+# Run without uploading
+whatcanirun run --model ./model.gguf --runtime mlx --no-submit
+
+# Upload a previously saved bundle
+whatcanirun submit ./bundles/bundle-abc123.zip
+
+# Validate a bundle
+whatcanirun validate ./bundles/bundle-abc123.zip
+
+# Inspect device, runtime, or model
+whatcanirun show device
+whatcanirun show runtime llama.cpp
+whatcanirun show model ./models/llama-3.2-1b.gguf
+```
+
+The short alias `wcir` is also available.
+
+## Supported Runtimes
+
+| Runtime    | Flag         |
+| ---------- | ------------ |
+| llama.cpp  | `llama.cpp`  |
+| MLX        | `mlx`        |
+| vLLM       | `vllm`       |
+
+## Scenarios
+
+| ID               | Description              |
+| ---------------- | ------------------------ |
+| `chat_short_v1`  | Short chat completion    |
+| `chat_long_v1`   | Long chat completion     |
+
+## Canonical Runs
+
+A run is considered **canonical** when all of these hold:
+
+- `batch_size = 1`
+- `temperature = 0`
+- `top_p = 1`
+- `trials >= 5`
+- `warmups >= 2`
+
+## Build
+
+```bash
+# Bundle for Bun
+bun run build
+
+# Compile to standalone binary
+bun run build:bin
+```
+
+## Lint & Format
+
+```bash
+bun run lint
+bunx prettier --check .
+bunx prettier --write .
+```
+
+## Development
+
+```bash
+bun run dev          # Runs src/cli.ts directly
+bun test             # Run tests
+```

package/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@whatcanirun/cli",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "Standardized local LLM inference benchmarks",
+  "bin": {
+    "whatcanirun": "./src/cli.ts",
+    "wcir": "./src/cli.ts"
+  },
+  "scripts": {
+    "dev": "bun run src/cli.ts",
+    "build": "bun build src/cli.ts --outdir dist --target bun",
+    "build:bin": "bun build src/cli.ts --compile --outfile dist/whatcanirun",
+    "test": "bun test",
+    "lint": "eslint"
+  },
+  "dependencies": {
+    "@whatcanirun/shared": "workspace:*",
+    "citty": "^0.1.6",
+    "smol-toml": "^1.3.1"
+  },
+  "devDependencies": {
+    "@trivago/prettier-plugin-sort-imports": "^6.0.2",
+    "@types/bun": "^1.3.2",
+    "eslint": "^9",
+    "eslint-plugin-prettier": "^5.5.5",
+    "prettier": "^3",
+    "typescript": "^5",
+    "typescript-eslint": "^8"
+  },
+  "files": [
+    "src"
+  ],
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/auth/login.ts

@@ -0,0 +1,129 @@
+import { randomBytes } from 'node:crypto';
+
+import { type AuthData, saveAuth } from './token';
+
+// -----------------------------------------------------------------------------
+// Constants
+// -----------------------------------------------------------------------------
+
+const API_BASE = process.env.WCIR_API_URL || 'https://whatcani.run';
+
+// -----------------------------------------------------------------------------
+// Functions
+// -----------------------------------------------------------------------------
+
+export async function loginViaBrowser(): Promise<AuthData> {
+  const state = randomBytes(32).toString('hex');
+
+  return new Promise((resolve, reject) => {
+    const server = Bun.serve({
+      port: 0,
+      fetch(req) {
+        const url = new URL(req.url);
+
+        if (url.pathname !== '/callback') {
+          return new Response('Not found', { status: 404 });
+        }
+
+        const code = url.searchParams.get('code');
+        const returnedState = url.searchParams.get('state');
+
+        if (returnedState !== state) {
+          return new Response(page('Authentication failed', 'State mismatch. Please try again.'), {
+            headers: { 'Content-Type': 'text/html' },
+            status: 400,
+          });
+        }
+
+        if (!code) {
+          return new Response(page('Authentication failed', 'Missing authorization code.'), {
+            headers: { 'Content-Type': 'text/html' },
+            status: 400,
+          });
+        }
+
+        // Exchange the code for a CLI token server-side.
+        exchangeCode(code)
+          .then((authData) => {
+            saveAuth(authData);
+            setTimeout(() => {
+              server.stop();
+              resolve(authData);
+            }, 100);
+          })
+          .catch((err) => {
+            setTimeout(() => {
+              server.stop();
+              reject(err);
+            }, 100);
+          });
+
+        return new Response(page('Authenticated', 'You can close this tab.'), {
+          headers: { 'Content-Type': 'text/html' },
+        });
+      },
+    });
+
+    const port = server.port;
+    const loginUrl = `${API_BASE}/cli-auth?port=${port}&state=${state}`;
+
+    // Open browser (await to prevent zombie process).
+    const cmd = process.platform === 'darwin' ? 'open' : 'xdg-open';
+    const browserProc = Bun.spawn([cmd, loginUrl], { stdout: 'ignore', stderr: 'ignore' });
+    browserProc.exited.catch(() => {});
+
+    console.log(`If the browser didn't open, visit: ${loginUrl}`);
+
+    // Timeout after 5 minutes.
+    const timeout = setTimeout(() => {
+      server.stop();
+      reject(new Error('Login timed out. Please try again.'));
+    }, 300_000);
+    timeout.unref();
+  });
+}
+
+// -----------------------------------------------------------------------------
+// Helpers
+// -----------------------------------------------------------------------------
+
+async function exchangeCode(code: string): Promise<AuthData> {
+  const res = await fetch(`${API_BASE}/api/v0/auth/cli-exchange`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ code }),
+  });
+
+  if (!res.ok) {
+    const body = await res.text();
+    throw new Error(`Code exchange failed (${res.status}): ${body}`);
+  }
+
+  return (await res.json()) as AuthData;
+}
+
+function escapeHtml(s: string): string {
+  return s
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;');
+}
+
+function page(title: string, message: string): string {
+  const t = escapeHtml(title);
+  const m = escapeHtml(message);
+  return `<!DOCTYPE html>
+<html>
+<head><title>${t}</title>
+<style>
+  body { font-family: system-ui, sans-serif; display: flex; justify-content: center;
+         align-items: center; min-height: 100vh; margin: 0; background: #0a0a0a; color: #fafafa; }
+  .card { text-align: center; }
+  h1 { font-size: 1.5rem; margin-bottom: 0.5rem; }
+  p { color: #a1a1aa; }
+</style>
+</head>
+<body><div class="card"><h1>${t}</h1><p>${m}</p></div></body>
+</html>`;
+}

package/src/auth/token.ts

@@ -0,0 +1,58 @@
+import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { dirname, join } from 'node:path';
+
+// -----------------------------------------------------------------------------
+// Types
+// -----------------------------------------------------------------------------
+
+export interface AuthUser {
+  id: string;
+  name: string;
+  email: string;
+}
+
+export interface AuthData {
+  token: string;
+  user: AuthUser;
+}
+
+// -----------------------------------------------------------------------------
+// Constants
+// -----------------------------------------------------------------------------
+
+const AUTH_FILE = join(homedir(), '.whatcanirun', 'auth.json');
+
+// -----------------------------------------------------------------------------
+// Functions
+// -----------------------------------------------------------------------------
+
+export function getAuth(): AuthData | null {
+  if (!existsSync(AUTH_FILE)) return null;
+  try {
+    const data = JSON.parse(readFileSync(AUTH_FILE, 'utf-8')) as AuthData;
+    if (data.token && data.user?.id) return data;
+    return null;
+  } catch (e: unknown) {
+    console.warn(
+      `Warning: could not parse ${AUTH_FILE}: ${e instanceof Error ? e.message : String(e)}. ` +
+        'Try deleting it and running `whatcanirun auth login` again.'
+    );
+    return null;
+  }
+}
+
+export function getToken(): string | null {
+  return getAuth()?.token ?? null;
+}
+
+export function saveAuth(data: AuthData): void {
+  mkdirSync(dirname(AUTH_FILE), { recursive: true });
+  writeFileSync(AUTH_FILE, JSON.stringify(data, null, 2) + '\n', { mode: 0o600 });
+}
+
+export function clearAuth(): void {
+  if (existsSync(AUTH_FILE)) {
+    unlinkSync(AUTH_FILE);
+  }
+}

package/src/bundle/create.ts

@@ -0,0 +1,164 @@
+import {
+  type AggregateMetrics,
+  type DerivedMetrics,
+  type Manifest,
+  type Results,
+  type ResultTrial,
+  SCHEMA_VERSION,
+} from '@whatcanirun/shared';
+import { existsSync, mkdirSync } from 'fs';
+import { join, resolve } from 'path';
+
+import type { DeviceInfo } from '../device/detect';
+import { formatSysinfo } from '../device/detect';
+import type { ModelInfo } from '../model/resolve';
+import type { BenchResult, RuntimeInfo } from '../runtime/types';
+import { bundleFilename, generateBundleId } from '../utils/id';
+
+// -----------------------------------------------------------------------------
+// Types
+// -----------------------------------------------------------------------------
+
+export interface BundleOpts {
+  outputDir: string;
+  device: DeviceInfo;
+  runtimeInfo: RuntimeInfo;
+  model: ModelInfo;
+  bench: BenchResult;
+  metrics: DerivedMetrics;
+  notes?: string;
+}
+
+// -----------------------------------------------------------------------------
+// Function
+// -----------------------------------------------------------------------------
+
+export async function createBundle(opts: BundleOpts): Promise<string> {
+  const bundleId = generateBundleId({
+    runtime: opts.runtimeInfo.name,
+    model: opts.model.display_name,
+  });
+  const now = new Date();
+  const filename = bundleFilename(bundleId);
+
+  if (!existsSync(opts.outputDir)) {
+    mkdirSync(opts.outputDir, { recursive: true });
+  }
+
+  const manifest: Manifest = {
+    schema_version: SCHEMA_VERSION,
+    bundle_id: bundleId,
+    created_at: now.toISOString(),
+    canonical: false,
+    harness: {
+      version: '0.1.0',
+      git_sha: await getGitSha(),
+    },
+    device: {
+      cpu: opts.device.cpu_model,
+      gpu: opts.device.gpu_model,
+      ram_gb: opts.device.ram_gb,
+      os_name: opts.device.os_name,
+      os_version: opts.device.os_version,
+    },
+    runtime: {
+      name: opts.runtimeInfo.name,
+      version: opts.runtimeInfo.version,
+      build_flags: opts.runtimeInfo.build_flags,
+    },
+    model: {
+      display_name: opts.model.display_name,
+      format: opts.model.format,
+      artifact_sha256: opts.model.artifact_sha256,
+      source: opts.model.source,
+      file_size_bytes: opts.model.file_size_bytes,
+      parameters: opts.model.parameters,
+      quant: opts.model.quant ?? undefined,
+      architecture: opts.model.architecture,
+    },
+    context_length: opts.bench.promptTokens + opts.bench.completionTokens,
+    notes: opts.notes,
+  };
+
+  const trials: ResultTrial[] = opts.bench.trials.map((t) => ({
+    input_tokens: opts.bench.promptTokens,
+    output_tokens: opts.bench.completionTokens,
+    ttft_ms: t.promptTps > 0 ? (opts.bench.promptTokens / t.promptTps) * 1000 : 0,
+    total_ms: 0,
+    decode_tps: t.generationTps,
+    weighted_tps:
+      opts.bench.promptTokens + opts.bench.completionTokens > 0
+        ? (opts.bench.promptTokens * t.promptTps + opts.bench.completionTokens * t.generationTps) /
+          (opts.bench.promptTokens + opts.bench.completionTokens)
+        : 0,
+    peak_rss_mb: Math.round(t.peakMemoryGb * 1024 * 10) / 10,
+    exit_status: 'ok',
+  }));
+
+  const aggregate: AggregateMetrics = {
+    ttft_p50_ms: opts.metrics.ttftP50Ms,
+    ttft_p95_ms: opts.metrics.ttftP95Ms,
+    decode_tps_mean: opts.metrics.decodeTpsMean,
+    weighted_tps_mean: opts.metrics.weightedTpsMean,
+    idle_rss_mb: 0,
+    peak_rss_mb: opts.metrics.peakRssMb,
+    trials_passed: opts.bench.trials.length,
+    trials_total: opts.bench.trials.length,
+  };
+
+  const results: Results = { trials, aggregate };
+
+  const sysinfo = formatSysinfo(opts.device);
+
+  // Create a temporary directory for bundle contents.
+  const tmpDir = join(opts.outputDir, `.tmp_${bundleId}`);
+  mkdirSync(tmpDir, { recursive: true });
+
+  // Write files with deterministic formatting.
+  await Bun.write(join(tmpDir, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n');
+  await Bun.write(join(tmpDir, 'results.json'), JSON.stringify(results, null, 2) + '\n');
+  await Bun.write(join(tmpDir, 'sysinfo.txt'), sysinfo + '\n');
+
+  // Create deterministic zip using system zip command.
+  const outputPath = resolve(opts.outputDir, filename);
+  const zipProc = Bun.spawn(
+    ['zip', '-rX', outputPath, 'manifest.json', 'results.json', 'sysinfo.txt'],
+    {
+      cwd: tmpDir,
+      stdout: 'ignore',
+      stderr: 'pipe',
+    }
+  );
+  const zipCode = await zipProc.exited;
+  if (zipCode !== 0) {
+    const stderr = await new Response(zipProc.stderr).text();
+    throw new Error(`Failed to create bundle zip: ${stderr.trim() || `exit code ${zipCode}`}`);
+  }
+
+  // Clean up temp dir.
+  const rmProc = Bun.spawn(['rm', '-rf', tmpDir], {
+    stdout: 'ignore',
+    stderr: 'ignore',
+  });
+  await rmProc.exited;
+
+  return outputPath;
+}
+
+// -----------------------------------------------------------------------------
+// Helpers
+// -----------------------------------------------------------------------------
+
+async function getGitSha(): Promise<string> {
+  try {
+    const proc = Bun.spawn(['git', 'rev-parse', '--short', 'HEAD'], {
+      stdout: 'pipe',
+      stderr: 'ignore',
+    });
+    const sha = (await new Response(proc.stdout).text()).trim();
+    await proc.exited;
+    return sha || 'unknown';
+  } catch {
+    return 'unknown';
+  }
+}

package/src/bundle/validate.ts

@@ -0,0 +1,82 @@
+import { validateManifest, validateResults } from '@whatcanirun/shared';
+import { mkdtempSync, rmSync } from 'fs';
+import { tmpdir } from 'os';
+import { join } from 'path';
+
+// -----------------------------------------------------------------------------
+// Types
+// -----------------------------------------------------------------------------
+
+export interface ValidationResult {
+  valid: boolean;
+  errors: string[];
+}
+
+// -----------------------------------------------------------------------------
+// Function
+// -----------------------------------------------------------------------------
+
+export async function validateBundle(bundlePath: string): Promise<ValidationResult> {
+  const errors: string[] = [];
+
+  // Extract to temp directory.
+  const tmpDir = mkdtempSync(join(tmpdir(), 'whatcanirun-validate-'));
+
+  try {
+    const proc = Bun.spawn(['unzip', '-o', bundlePath, '-d', tmpDir], {
+      stdout: 'ignore',
+      stderr: 'pipe',
+    });
+    const code = await proc.exited;
+    if (code !== 0) {
+      const stderr = await new Response(proc.stderr).text();
+      errors.push(`Failed to extract bundle: ${stderr.trim()}`);
+      return { valid: false, errors };
+    }
+
+    // Check required files.
+    const requiredFiles = ['manifest.json', 'results.json', 'sysinfo.txt'];
+
+    for (const file of requiredFiles) {
+      const f = Bun.file(join(tmpDir, file));
+      if (!(await f.exists())) {
+        errors.push(`Missing required file: ${file}`);
+      }
+    }
+
+    if (errors.length > 0) {
+      return { valid: false, errors };
+    }
+
+    // Validate manifest.
+    let manifest: unknown;
+    try {
+      manifest = JSON.parse(await Bun.file(join(tmpDir, 'manifest.json')).text());
+    } catch (e: unknown) {
+      errors.push(`Invalid manifest.json: ${e instanceof Error ? e.message : String(e)}`);
+      return { valid: false, errors };
+    }
+    errors.push(...validateManifest(manifest));
+
+    // Validate results
+    let results: unknown;
+    try {
+      results = JSON.parse(await Bun.file(join(tmpDir, 'results.json')).text());
+    } catch (e: unknown) {
+      errors.push(`Invalid results.json: ${e instanceof Error ? e.message : String(e)}`);
+      return { valid: false, errors };
+    }
+    errors.push(...validateResults(results));
+
+    // Check artifact hash presence
+    const m = manifest as Record<string, unknown>;
+    const model = m.model as Record<string, unknown> | undefined;
+    if (!model?.artifact_sha256) {
+      errors.push('Missing model `artifact_sha256`.');
+    }
+
+    return { valid: errors.length === 0, errors };
+  } finally {
+    rmSync(tmpDir, { recursive: true, force: true });
+  }
+}

package/src/cli.ts

@@ -0,0 +1,23 @@
+#!/usr/bin/env bun
+import { defineCommand, runMain } from 'citty';
+
+import { auth, run, show, submit, update, validate, version } from './commands';
+
+const main = defineCommand({
+  meta: {
+    name: 'whatcanirun',
+    version: '0.1.0',
+    description: 'Standardized local LLM inference benchmarks',
+  },
+  subCommands: {
+    auth,
+    run,
+    show,
+    submit,
+    update,
+    validate,
+    version,
+  },
+});
+
+runMain(main);

package/src/commands/auth.ts

@@ -0,0 +1,75 @@
+import { defineCommand } from 'citty';
+
+import { loginViaBrowser } from '../auth/login';
+import { clearAuth, getAuth } from '../auth/token';
+import * as log from '../utils/log';
+
+const login = defineCommand({
+  meta: {
+    name: 'login',
+    description: 'Authenticate with whatcani.run',
+  },
+  async run() {
+    const existing = getAuth();
+    if (existing) {
+      log.info(`Already logged in as ${existing.user.name} (${existing.user.email}).`);
+      log.info('Run `whatcanirun auth logout` first to switch accounts.');
+      return;
+    }
+
+    log.info('Opening browser to sign in...');
+    try {
+      const auth = await loginViaBrowser();
+      log.blank();
+      log.success(`Logged in as ${auth.user.name} (${auth.user.email}).`);
+    } catch (e: unknown) {
+      log.error(e instanceof Error ? e.message : String(e));
+      process.exit(1);
+    }
+  },
+});
+
+const logout = defineCommand({
+  meta: {
+    name: 'logout',
+    description: 'Remove stored credentials',
+  },
+  run() {
+    const existing = getAuth();
+    if (!existing) {
+      log.info('Not logged in.');
+      return;
+    }
+    clearAuth();
+    log.success('Logged out.');
+  },
+});
+
+const status = defineCommand({
+  meta: {
+    name: 'status',
+    description: 'Show current authentication status',
+  },
+  run() {
+    const auth = getAuth();
+    if (auth) {
+      log.label('Logged in as', `${auth.user.name} (${auth.user.email})`);
+    } else {
+      log.info('Not logged in. Run `whatcanirun auth login` to authenticate.');
+    }
+  },
+});
+
+const command = defineCommand({
+  meta: {
+    name: 'auth',
+    description: 'Manage authentication',
+  },
+  subCommands: {
+    login,
+    logout,
+    status,
+  },
+});
+
+export default command;

package/src/commands/index.ts

@@ -0,0 +1,7 @@
+export { default as auth } from './auth';
+export { default as run } from './run';
+export { default as show } from './show';
+export { default as submit } from './submit';
+export { default as update } from './update';
+export { default as validate } from './validate';
+export { default as version } from './version';