cortico 0.1.0

package/src/providers/llamacpp/catalog.ts

@@ -0,0 +1,247 @@
+/**
+ * llama-server's own model endpoints. In router mode `GET /models` lists everything it can
+ * serve with a status per model, `POST /models` pulls one from HuggingFace into `LLAMA_CACHE`,
+ * and `/models/load` / `/models/unload` move one in and out of memory (unload also cancels a
+ * download). A single-model server answers the listing without statuses.
+ */
+import type { CatalogModel } from '../openai-responses-compat/native.ts';
+import { EventDecoder } from '../transport/response-http.ts';
+
+export type RouterModelStatus = 'loaded' | 'loading' | 'unloaded' | 'downloading' | 'sleeping' | 'failed' | 'unknown';
+
+export interface RouterModel {
+  id: string;
+  status: RouterModelStatus;
+  path: string | null;
+  /** From `architecture.input_modalities`; null when the server does not state it. */
+  inputModalities: string[] | null;
+  /** Bytes done / total across the files of a running download. */
+  progress: { done: number; total: number } | null;
+}
+
+export interface RouterEndpoint {
+  baseUrl: string;
+  apiKey?: string;
+}
+
+/** `/props` re-probe interval; a reloaded model may run with a different `-c`. */
+const PROPS_REFRESH_MS = 60_000;
+
+export class RouterCatalog {
+  private readonly fetchImpl: typeof fetch;
+  private readonly refreshMs: number;
+  private last: RouterModel[] = [];
+  private readonly windows = new Map<string, { value: number | undefined; probedAt: number; inFlight: Promise<void> | null }>();
+  /** Byte progress per downloading model, fed by `/models/sse`; the listing itself carries none. */
+  private readonly progress = new Map<string, { done: number; total: number }>();
+  private sse: AbortController | null = null;
+
+  constructor(
+    private readonly endpoint: () => RouterEndpoint,
+    opts: { fetchImpl?: typeof fetch; refreshMs?: number } = {},
+  ) {
+    this.fetchImpl = opts.fetchImpl ?? fetch;
+    this.refreshMs = opts.refreshMs ?? PROPS_REFRESH_MS;
+  }
+
+  private origin(): string {
+    return new URL(this.endpoint().baseUrl).origin;
+  }
+
+  private headers(): Record<string, string> {
+    const { apiKey } = this.endpoint();
+    return { 'Content-Type': 'application/json', ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}) };
+  }
+
+  private async post(path: string, body: Record<string, unknown>): Promise<void> {
+    const res = await this.fetchImpl(`${this.origin()}${path}`, {
+      method: 'POST',
+      headers: this.headers(),
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(15_000),
+    });
+    if (!res.ok) throw new Error(`POST ${path} ${res.status}: ${(await res.text()).slice(0, 300)}`);
+  }
+
+  async list(reload = false): Promise<RouterModel[]> {
+    const res = await this.fetchImpl(`${this.origin()}/models${reload ? '?reload=1' : ''}`, {
+      headers: this.headers(),
+      signal: AbortSignal.timeout(15_000),
+    });
+    if (!res.ok) throw new Error(`GET /models ${res.status}: ${(await res.text()).slice(0, 300)}`);
+    const json = (await res.json()) as { data?: unknown };
+    if (!Array.isArray(json.data)) throw new Error('GET /models returned no data array');
+    const models: RouterModel[] = [];
+    for (const row of json.data) {
+      if (!row || typeof row !== 'object' || typeof (row as { id?: unknown }).id !== 'string') continue;
+      const model = normalizeModel(row as Record<string, unknown>);
+      if (model.status === 'downloading') {
+        model.progress ??= this.progress.get(model.id) ?? null;
+        this.watchDownloads();
+      }
+      models.push(model);
+    }
+    this.last = models.sort((a, b) => a.id.localeCompare(b.id));
+    return this.last;
+  }
+
+  /** Triggers a HuggingFace pull; `list()` then shows the model as `downloading` with byte progress. */
+  async download(model: string): Promise<void> {
+    await this.post('/models', { model });
+    this.watchDownloads();
+  }
+
+  /** Stops the progress subscription (tests and shutdown); it also ends itself when nothing downloads. */
+  stop(): void {
+    this.sse?.abort();
+    this.sse = null;
+  }
+
+  /**
+   * One `/models/sse` subscription while a download runs: `download_progress` frames carry
+   * per-file byte counts, `download_finished` / `download_failed` end the model's entry, and
+   * the connection closes once no model is downloading.
+   */
+  private watchDownloads(): void {
+    if (this.sse) return;
+    const controller = new AbortController();
+    this.sse = controller;
+    void (async () => {
+      try {
+        const res = await this.fetchImpl(`${this.origin()}/models/sse`, { headers: this.headers(), signal: controller.signal });
+        if (!res.ok || !res.body) return;
+        const decoder = new EventDecoder();
+        const text = new TextDecoder();
+        const reader = res.body.getReader();
+        for (;;) {
+          const { value, done } = await reader.read();
+          if (done) break;
+          for (const raw of decoder.feed(text.decode(value, { stream: true }))) this.onDownloadEvent(raw);
+          if (this.progress.size === 0 && !this.last.some((model) => model.status === 'downloading')) break;
+        }
+      } catch {
+        // Aborted or unreachable: the next listing restarts the subscription if a download is still running.
+      } finally {
+        if (this.sse === controller) this.sse = null;
+        controller.abort();
+      }
+    })();
+  }
+
+  private onDownloadEvent(raw: string): void {
+    let frame: { model?: unknown; event?: unknown; data?: unknown };
+    try {
+      frame = JSON.parse(raw) as typeof frame;
+    } catch {
+      return;
+    }
+    if (typeof frame.model !== 'string') return;
+    if (frame.event === 'download_progress' && frame.data && typeof frame.data === 'object') {
+      let done = 0;
+      let total = 0;
+      for (const file of Object.values(frame.data as Record<string, { done?: unknown; total?: unknown }>)) {
+        if (typeof file?.done === 'number') done += file.done;
+        if (typeof file?.total === 'number') total += file.total;
+      }
+      this.progress.set(frame.model, { done, total });
+      const listed = this.last.find((model) => model.id === frame.model);
+      if (listed) listed.progress = { done, total };
+    } else if (frame.event === 'download_finished' || frame.event === 'download_failed' || frame.event === 'model_remove') {
+      this.progress.delete(frame.model);
+    }
+  }
+
+  load(model: string): Promise<void> {
+    return this.post('/models/load', { model });
+  }
+
+  /** Unloads a loaded model, or cancels its download. */
+  unload(model: string): Promise<void> {
+    return this.post('/models/unload', { model });
+  }
+
+  /** The `ProviderInstance.listModels` view: ids, plus the window when already probed. */
+  async listModels(): Promise<CatalogModel[]> {
+    const models = await this.list();
+    return models.map((model) => {
+      const window = this.windows.get(model.id)?.value;
+      return window ? { id: model.id, contextWindow: window } : { id: model.id };
+    });
+  }
+
+  /** Last listing's answer on images for this model; undefined when never listed or not stated. */
+  acceptsImage(model: string): boolean | undefined {
+    const found = this.last.find((row) => row.id === model);
+    if (!found?.inputModalities) return undefined;
+    return found.inputModalities.includes('image');
+  }
+
+  /**
+   * `default_generation_settings.n_ctx` from `/props?model=`: the per-slot context the server
+   * enforces. Never blocks; refreshes in the background and keeps the last value while the
+   * server is away.
+   */
+  contextWindow(model: string): number | undefined {
+    const entry = this.windows.get(model) ?? { value: undefined, probedAt: 0, inFlight: null };
+    this.windows.set(model, entry);
+    if (!entry.inFlight && Date.now() - entry.probedAt >= this.refreshMs) {
+      entry.inFlight = this.probeProps(model)
+        .then((value) => {
+          if (value !== undefined) entry.value = value;
+        })
+        .finally(() => {
+          entry.inFlight = null;
+          entry.probedAt = Date.now();
+        });
+    }
+    return entry.value;
+  }
+
+  /** Resolves once the probe started by the last lookup has settled (tests). */
+  settled(model: string): Promise<void> {
+    return this.windows.get(model)?.inFlight ?? Promise.resolve();
+  }
+
+  private async probeProps(model: string): Promise<number | undefined> {
+    try {
+      const res = await this.fetchImpl(`${this.origin()}/props?model=${encodeURIComponent(model)}`, {
+        headers: this.headers(),
+        signal: AbortSignal.timeout(2000),
+      });
+      if (!res.ok) return undefined;
+      const json = (await res.json()) as { default_generation_settings?: { n_ctx?: unknown } };
+      const n = json.default_generation_settings?.n_ctx;
+      return typeof n === 'number' && Number.isInteger(n) && n > 0 ? n : undefined;
+    } catch {
+      return undefined;
+    }
+  }
+}
+
+function normalizeModel(row: Record<string, unknown>): RouterModel {
+  const status = row.status as Record<string, unknown> | undefined;
+  const value = typeof status?.value === 'string' ? status.value : null;
+  let progress: RouterModel['progress'] = null;
+  if (status?.progress && typeof status.progress === 'object' && value === 'downloading') {
+    let done = 0;
+    let total = 0;
+    for (const file of Object.values(status.progress as Record<string, { done?: unknown; total?: unknown }>)) {
+      if (typeof file?.done === 'number') done += file.done;
+      if (typeof file?.total === 'number') total += file.total;
+    }
+    progress = { done, total };
+  }
+  const architecture = row.architecture as { input_modalities?: unknown } | undefined;
+  const modalities = Array.isArray(architecture?.input_modalities)
+    ? architecture.input_modalities.filter((item): item is string => typeof item === 'string')
+    : null;
+  return {
+    id: row.id as string,
+    status: status?.failed === true ? 'failed'
+      : value === 'loaded' || value === 'loading' || value === 'unloaded' || value === 'downloading' || value === 'sleeping' ? value
+      : 'unknown',
+    path: typeof row.path === 'string' ? row.path : null,
+    inputModalities: modalities,
+    progress,
+  };
+}

package/src/providers/llamacpp/config.ts

@@ -0,0 +1,36 @@
+import type { ConfigGroup, ConfigProperty } from '../../core/config-schema.ts';
+import type { Language } from '../../core/language.ts';
+import type { LLMProviderEntry } from '../../core/types.ts';
+import { backendChoices, llamacppOptions } from './options.ts';
+import { panel } from './strings.ts';
+
+export function runtimeConfig(name: string, entry: LLMProviderEntry, language: Language): ConfigGroup[] {
+  if (!llamacppOptions(entry).runtime) return [];
+  const S = panel[language];
+  const group = (id: string, title: string, properties: Record<string, ConfigProperty>): ConfigGroup => ({
+    id: `llm.llamacpp.${name}.${id}`,
+    owner: 'provider:llamacpp',
+    schema: {
+      type: 'object', title,
+      properties: Object.fromEntries(Object.entries(properties).map(([path, property]) => [
+        `providers.${name}.options.${path}`, { ...property, 'x-hot': true },
+      ])),
+    },
+  });
+  return [
+    group('runtime', S.runtimeSection, {
+      'runtime.release': { type: 'string', title: S.release, description: S.releaseHint },
+      'runtime.backend': { type: 'string', title: S.backend, enum: backendChoices() },
+      'runtime.runtimeDir': {
+        type: 'string', title: S.runtimeDir, description: S.runtimeDirHint, 'x-path': { kind: 'directory' },
+      },
+    }),
+    group('launch', S.launchSection, {
+      'launch.contextSize': { type: 'integer', title: S.contextSize, minimum: 1, description: S.launchSectionDesc },
+      'launch.nGpuLayers': { type: 'integer', title: S.nGpuLayers, minimum: 0 },
+      'launch.parallel': { type: 'integer', title: S.parallel, minimum: 1 },
+      'launch.extraArgs': { type: 'string', title: S.extraArgs, description: S.extraArgsHint },
+      autoStart: { type: 'boolean', title: S.autoStart },
+    }),
+  ];
+}

package/src/providers/llamacpp/console/client.ts

@@ -0,0 +1,6 @@
+import type { ConsoleClientBundle } from '../../../web/shared/client-panel.ts';
+import { modelsPanel } from './models-panel.ts';
+import { runtimePanel } from './runtime-panel.ts';
+
+// The endpoint table (`builtin: 'llm-settings'`) is the console's own; only these two ship here.
+export default { panels: { runtime: runtimePanel, models: modelsPanel } } satisfies ConsoleClientBundle;

package/src/providers/llamacpp/console/models-panel.ts

@@ -0,0 +1,120 @@
+/**
+ * Model section of one endpoint: what llama-server knows, and pulling more.
+ * The endpoint comes from `ctx.scope.instance`.
+ */
+import type { ConsolePanel, ConsolePanelContext } from '../../../web/shared/client-panel.ts';
+import type { ModelsState } from './server.ts';
+import { panel } from '../strings.ts';
+
+const POLL_MS = 2_000;
+
+function bytes(n: number): string {
+  if (n >= 1024 ** 3) return `${(n / 1024 ** 3).toFixed(2)} GB`;
+  if (n >= 1024 ** 2) return `${(n / 1024 ** 2).toFixed(1)} MB`;
+  return `${Math.round(n / 1024)} KB`;
+}
+
+export const modelsPanel: ConsolePanel = {
+  mount: async (ctx: ConsolePanelContext) => {
+    const { ui, root } = ctx;
+    const S = ctx.language === 'en' ? panel.en : panel.zh;
+    const name = ctx.scope.instance;
+    const card = ui.sheet({ title: S.modelsTitle });
+    const message = ui.msgline();
+    root.append(card.el, message);
+    /** The pull input survives a re-render: the operator types while the poll redraws the card. */
+    let draft = '';
+    let lastSnapshot = '';
+
+    async function act(method: string, extra: Record<string, unknown> = {}): Promise<void> {
+      message.textContent = '';
+      try {
+        await ctx.invoke(method, [{ name, ...extra }]);
+      } catch (error) {
+        message.textContent = String(error);
+      }
+      await load(true);
+    }
+
+    function render(state: ModelsState, body: HTMLElement): void {
+      body.append(ui.kv([
+        { k: S.cacheDir, v: state.cacheDir },
+        { k: S.localDir, v: state.localModelsDir },
+      ]));
+      if (!state.reachable) {
+        body.append(ui.placeholder(S.serverDown));
+        return;
+      }
+      const bar = ui.rowbar();
+      const pull = ui.input({
+        placeholder: S.pullPlaceholder,
+        cls: 'mono',
+        value: draft,
+        onInput: (value) => { draft = value; },
+        onCommit: (value) => void submit(value),
+      });
+      const submit = async (value: string): Promise<void> => {
+        const model = value.trim();
+        if (!model) return;
+        draft = '';
+        await act('pull', { model });
+      };
+      bar.append(
+        pull,
+        ui.button(S.pull, { variant: 'primary', onClick: () => void submit(pull.value) }),
+        ui.button(S.reload, { onClick: () => void act('reload') }),
+      );
+      body.append(ui.field(S.pull, bar));
+      const table = ui.table({ head: [S.modelId, S.modelStatus, S.modality, S.path, S.actions] });
+      if (state.models.length === 0) table.clear(S.noModels);
+      for (const model of state.models) {
+        const status = ui.h('div');
+        status.append(ui.pill(S.modelStatusLabel[model.status] ?? model.status,
+          model.status === 'loaded' ? 'on' : model.status === 'failed' ? 'off' : 'plain'));
+        if (model.status === 'downloading' && model.progress) {
+          const progress = ui.progress({
+            value: model.progress.done,
+            max: model.progress.total || 1,
+            format: (value, max) => (model.progress?.total ? `${bytes(value)} / ${bytes(max)}` : bytes(value)),
+          });
+          status.append(progress.el);
+        }
+        const actions = ui.h('div');
+        if (model.status === 'downloading')
+          actions.append(ui.button(S.cancel, { size: 'sm', onClick: () => void act('cancel', { model: model.id }) }));
+        else if (model.status === 'loaded' || model.status === 'sleeping')
+          actions.append(ui.button(S.unload, { size: 'sm', onClick: () => void act('unload', { model: model.id }) }));
+        else if (model.status === 'unloaded' || model.status === 'failed')
+          actions.append(ui.button(S.load, { size: 'sm', onClick: () => void act('load', { model: model.id }) }));
+        table.addRow([
+          { text: model.id, cls: 'mono' },
+          status,
+          model.inputModalities ? model.inputModalities.join(' + ') : '',
+          { text: model.path ?? '', cls: 'mono' },
+          actions,
+        ]);
+      }
+      body.append(table.el);
+    }
+
+    async function load(force = false): Promise<void> {
+      let state: ModelsState;
+      try {
+        state = await ctx.invoke<ModelsState>('state', [{ name }]);
+      } catch (error) {
+        message.textContent = String(error);
+        return;
+      }
+      if (ctx.signal.aborted) return;
+      const snapshot = JSON.stringify(state);
+      const focused = card.el.ownerDocument.activeElement;
+      if (!force && (snapshot === lastSnapshot || (focused !== null && card.el.contains(focused)))) return;
+      lastSnapshot = snapshot;
+      card.body.replaceChildren();
+      render(state, card.body);
+    }
+
+    await load(true);
+    ctx.interval(() => void load(), POLL_MS);
+  },
+};

package/src/providers/llamacpp/console/runtime-panel.ts

@@ -0,0 +1,173 @@
+/**
+ * Runtime section of one endpoint: the binary, its launch parameters and the llama-server process.
+ * The endpoint comes from `ctx.scope.instance`; edits are written on change.
+ */
+import type { ConsolePanel, ConsolePanelContext } from '../../../web/shared/client-panel.ts';
+import type { ConfigValues } from '../../../core/config-schema.ts';
+import { configField } from '../../../web/client/features/config/view.ts';
+import type { RuntimePanelState } from './server.ts';
+import { panel } from '../strings.ts';
+
+type Row = RuntimePanelState;
+
+const POLL_MS = 2_000;
+
+function bytes(n: number): string {
+  if (n >= 1024 ** 3) return `${(n / 1024 ** 3).toFixed(2)} GB`;
+  if (n >= 1024 ** 2) return `${(n / 1024 ** 2).toFixed(1)} MB`;
+  return `${Math.round(n / 1024)} KB`;
+}
+
+export const runtimePanel: ConsolePanel = {
+  mount: async (ctx: ConsolePanelContext) => {
+    const { ui, root } = ctx;
+    const S = ctx.language === 'en' ? panel.en : panel.zh;
+    const name = ctx.scope.instance;
+    const card = ui.sheet({ title: S.runtimeTitle });
+    const message = ui.msgline();
+    root.append(card.el, message);
+    let busy = false;
+    let lastSnapshot = '';
+
+    async function act(method: string, extra: Record<string, unknown> = {}): Promise<void> {
+      if (busy) return;
+      busy = true;
+      message.textContent = '';
+      try {
+        await ctx.invoke(method, [{ name, ...extra }]);
+      } catch (error) {
+        message.textContent = String(error);
+      } finally {
+        busy = false;
+      }
+      await load(true);
+    }
+
+    async function saveConfig(groupId: string, values: ConfigValues): Promise<void> {
+      if (busy) return;
+      busy = true;
+      message.textContent = '';
+      try {
+        await ctx.setConfig(groupId, values);
+      } catch (error) {
+        message.textContent = String(error);
+      } finally {
+        busy = false;
+      }
+      await load(true);
+    }
+
+    function renderConfig(row: Row, index: number, body: HTMLElement): void {
+      const { group, values } = row.config[index];
+      body.append(ui.section(group.schema.title));
+      for (const [path, property] of Object.entries(group.schema.properties)) {
+        const field = configField(ui, property, values[path], () => {
+          if (field.read) void saveConfig(group.id, { [path]: field.read() });
+        }, ctx.signal);
+        field.node.setAttribute('aria-label', property.title);
+        body.append(ui.field(property.title, field.node));
+        if (property.description) body.append(ui.msgline(property.description));
+      }
+    }
+
+    function renderUnmanaged(row: Row, body: HTMLElement): void {
+      body.append(ui.msgline(S.managedOff));
+      const bar = ui.rowbar();
+      const backend = ui.select({ options: row.backendChoices });
+      bar.append(ui.field(S.backend, backend), ui.button(S.enable, {
+        variant: 'primary',
+        onClick: () => void act('enable', { backend: backend.value }),
+      }));
+      body.append(bar);
+    }
+
+    function renderManaged(row: Row, body: HTMLElement): void {
+      const install = row.install;
+      const installPill = install.phase === 'installed' ? ui.pill(S.installed, 'on')
+        : install.phase === 'downloading' ? ui.pill(S.downloading, 'plain')
+        : install.phase === 'extracting' ? ui.pill(S.extracting, 'plain')
+        : ui.pill(S.absent, 'off');
+
+      renderConfig(row, 0, body);
+      body.append(ui.kv([{ k: S.installStatus, v: installPill }]));
+      if (row.smartAppControl === 1) body.append(ui.msgline(S.sacWarning, true));
+      if (install.phase === 'downloading' || install.phase === 'extracting') {
+        const progress = ui.progress({
+          label: `${install.phase === 'downloading' ? S.downloading : S.extracting} ${install.file ?? ''}`,
+          value: install.done,
+          max: install.total ?? 1,
+          format: (value, max) => (install.total ? `${bytes(value)} / ${bytes(max)}` : bytes(value)),
+        });
+        body.append(progress.el);
+      }
+      if (install.detail) body.append(ui.msgline(install.detail, true));
+      const installBar = ui.rowbar();
+      if (!row.own) {
+        const button = ui.button(install.phase === 'installed' ? S.reinstall : S.install, {
+          variant: install.phase === 'installed' ? 'plain' : 'primary',
+          onClick: () => void act('install'),
+        });
+        button.disabled = !row.supported || install.phase === 'downloading' || install.phase === 'extracting';
+        installBar.append(button);
+      }
+      installBar.append(ui.h('span', 'grow'), ui.button(S.disable, { onClick: () => void act('disable') }));
+      body.append(installBar);
+
+      renderConfig(row, 1, body);
+
+      body.append(ui.section(S.serverSection, S.serverSectionDesc));
+      const server = row.server;
+      if (server) {
+        const endpointRow = ui.rowbar();
+        const epText = ui.h('span', 'mono', server.baseUrl);
+        const epPill = ui.pill(server.reachable ? S.reachable : S.unreachable, server.reachable ? 'on' : 'off');
+        const copyBtn = ui.copyButton(server.baseUrl, { size: 'sm' });
+        endpointRow.append(epText, epPill, copyBtn);
+
+        body.append(ui.kv([
+          { k: S.status, v: ui.pill(S.phase[server.phase] ?? server.phase, server.phase === 'running' ? 'on' : server.phase === 'error' ? 'off' : 'plain') },
+          { k: S.endpoint, v: endpointRow },
+          { k: S.pid, v: server.pid === null ? '' : String(server.pid) },
+        ]));
+        if (server.configurationPending) body.append(ui.msgline(S.pendingNote));
+        if (server.detail) body.append(ui.msgline(server.detail, server.phase === 'error'));
+      }
+      const serverBar = ui.rowbar();
+      const start = ui.button(S.start, { variant: 'primary', onClick: () => void act('start') });
+      start.disabled = install.phase !== 'installed' || server?.phase === 'running' || server?.phase === 'starting';
+      const stop = ui.button(S.stop, { onClick: () => void act('stop') });
+      stop.disabled = !server || server.phase === 'stopped';
+      serverBar.append(start, stop);
+      body.append(serverBar);
+      if (install.phase !== 'installed') {
+        body.append(ui.msgline(S.installRequired));
+      }
+    }
+
+    /** The poll redraws the card, so it stands down while the operator is in one of its fields. */
+    function editing(): boolean {
+      const focused = card.el.ownerDocument.activeElement;
+      return focused !== null && card.el.contains(focused);
+    }
+
+    async function load(force = false): Promise<void> {
+      let row: Row;
+      try {
+        row = await ctx.invoke<Row>('state', [{ name }]);
+      } catch (error) {
+        message.textContent = String(error);
+        return;
+      }
+      if (ctx.signal.aborted) return;
+      const snapshot = JSON.stringify(row);
+      if (!force && (snapshot === lastSnapshot || editing())) return;
+      lastSnapshot = snapshot;
+      card.body.replaceChildren();
+      if (row.managed) renderManaged(row, card.body);
+      else renderUnmanaged(row, card.body);
+    }
+
+    await load(true);
+    ctx.interval(() => void load(), POLL_MS);
+  },
+};