cortico 0.1.0

package/src/providers/llamacpp/console/server.ts

@@ -0,0 +1,153 @@
+/**
+ * Named-endpoint operations for the runtime and model sections of the endpoint page.
+ * Runtime operations manage installation, launch configuration and the process; model operations
+ * call llama-server /models*.
+ */
+import type { ConsolePageContribution } from '../../../web/shared/console-protocol.ts';
+import type { ProviderConsoleHost } from '../../console/types.ts';
+import type { RouterCatalog, RouterModel } from '../catalog.ts';
+import { LAUNCH_DEFAULTS, PINNED_RELEASE, backendChoices, defaultBackend, llamacppOptions, type LaunchOptions } from '../options.ts';
+import type { LlamaRuntime } from '../runtime.ts';
+import type { RuntimeState } from '../runtime.ts';
+import { runtimeConfig } from '../config.ts';
+import { getByPath, type ConfigGroup, type ConfigValues } from '../../../core/config-schema.ts';
+import { text } from '../strings.ts';
+
+interface Control {
+  runtime: LlamaRuntime;
+  catalog: RouterCatalog;
+}
+
+export interface ModelsState {
+  name: string;
+  reachable: boolean;
+  cacheDir: string;
+  localModelsDir: string;
+  models: RouterModel[];
+}
+
+export type RuntimePanelState = RuntimeState & {
+  name: string;
+  config: Array<{ group: ConfigGroup; values: ConfigValues }>;
+};
+
+/** The launch fields the panel edits, each optional and applied over the stored values. */
+type LaunchPatch = Partial<Record<keyof LaunchOptions, unknown>>;
+
+export function llamacppConsole(host: ProviderConsoleHost): Partial<ConsolePageContribution> {
+  const S = text(host.language);
+  const control = (name: string): Control => host.instance(name).control as Control;
+  const body = (args: unknown[]): Record<string, unknown> => {
+    const [raw] = args;
+    if (!raw || typeof raw !== 'object' || Array.isArray(raw)) throw new Error(S.bodyRequired);
+    const value = raw as Record<string, unknown>;
+    if (typeof value.name !== 'string') throw new Error(S.instanceNameRequired);
+    return value;
+  };
+  const entryOf = (name: string) => {
+    const found = host.entries().find((entry) => entry.name === name);
+    if (!found) throw new Error(S.instanceNameRequired);
+    return found.entry;
+  };
+  return {
+    config: [],
+    panels: [
+      { id: 'runtime', title: S.runtimePanel, description: S.runtimePanelDescription, slot: 'instance' },
+      { id: 'models', title: S.modelsPanel, description: S.modelsPanelDescription, slot: 'instance' },
+    ],
+    invoke: async (panel, method, args) => {
+      if (panel === 'runtime') {
+        const value = body(args);
+        const name = value.name as string;
+        if (method === 'state') {
+          const entry = entryOf(name);
+          const prefix = `providers.${name}.`;
+          const config = runtimeConfig(name, entry, host.language).map((group) => ({
+            group,
+            values: Object.fromEntries(Object.keys(group.schema.properties).map((path) => [
+              path, getByPath(entry as unknown as Record<string, unknown>, path.slice(prefix.length)) ?? '',
+            ])) as ConfigValues,
+          }));
+          return { name, ...(await control(name).runtime.state(host.language)), config } satisfies RuntimePanelState;
+        }
+        if (method === 'enable') {
+          const entry = entryOf(name);
+          const options = llamacppOptions(entry);
+          const backend = typeof value.backend === 'string' && backendChoices().includes(value.backend) ? value.backend : defaultBackend();
+          host.save(name, {
+            ...entry,
+            options: {
+              ...entry.options,
+              runtime: { release: typeof value.release === 'string' && value.release.trim() ? value.release.trim() : PINNED_RELEASE, backend },
+              launch: { ...LAUNCH_DEFAULTS, ...(options.launch ?? {}) },
+            },
+          });
+          return { ok: true };
+        }
+        if (method === 'disable') {
+          await control(name).runtime.stop(host.language);
+          const entry = entryOf(name);
+          const { runtime: _runtime, launch: _launch, ...rest } = entry.options ?? {};
+          host.save(name, { ...entry, options: rest });
+          return { ok: true };
+        }
+        if (method === 'configure') {
+          const entry = entryOf(name);
+          const options = llamacppOptions(entry);
+          if (!options.runtime) throw new Error(S.notManaged);
+          const runtimeOptions = { ...options.runtime };
+          if (typeof value.release === 'string') runtimeOptions.release = value.release.trim();
+          if (typeof value.backend === 'string') runtimeOptions.backend = value.backend;
+          if (typeof value.runtimeDir === 'string') {
+            const dir = value.runtimeDir.trim();
+            if (dir) runtimeOptions.runtimeDir = dir;
+            else delete runtimeOptions.runtimeDir;
+          }
+          const launch = { ...LAUNCH_DEFAULTS, ...(options.launch ?? {}), ...((value.launch ?? {}) as LaunchPatch) };
+          host.save(name, {
+            ...entry,
+            options: {
+              ...entry.options,
+              runtime: runtimeOptions,
+              launch,
+              ...(value.autoStart === undefined ? {} : { autoStart: value.autoStart }),
+            },
+          });
+          return { ok: true };
+        }
+        if (method === 'install') {
+          await control(name).runtime.install(host.language);
+          return { ok: true };
+        }
+        if (method === 'start') return control(name).runtime.start(host.language);
+        if (method === 'stop') return control(name).runtime.stop(host.language);
+        throw new Error(S.unknownMethod);
+      }
+      if (panel === 'models') {
+        const value = body(args);
+        const name = value.name as string;
+        const { runtime, catalog } = control(name);
+        if (method === 'state') {
+          const dirs = runtime.modelDirs();
+          try {
+            return { name, reachable: true, ...dirs, models: await catalog.list() } satisfies ModelsState;
+          } catch {
+            return { name, reachable: false, ...dirs, models: [] } satisfies ModelsState;
+          }
+        }
+        if (method === 'reload') {
+          await catalog.list(true);
+          return { ok: true };
+        }
+        if (typeof value.model !== 'string' || !value.model.trim()) throw new Error(S.modelIdRequired);
+        const model = value.model.trim();
+        if (method === 'pull') await catalog.download(model);
+        else if (method === 'load') await catalog.load(model);
+        else if (method === 'unload' || method === 'cancel') await catalog.unload(model);
+        else throw new Error(S.unknownMethod);
+        return { ok: true };
+      }
+      throw new Error(S.unknownPanel);
+    },
+  };
+}

package/src/providers/llamacpp/index.ts

@@ -0,0 +1,95 @@
+import type { ProviderModule } from '../base.ts';
+import type { ReasoningTier } from '../../core/types.ts';
+import type { Language } from '../../core/language.ts';
+import { isContextOverflow } from '../transport/errors.ts';
+import { modelsRoot, runtimesRoot } from '../../paths.ts';
+import { RouterCatalog } from './catalog.ts';
+import { runtimeConfig } from './config.ts';
+import { llamacppConsole } from './console/server.ts';
+import { LlamaCppProvider } from './native.ts';
+import { backendChoices, llamacppOptions, normalizeLlamaCpp } from './options.ts';
+import { LlamaRuntime } from './runtime.ts';
+import { nullLogger } from '../../core/util.ts';
+import { text } from './strings.ts';
+
+/** Two tiers: the transport sends the template's thinking switch, effort only when the operator typed one. */
+const reasoningTiers = (language: Language): ReasoningTier[] => {
+  const S = text(language);
+  return [
+    { id: 'off', label: S.tierOff, thinking: false },
+    { id: 'on', label: S.tierOn, thinking: true },
+  ];
+};
+
+export default {
+  id: 'llamacpp',
+  title: 'llama.cpp',
+  defaultBaseUrl: 'http://127.0.0.1:8090/v1',
+  baseUrlSuggestions: ['http://127.0.0.1:8090/v1', 'http://127.0.0.1:8080/v1'],
+  normalize: normalizeLlamaCpp,
+  console: llamacppConsole,
+  config: runtimeConfig,
+  reasoningTiers: reasoningTiers('zh'),
+  localize: (language) => ({ reasoningTiers: reasoningTiers(language) }),
+  serviceTiers: [],
+  validateEntry: (entry, language) => {
+    const S = text(language);
+    const options = llamacppOptions(entry);
+    if (options.autoStart !== undefined && typeof options.autoStart !== 'boolean') throw new Error(S.autoStartBoolean);
+    if (!options.runtime) return;
+    const { runtime, launch } = options;
+    if (typeof runtime.release !== 'string' || !runtime.release.trim()) throw new Error(S.releaseRequired);
+    if (runtime.runtimeDir !== undefined && typeof runtime.runtimeDir !== 'string') throw new Error(S.runtimeDirString);
+    if (!runtime.runtimeDir && !backendChoices().includes(runtime.backend)) throw new Error(S.backendUnsupported(String(runtime.backend)));
+    if (launch) {
+      for (const key of ['contextSize', 'parallel'] as const)
+        if (!Number.isInteger(launch[key]) || launch[key] <= 0) throw new Error(S.positiveInteger(key));
+      if (!Number.isInteger(launch.nGpuLayers) || launch.nGpuLayers < 0) throw new Error(S.nonNegativeInteger('nGpuLayers'));
+      if (typeof launch.extraArgs !== 'string') throw new Error(S.extraArgsString);
+    }
+  },
+  contextOverflow: isContextOverflow,
+  availability: (name, entry, language) => {
+    if (!llamacppOptions(entry).runtime) return { ready: true };
+    const runtime = new LlamaRuntime({
+      name,
+      entry: () => entry,
+      secret: () => '',
+      log: nullLogger(),
+      roots: { runtimes: runtimesRoot(), models: modelsRoot() },
+    });
+    if (runtime.installState(language).phase === 'installed') return { ready: true };
+    return { ready: false, reason: text(language).runtimeNotInstalled };
+  },
+  create(name, entry, host) {
+    const current = host.currentEntry ?? (() => entry);
+    const apiKey = entry.secret ? host.secret(entry.secret) : undefined;
+    const catalog = new RouterCatalog(() => ({ baseUrl: current().baseUrl, apiKey }));
+    const createRuntime = () =>
+      new LlamaRuntime({
+        name,
+        entry: current,
+        secret: host.secret,
+        log: host.log,
+        roots: { runtimes: runtimesRoot(), models: modelsRoot() },
+      });
+    const runtime = host.resource?.('runtime', createRuntime) ?? createRuntime();
+    return {
+      control: { runtime, catalog },
+      listModels: () => catalog.listModels(),
+      contextWindow: (model) => catalog.contextWindow(model),
+      compatibilityKey: () => [name],
+      start: () => (llamacppOptions(current()).autoStart === true ? runtime.start() : Promise.resolve(undefined)),
+      stop: async () => {
+        catalog.stop();
+        await runtime.stop();
+      },
+      client: new LlamaCppProvider({
+        baseUrl: entry.baseUrl,
+        apiKey,
+        log: host.log,
+        media: { enabled: () => entry.multimodal === true, read: host.readBlob },
+      }),
+    };
+  },
+} satisfies ProviderModule;

package/src/providers/llamacpp/native.ts

@@ -0,0 +1,51 @@
+/**
+ * Chat Completions against llama-server. Thinking is the template's `enable_thinking` switch
+ * sent per request through `chat_template_kwargs`; templates without that variable ignore it.
+ * The server returns the chain of thought as `reasoning_content` (`--reasoning-format deepseek`),
+ * which the shared Chat assembly already maps; past thinking is not replayed.
+ */
+import type { NativeChatMessage } from '../transport/native-types.ts';
+import type { ModelSpec, ToolSchema, Logger } from '../../core/types.ts';
+import { nullLogger } from '../../core/util.ts';
+import { OpenAIHttpClient } from '../transport/chat.ts';
+import { mapTools, renderMessagesWithMedia, type CompatMediaOptions } from '../transport/history.ts';
+
+export function buildLlamaCppRequestBody(
+  spec: ModelSpec,
+  messages: NativeChatMessage[],
+  tools?: ToolSchema[],
+  media?: CompatMediaOptions,
+): Record<string, unknown> {
+  const body: Record<string, unknown> = {
+    model: spec.model,
+    messages: renderMessagesWithMedia(messages, media),
+    chat_template_kwargs: { enable_thinking: spec.thinking },
+  };
+  if (spec.thinking && spec.reasoningEffort) body.reasoning_effort = spec.reasoningEffort;
+  if (spec.temperature !== undefined) body.temperature = spec.temperature;
+  if (spec.maxTokens !== undefined) body.max_tokens = spec.maxTokens;
+  const mapped = mapTools(tools);
+  if (mapped) body.tools = mapped;
+  return body;
+}
+
+export class LlamaCppProvider extends OpenAIHttpClient {
+  private readonly apiKey?: string;
+  private readonly media?: CompatMediaOptions;
+
+  constructor(opts: { baseUrl: string; apiKey?: string; log?: Logger; media?: CompatMediaOptions }) {
+    super(opts.baseUrl, opts.log ?? nullLogger());
+    this.apiKey = opts.apiKey;
+    this.media = opts.media;
+  }
+
+  protected buildBody(spec: ModelSpec, messages: NativeChatMessage[], tools?: ToolSchema[]): Record<string, unknown> {
+    return buildLlamaCppRequestBody(spec, messages, tools, this.media);
+  }
+
+  protected headers(): Record<string, string> {
+    const headers: Record<string, string> = { 'Content-Type': 'application/json' };
+    if (this.apiKey) headers.Authorization = `Bearer ${this.apiKey}`;
+    return headers;
+  }
+}

package/src/providers/llamacpp/options.ts

@@ -0,0 +1,117 @@
+/**
+ * Endpoint options of the llamacpp module, and the upstream release table.
+ *
+ * `options.runtime` present = managed: Cortico downloads the pinned release and starts
+ * llama-server in router mode. Absent = external: the endpoint is a llama-server someone else
+ * started, and only the catalog and the probes apply.
+ */
+import type { LLMProviderEntry } from '../../core/types.ts';
+
+/** Upstream build this module was verified against. Endpoints may pin another tag. */
+export const PINNED_RELEASE = 'b10930';
+const RELEASE_BASE = 'https://github.com/ggml-org/llama.cpp/releases/download';
+
+export interface RuntimeOptions {
+  release: string;
+  backend: string;
+  /** A directory holding llama-server and its libraries; when set nothing is downloaded. */
+  runtimeDir?: string;
+}
+export interface LaunchOptions {
+  contextSize: number;
+  nGpuLayers: number;
+  parallel: number;
+  /** Whitespace-separated extra arguments, appended verbatim. */
+  extraArgs: string;
+}
+export interface LlamaCppOptions {
+  runtime?: RuntimeOptions;
+  launch?: LaunchOptions;
+  autoStart?: boolean;
+}
+
+export const LAUNCH_DEFAULTS: LaunchOptions = { contextSize: 16384, nGpuLayers: 99, parallel: 1, extraArgs: '' };
+
+export function llamacppOptions(entry: LLMProviderEntry): LlamaCppOptions {
+  return (entry.options ?? {}) as LlamaCppOptions;
+}
+
+/** Fills the release and launch defaults for managed endpoints; external ones carry no such keys. */
+export function normalizeLlamaCpp(entry: LLMProviderEntry): LLMProviderEntry {
+  const options = { ...entry.options } as LlamaCppOptions;
+  if (options.runtime && typeof options.runtime === 'object') {
+    const runtime = options.runtime as Partial<RuntimeOptions>;
+    options.runtime = { ...runtime, release: runtime.release ?? PINNED_RELEASE, backend: runtime.backend ?? defaultBackend() };
+    if (typeof options.runtime.release === 'string') options.runtime.release = options.runtime.release.trim();
+    if (typeof options.runtime.runtimeDir === 'string') options.runtime.runtimeDir = options.runtime.runtimeDir.trim();
+    if (options.runtime.runtimeDir === '') delete options.runtime.runtimeDir;
+    options.launch = { ...LAUNCH_DEFAULTS, ...(options.launch ?? {}) };
+  }
+  return { ...entry, options: options as Record<string, unknown> };
+}
+
+export interface ReleaseArchive {
+  url: string;
+  file: string;
+  format: 'zip' | 'tgz';
+  /** Leading path components to drop; the tarballs wrap everything in `llama-<tag>/`. */
+  stripComponents: number;
+}
+export interface ReleasePlan {
+  /** Directory name under the release: `win-cuda-13.3-x64`, `ubuntu-vulkan-x64`, `macos-arm64`. */
+  key: string;
+  archives: ReleaseArchive[];
+  /** Server executable name inside the directory. */
+  serverExe: string;
+}
+
+type Platform = NodeJS.Platform;
+type Arch = string;
+
+/** Backends the upstream ships a prebuilt archive for, per platform. First one is the default. */
+export function backendChoices(platform: Platform = process.platform, arch: Arch = process.arch): string[] {
+  if (platform === 'win32') {
+    if (arch === 'x64') return ['cuda-13.3', 'cuda-12.4', 'vulkan', 'cpu'];
+    if (arch === 'arm64') return ['cuda-13.4', 'cpu'];
+    return [];
+  }
+  if (platform === 'linux') {
+    if (arch === 'x64') return ['vulkan', 'rocm-10.0', 'cpu'];
+    if (arch === 'arm64') return ['vulkan', 'cpu'];
+    return [];
+  }
+  if (platform === 'darwin') return arch === 'x64' || arch === 'arm64' ? ['metal'] : [];
+  return [];
+}
+
+export function defaultBackend(platform: Platform = process.platform, arch: Arch = process.arch): string {
+  return backendChoices(platform, arch)[0] ?? 'cpu';
+}
+
+/** The archives that make up one build on one platform; null when the upstream ships none. */
+export function releasePlan(
+  release: string,
+  backend: string,
+  platform: Platform = process.platform,
+  arch: Arch = process.arch,
+): ReleasePlan | null {
+  if (!backendChoices(platform, arch).includes(backend)) return null;
+  const asset = (file: string, format: 'zip' | 'tgz', stripComponents: number): ReleaseArchive => ({
+    url: `${RELEASE_BASE}/${release}/${file}`,
+    file,
+    format,
+    stripComponents,
+  });
+  if (platform === 'win32') {
+    const key = `win-${backend}-${arch}`;
+    const archives = [asset(`llama-${release}-bin-${key}.zip`, 'zip', 0)];
+    if (backend.startsWith('cuda-')) archives.push(asset(`cudart-llama-bin-win-${backend}-${arch}.zip`, 'zip', 0));
+    return { key, archives, serverExe: 'llama-server.exe' };
+  }
+  if (platform === 'linux') {
+    const key = backend === 'cpu' ? `ubuntu-${arch}` : `ubuntu-${backend}-${arch}`;
+    return { key, archives: [asset(`llama-${release}-bin-${key}.tar.gz`, 'tgz', 1)], serverExe: 'llama-server' };
+  }
+  const key = `macos-${arch}`;
+  return { key, archives: [asset(`llama-${release}-bin-${key}.tar.gz`, 'tgz', 1)], serverExe: 'llama-server' };
+}

package/src/providers/llamacpp/runtime-store.ts

@@ -0,0 +1,139 @@
+/**
+ * Install builds under <runtimes root>/llama.cpp/<release>/<plan key>/.
+ * Download and unpack into <dir>.partial, write the completion marker, then rename to the destination.
+ * A later installation removes any remaining partial directory before starting.
+ */
+import { existsSync, mkdirSync, readFileSync, rmSync, renameSync, unlinkSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import type { Logger } from '../../core/types.ts';
+import type { Language } from '../../core/language.ts';
+import { downloadFile, extractArchive } from './archive.ts';
+import type { ReleasePlan } from './options.ts';
+import { text, type Text } from './strings.ts';
+
+export const RUNTIME_MARKER = 'cortico-runtime.json';
+const RUNTIME_ID = 'llama.cpp';
+
+export type InstallPhase = 'absent' | 'downloading' | 'extracting' | 'installed' | 'error';
+
+export interface InstallState {
+  phase: InstallPhase;
+  /** Archive being downloaded or unpacked. */
+  file: string | null;
+  done: number;
+  total: number | null;
+  detail: string | null;
+}
+
+export interface RuntimeMarker {
+  runtime: typeof RUNTIME_ID;
+  release: string;
+  key: string;
+  archives: string[];
+  installedAt: string;
+}
+
+type Detail = (S: Text) => string;
+
+export class RuntimeStore {
+  private active: { dir: string; phase: InstallPhase; file: string | null; done: number; total: number | null } | null = null;
+  private failures = new Map<string, Detail>();
+  private readonly fetchImpl: typeof fetch;
+
+  constructor(
+    private readonly root: string,
+    private readonly log: Logger,
+    fetchImpl?: typeof fetch,
+  ) {
+    this.fetchImpl = fetchImpl ?? fetch;
+  }
+
+  dir(release: string, plan: ReleasePlan): string {
+    return join(this.root, RUNTIME_ID, release, plan.key);
+  }
+
+  installed(dir: string): boolean {
+    return existsSync(join(dir, RUNTIME_MARKER));
+  }
+
+  marker(dir: string): RuntimeMarker | null {
+    try {
+      return JSON.parse(readFileSync(join(dir, RUNTIME_MARKER), 'utf8')) as RuntimeMarker;
+    } catch {
+      return null;
+    }
+  }
+
+  state(dir: string, language: Language = 'zh'): InstallState {
+    if (this.active?.dir === dir) {
+      const { phase, file, done, total } = this.active;
+      return { phase, file, done, total, detail: null };
+    }
+    const failure = this.failures.get(dir);
+    if (failure) return { phase: 'error', file: null, done: 0, total: null, detail: failure(text(language)) };
+    return { phase: this.installed(dir) ? 'installed' : 'absent', file: null, done: 0, total: null, detail: null };
+  }
+
+  /** Downloads and unpacks every archive of the plan; a second concurrent install is refused. */
+  async install(release: string, plan: ReleasePlan, language: Language = 'zh'): Promise<void> {
+    const dir = this.dir(release, plan);
+    if (this.active) throw new Error(text(language).installBusy);
+    this.failures.delete(dir);
+    const partial = `${dir}.partial`;
+    rmSync(partial, { recursive: true, force: true });
+    rmSync(dir, { recursive: true, force: true });
+    mkdirSync(partial, { recursive: true });
+    this.active = { dir, phase: 'downloading', file: null, done: 0, total: null };
+    try {
+      for (const archive of plan.archives) {
+        const file = join(partial, archive.file);
+        this.active = { dir, phase: 'downloading', file: archive.file, done: 0, total: null };
+        this.log.info(`llama.cpp 运行时下载 ${archive.url}`);
+        try {
+          await downloadFile(archive.url, file, {
+            fetchImpl: this.fetchImpl,
+            onProgress: (done, total) => {
+              if (this.active?.dir === dir) Object.assign(this.active, { done, total });
+            },
+          });
+        } catch (error) {
+          throw { detail: (S: Text) => S.downloadFailed(archive.file, message(error)) };
+        }
+        this.active = { dir, phase: 'extracting', file: archive.file, done: 0, total: null };
+        try {
+          await extractArchive(file, archive.format, partial, archive.stripComponents);
+        } catch (error) {
+          throw { detail: (S: Text) => S.extractFailed(archive.file, message(error)) };
+        }
+        unlinkSync(file);
+      }
+      const marker: RuntimeMarker = {
+        runtime: RUNTIME_ID,
+        release,
+        key: plan.key,
+        archives: plan.archives.map((archive) => archive.file),
+        installedAt: new Date().toISOString(),
+      };
+      writeFileSync(join(partial, RUNTIME_MARKER), JSON.stringify(marker, null, 2) + '\n');
+      mkdirSync(join(dir, '..'), { recursive: true });
+      renameSync(partial, dir);
+      this.log.info(`llama.cpp 运行时已安装 ${dir}`);
+    } catch (error) {
+      rmSync(partial, { recursive: true, force: true });
+      const detail: Detail = isDetail(error) ? error.detail : () => message(error);
+      this.failures.set(dir, detail);
+      this.log.warn(`llama.cpp 运行时安装失败: ${detail(text('zh'))}`);
+      throw new Error(detail(text(language)));
+    } finally {
+      this.active = null;
+    }
+  }
+}
+
+function isDetail(value: unknown): value is { detail: Detail } {
+  return typeof value === 'object' && value !== null && typeof (value as { detail?: unknown }).detail === 'function';
+}
+
+function message(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}