@fugood/buttress-server 2.25.0-beta.64 → 2.25.0-beta.73

package/README.md

@@ -163,7 +163,7 @@ Most ggml-llm `[generators.model]` keys can also live in `[runtime]` as defaults
 | `no_extra_bufts`             | boolean                       | Disable extra compute buffer types                                     |
 | `cpu_mask`, `cpu_strict`     | string / boolean              | CPU affinity (advanced)                                                |
 | `devices`                    | string[]                      | Restrict to specific GGML devices                                      |
-| Speculative keys             | various                       | `speculative`, `spec_type`, `spec_draft_n_max/n_min/p_min/p_split`     |
+| Speculative keys             | various                       | `speculative`, `spec_type`, `spec_draft_n_max/n_min/p_min/p_split`, plus draft-model GPU/cache settings |
 
 ### `[autodiscover]`
 
@@ -257,14 +257,23 @@ Loads a GGUF LLM. Runtime keys above can be overridden per-generator under `[gen
 
 **Speculative decoding**
 
-| Key                  | Type   | Notes                                              |
-| -------------------- | ------ | -------------------------------------------------- |
-| `speculative`        | string | Draft model identifier                             |
-| `spec_type`          | string | Strategy (backend-defined)                         |
-| `spec_draft_n_max`   | int    | Max drafted tokens per step                        |
-| `spec_draft_n_min`   | int    | Min drafted tokens                                 |
-| `spec_draft_p_min`   | number | Min acceptance probability                         |
-| `spec_draft_p_split` | number | Split threshold                                    |
+`model_draft` may be a direct URL/path string or a table with the same
+`repo_id`/`filename`/`url`/`local_path` model keys described above. Local draft paths require
+`allow_local_file = true`. When `download = true`, Buttress pre-downloads both target and draft
+models and includes both in its memory plan.
+
+| Key                        | Type            | Notes                                                        |
+| -------------------------- | --------------- | ------------------------------------------------------------ |
+| `model_draft`              | string \| table | Optional separate GGUF draft model                           |
+| `speculative`              | bool \| string \| table | Enable speculative decoding and optionally select a strategy |
+| `spec_type`                | string          | Strategy, such as `"draft-mtp"`                              |
+| `spec_draft_n_max`         | int             | Max drafted tokens per step                                  |
+| `spec_draft_n_min`         | int             | Min drafted tokens                                           |
+| `spec_draft_p_min`         | number          | Min acceptance probability                                   |
+| `spec_draft_p_split`       | number          | Split threshold                                              |
+| `spec_draft_n_gpu_layers`  | int             | Draft-model layers offloaded to GPU (`-1` auto, `-2` all)    |
+| `spec_draft_cache_type_k`  | string          | Draft-model K-cache dtype                                    |
+| `spec_draft_cache_type_v`  | string          | Draft-model V-cache dtype                                    |
 
 **Example**
 
@@ -281,6 +290,19 @@ n_ctx = 12800
 download = true
 ```
 
+A separate draft model can be configured inline:
+
+```toml
+[generators.model]
+repo_id = "org/target-model-GGUF"
+model_draft = { repo_id = "org/draft-model-GGUF", filename = "draft-q8_0.gguf" }
+speculative = { type = "draft-mtp" }
+spec_draft_n_max = 4
+spec_draft_n_gpu_layers = -1
+spec_draft_cache_type_k = "f16"
+spec_draft_cache_type_v = "f16"
+```
+
 ---
 
 ### `ggml-stt` (whisper.cpp via `@fugood/whisper.node`)

package/config/sample.toml

@@ -73,6 +73,15 @@ quantization = "mxfp4"
 download = true
 n_ctx = 12800 # Max: 131072
 
+# Optional separate draft model for speculative decoding. Buttress includes the
+# draft model in pre-downloads and memory planning.
+# model_draft = { repo_id = "org/draft-model-GGUF", filename = "draft-q8_0.gguf" }
+# speculative = { type = "draft-mtp" }
+# spec_draft_n_max = 4
+# spec_draft_n_gpu_layers = -1
+# spec_draft_cache_type_k = "f16"
+# spec_draft_cache_type_v = "f16"
+
 [[generators]]
 type = "ggml-llm"
 [generators.backend]

package/lib/autodiscover/index.d.ts

@@ -0,0 +1,20 @@
+import type { AutodiscoverConfig } from '../types';
+import type { GetServerInfoFn } from './types';
+import { type AnnounceSigner } from './udp';
+export type { GetServerInfoFn } from './types';
+export { signEnvelope, buildAnnounceSigner, type AnnounceSigner } from './sign';
+/**
+ * Autodiscover service that manages discovery transports.
+ * Currently supports UDP announcements/responses.
+ * HTTP discovery is handled by the info route.
+ */
+export declare class AutodiscoverService {
+    private config;
+    private getServerInfo;
+    private signer;
+    private transports;
+    private started;
+    constructor(config: AutodiscoverConfig, getServerInfo: GetServerInfoFn, signer: AnnounceSigner | null);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+}

package/lib/autodiscover/sign.d.ts

@@ -0,0 +1,10 @@
+import crypto from 'node:crypto';
+import type { WorkspaceState } from '../utils/workspaceState';
+import { type IDiscoveryProtocol } from './types';
+export interface AnnounceSigner {
+    kid: string;
+    privateKey: crypto.KeyObject;
+}
+export declare const canonicalBytes: (t: 'ANNOUNCE' | 'RESPONSE', d: unknown, ts: number) => Buffer;
+export declare const signEnvelope: (signer: AnnounceSigner | null, t: 'ANNOUNCE' | 'RESPONSE', d: IDiscoveryProtocol['d']) => IDiscoveryProtocol | null;
+export declare const buildAnnounceSigner: (workspaceState: WorkspaceState) => AnnounceSigner | null;

package/lib/autodiscover/types.d.ts

@@ -0,0 +1,32 @@
+import type { ServerInfo } from '../types';
+export declare const PROTOCOL_VERSION = "2.0";
+export declare const DEFAULT_PORT = 8089;
+export interface ITransport {
+    name: string;
+    start(): Promise<void>;
+    stop(): Promise<void>;
+}
+interface DiscoveryAnnounce {
+    info: Partial<ServerInfo>;
+}
+export interface DiscoveryRequest {
+    id: string;
+    filters?: {
+        generators?: string[];
+        min_version?: string;
+    };
+}
+interface DiscoveryResponse {
+    request_id: string;
+    info: Partial<ServerInfo>;
+}
+export interface IDiscoveryProtocol {
+    t: 'ANNOUNCE' | 'QUERY' | 'RESPONSE';
+    v: string;
+    d: DiscoveryAnnounce | DiscoveryRequest | DiscoveryResponse;
+    ts?: number;
+    kid?: string;
+    sig?: string;
+}
+export type GetServerInfoFn = () => ServerInfo;
+export {};

package/lib/autodiscover/udp.d.ts

@@ -0,0 +1,22 @@
+import type { AutodiscoverConfig } from '../types';
+import { ITransport, GetServerInfoFn } from './types';
+import { type AnnounceSigner } from './sign';
+export type { AnnounceSigner } from './sign';
+export declare class UdpTransport implements ITransport {
+    name: string;
+    private receiver;
+    private senders;
+    private announcementTimer;
+    private config;
+    private getServerInfo;
+    private port;
+    private signer;
+    constructor(config: AutodiscoverConfig['udp'], getServerInfo: GetServerInfoFn, signer: AnnounceSigner | null);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    private bindReceiver;
+    private createSenders;
+    private handleMessage;
+    private sendAnnouncement;
+    private sendResponse;
+}

package/lib/cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export default _default;
+declare function _default(): Promise<void>;

package/lib/index.d.ts

@@ -0,0 +1,29 @@
+import type { AnyElysia } from 'elysia';
+import * as backendCore from '@fugood/buttress-backend-core';
+import { AutodiscoverService } from './autodiscover';
+import type { Config } from './types';
+export { startModelDownload } from '@fugood/buttress-backend-core';
+export { processConfig } from './utils/config';
+export declare const checkForUpdates: () => Promise<string | null>;
+export declare const compareVersions: (current: string, latest: string) => boolean;
+export declare const logUpdateMessage: (latestVersion: string) => void;
+export declare const checkAndNotifyUpdates: () => Promise<void>;
+export type Backend = typeof backendCore;
+export interface StartServerOptions {
+    backend?: Backend;
+    router?: AnyElysia;
+    config: Config;
+    enableOpenAICompat?: boolean;
+    enableAnthropicMessages?: boolean;
+}
+export declare const createServer: ({ backend, router, config, enableOpenAICompat, enableAnthropicMessages, }: StartServerOptions) => Promise<{
+    app: AnyElysia;
+    config: Config;
+}>;
+export declare const startServer: ({ backend, router, config, enableOpenAICompat, enableAnthropicMessages, }: StartServerOptions) => Promise<{
+    app: AnyElysia;
+    port: number;
+    openaiEnabled: boolean;
+    anthropicMessagesEnabled: boolean;
+    autoDiscover: AutodiscoverService | null;
+}>;