@bendyline/gezel-inference 0.0.0-stage → 0.1.0

package/dist/http/routes/v1-embeddings.js

@@ -0,0 +1,82 @@
+import { randomUUID } from 'node:crypto';
+import { Hono } from 'hono';
+import { bodyLimit } from 'hono/body-limit';
+import { ZodError, z } from 'zod';
+import { MAX_EMBEDDING_INPUTS, createEmbeddings } from '../openai-compat/embeddings.js';
+/**
+ * `POST /v1/embeddings` — OpenAI-compatible embeddings.
+ *
+ * `gezel:embeddings` (or the embedder's own model id) runs the on-device
+ * text embedder every Gezel host already carries for memory and search. A
+ * `<provider>:<model>` id is served by that provider's `createEmbedding`;
+ * a provider without one fails with `400 embeddings_not_supported` rather
+ * than falling back to a different model.
+ */
+const EmbeddingsRequestSchema = z.object({
+    model: z.string().min(1),
+    input: z.union([z.string(), z.array(z.string()).min(1).max(MAX_EMBEDDING_INPUTS)]),
+    encoding_format: z.enum(['float', 'base64']).optional(),
+    dimensions: z.number().int().positive().optional(),
+    user: z.string().optional(),
+});
+/** 2048 inputs of the embedder's ~8,000-character working length. */
+const MAX_BODY_BYTES = 16 * 1024 * 1024;
+function invalidRequest(code, message) {
+    return { error: { message, type: 'invalid_request_error', code } };
+}
+export function v1EmbeddingsRoutes(ctx) {
+    const app = new Hono();
+    app.use('*', bodyLimit({
+        maxSize: MAX_BODY_BYTES,
+        onError: (c) => c.json(invalidRequest('request_too_large', 'Request body is too large.'), 413),
+    }));
+    app.post('/', async (c) => {
+        // Same request-boundary handling as /v1/chat/completions: malformed
+        // JSON and schema mismatches are the caller's mistake — 400 with the
+        // OpenAI error envelope, never the internal 422/500 shapes.
+        let raw;
+        try {
+            raw = await c.req.json();
+        }
+        catch {
+            return c.json(invalidRequest('invalid_json', 'Request body is not valid JSON.'), 400);
+        }
+        let parsed;
+        try {
+            parsed = EmbeddingsRequestSchema.parse(raw);
+        }
+        catch (err) {
+            if (err instanceof ZodError) {
+                return c.json(invalidRequest('invalid_body', err.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ')), 400);
+            }
+            throw err;
+        }
+        if (parsed.encoding_format === 'base64') {
+            return c.json(invalidRequest('encoding_format_not_supported', 'base64 encoding is not supported in gezel /v1 yet.'), 400);
+        }
+        const outcome = await createEmbeddings(ctx, parsed.model, parsed.input);
+        if (!outcome.ok) {
+            const type = outcome.status >= 500 ? 'server_error' : 'invalid_request_error';
+            return c.json({ error: { message: outcome.message, type, code: outcome.code } }, outcome.status);
+        }
+        // None of these models can shorten its vectors, so a request for a
+        // different size is refused rather than answered at the wrong one.
+        const width = outcome.vectors[0]?.length;
+        if (parsed.dimensions !== undefined && width !== undefined && width !== parsed.dimensions) {
+            return c.json(invalidRequest('dimensions_not_supported', `"${parsed.model}" returns ${width}-dimensional vectors and cannot produce ${parsed.dimensions}; omit "dimensions".`), 400);
+        }
+        return c.json({
+            object: 'list',
+            data: outcome.vectors.map((embedding, index) => ({
+                object: 'embedding',
+                index,
+                embedding,
+            })),
+            model: outcome.model,
+            usage: { prompt_tokens: outcome.inputTokens, total_tokens: outcome.inputTokens },
+            // Audit identifier so callers can correlate to gezel-side logs.
+            id: `embd-${randomUUID()}`,
+        });
+    });
+    return app;
+}

package/dist/http/routes/v1-knowledge.d.ts

@@ -0,0 +1,4 @@
+import { Hono } from 'hono';
+import type { InferenceContext as ServiceContext } from '../inference-context.js';
+/** Public app surface: registered catalogs and reference passages only. */
+export declare function v1KnowledgeRoutes(ctx: Pick<ServiceContext, 'knowledge' | 'relevance'>): Hono;

package/dist/http/routes/v1-knowledge.js

@@ -0,0 +1,304 @@
+import { AppKnowledgeActionSchema, AppKnowledgeQuerySchema, AppKnowledgeRelevanceSchema, AppKnowledgeRetrievalSchema, AppKnowledgeStateSchema, } from '@bendyline/gezel/app-models';
+import { Hono } from 'hono';
+import { bodyLimit } from 'hono/body-limit';
+import { admitsUnjudgedKnowledge } from '../../knowledge/admission.js';
+/**
+ * Bundled catalogs (the Handboek, Gezel's own manual) belong to Gezel, not to
+ * connected apps. They are enabled on every install, so exposing them would
+ * hand each app a catalog the person never chose — and with it a relevance
+ * model requirement and Gezel help text in unrelated prompts.
+ */
+function appCatalog(entry) {
+    return entry.source !== 'bundled';
+}
+/** Public app surface: registered catalogs and reference passages only. */
+export function v1KnowledgeRoutes(ctx) {
+    const app = new Hono();
+    let active = 0;
+    app.use('*', bodyLimit({ maxSize: 40_000 }));
+    app.use('*', async (c, next) => {
+        if (!ctx.knowledge)
+            return c.json({
+                error: {
+                    code: 'knowledge_unavailable',
+                    message: 'Knowledge catalogs are unavailable in this service.',
+                },
+            }, 503);
+        if (active >= 4)
+            return c.json({
+                error: {
+                    code: 'rate_limited',
+                    message: 'Knowledge service is busy.',
+                },
+            }, 429);
+        active++;
+        try {
+            await next();
+        }
+        finally {
+            active--;
+        }
+    });
+    app.get('/state', async (c) => {
+        const manager = ctx.knowledge;
+        const [listed, available, relevance] = await Promise.all([
+            manager.list(),
+            manager.available(),
+            ctx.relevance.status(),
+        ]);
+        const installed = listed.filter(appCatalog);
+        const hidden = new Set(listed.filter((entry) => !appCatalog(entry)).map((entry) => entry.ref.catalogId));
+        const jobs = new Map(manager.activeInstalls().map((job) => [job.catalogId, job]));
+        const catalogs = new Map();
+        for (const entry of available) {
+            if (hidden.has(entry.id))
+                continue;
+            catalogs.set(entry.id, {
+                id: entry.id,
+                name: entry.name,
+                description: entry.description,
+                version: entry.version,
+                installedVersion: null,
+                enabled: false,
+                updateAvailable: false,
+                downloadBytes: entry.archiveBytes,
+                documents: entry.documents,
+                state: 'available',
+                percent: null,
+                message: null,
+            });
+        }
+        for (const entry of installed) {
+            const previous = catalogs.get(entry.ref.catalogId);
+            catalogs.set(entry.ref.catalogId, {
+                id: entry.ref.catalogId,
+                name: entry.name ?? entry.ref.catalogId,
+                description: entry.description ?? '',
+                version: entry.availableVersion ?? entry.ref.version,
+                installedVersion: entry.ref.version,
+                enabled: entry.enabled,
+                updateAvailable: entry.updateAvailable,
+                downloadBytes: previous?.downloadBytes ?? null,
+                documents: entry.documents ?? null,
+                state: entry.disabledReason ? 'error' : 'installed',
+                percent: null,
+                message: entry.disabledReason ?? null,
+            });
+        }
+        for (const entry of catalogs.values()) {
+            const job = jobs.get(entry.id);
+            const snapshot = manager.getJob(entry.id);
+            if (job) {
+                entry.state = 'downloading';
+                entry.percent =
+                    job.bytesTotal > 0 ? Math.min(100, (job.bytesDone / job.bytesTotal) * 100) : null;
+                entry.message = 'Downloading catalog…';
+            }
+            else if (snapshot?.error) {
+                entry.state = 'error';
+                entry.message = 'The catalog download failed. Try downloading it again.';
+            }
+        }
+        const selected = relevance.models.find((model) => model.id === relevance.modelId);
+        return c.json(AppKnowledgeStateSchema.parse({
+            catalogs: [...catalogs.values()],
+            reranker: {
+                ready: selected?.installed === true,
+                downloading: relevance.progress !== undefined,
+                message: relevance.error ??
+                    (selected?.installed
+                        ? null
+                        : 'Download the relevance model before using knowledge catalogs.'),
+            },
+        }));
+    });
+    app.post('/update', async (c) => {
+        const action = AppKnowledgeActionSchema.parse(await c.req.json());
+        const manager = ctx.knowledge;
+        if (action.action === 'prepare-reranker') {
+            const setting = await ctx.relevance.setting();
+            if (!setting.spec)
+                return c.json({
+                    error: {
+                        code: 'reranker_unavailable',
+                        message: 'No relevance model is configured.',
+                    },
+                }, 409);
+            const result = await ctx.relevance.install(setting.spec.id);
+            if (!result.started && !result.installed)
+                return c.json({
+                    error: {
+                        code: 'reranker_unavailable',
+                        message: result.reason ?? 'The relevance model could not be downloaded.',
+                    },
+                }, 409);
+        }
+        else if (action.action === 'install') {
+            const available = await manager.available();
+            if (!available.some((entry) => entry.id === action.catalogId))
+                return c.json({
+                    error: {
+                        code: 'catalog_not_found',
+                        message: 'That catalog is not available.',
+                    },
+                }, 404);
+            if (manager.activeInstalls().length >= 2)
+                return c.json({
+                    error: {
+                        code: 'rate_limited',
+                        message: 'Wait for a catalog download to finish.',
+                    },
+                }, 429);
+            manager.startInstall({
+                kind: 'catalog',
+                id: action.catalogId,
+                placement: 'user',
+            });
+        }
+        else if (action.action === 'cancel') {
+            manager.cancelJob(action.catalogId);
+        }
+        else {
+            const target = (await manager.list()).find((entry) => entry.ref.catalogId === action.catalogId);
+            const changed = target !== undefined &&
+                appCatalog(target) &&
+                (action.action === 'remove'
+                    ? await manager.remove(action.catalogId)
+                    : await manager.setEnabled(action.catalogId, action.action === 'enable'));
+            if (!changed)
+                return c.json({
+                    error: {
+                        code: 'catalog_not_found',
+                        message: 'That catalog is not installed.',
+                    },
+                }, 404);
+        }
+        return c.json({ ok: true });
+    });
+    app.get('/relevance', async (c) => {
+        const relevance = await ctx.relevance.status();
+        const selected = relevance.models.find((model) => model.id === relevance.modelId);
+        const progress = relevance.progress;
+        return c.json(AppKnowledgeRelevanceSchema.parse({
+            ready: selected?.installed === true,
+            downloading: progress !== undefined,
+            percent: progress && progress.bytesTotal > 0
+                ? Math.min(100, (progress.bytesDone / progress.bytesTotal) * 100)
+                : null,
+            downloadBytes: selected && !selected.installed && relevance.status !== 'blocked-network'
+                ? selected.approxBytes
+                : null,
+        }));
+    });
+    app.post('/retrieve', async (c) => {
+        const query = AppKnowledgeQuerySchema.parse(await c.req.json());
+        const required = query.rerank === 'required';
+        const manager = ctx.knowledge;
+        const installed = await manager.list();
+        const enabled = installed.filter((entry) => entry.enabled && appCatalog(entry));
+        if (enabled.some((entry) => !entry.mounted))
+            return c.json({
+                error: {
+                    code: 'knowledge_unavailable',
+                    message: 'An enabled knowledge catalog is unavailable.',
+                },
+            }, 409);
+        if (enabled.length === 0)
+            return c.json({ reranked: true, passages: [] });
+        const activeModel = await ctx.relevance.forSurface('search', {
+            enabled: true,
+        });
+        if (!activeModel && required)
+            return c.json({
+                error: {
+                    code: 'reranker_required',
+                    message: 'Download the relevance model in AI settings before using knowledge catalogs.',
+                },
+            }, 409);
+        c.req.raw.signal.throwIfAborted();
+        const hits = await manager.searchUnified(query.query, {
+            vector: null,
+            localModelsOnly: true,
+            maxResults: 24,
+            queryEmbedBudgetMs: 5000,
+            catalogs: enabled.map((entry) => entry.ref.catalogId),
+        });
+        c.req.raw.signal.throwIfAborted();
+        const candidates = [];
+        for (const hit of hits.slice(0, 24)) {
+            if (!hit.uri)
+                continue;
+            // Without the model, skip what the unjudged bar would drop anyway.
+            if (!activeModel && !admitsUnjudgedKnowledge(hit))
+                continue;
+            const citation = await manager.resolveCitation(hit.uri);
+            if (!citation.ok)
+                continue;
+            const text = (citation.chunk?.text ?? citation.markdown ?? '').slice(0, 6000);
+            if (!text)
+                continue;
+            candidates.push({
+                hit,
+                passage: {
+                    uri: hit.uri,
+                    title: citation.title.slice(0, 256),
+                    text,
+                    catalogId: citation.uri.catalogId,
+                    version: citation.catalogVersion,
+                },
+            });
+        }
+        const fit = (ranked) => {
+            const selected = [];
+            let remaining = query.maxCharacters;
+            for (const passage of ranked) {
+                if (!remaining || selected.length >= query.maxResults)
+                    break;
+                const text = passage.text.slice(0, remaining);
+                selected.push({ ...passage, text });
+                remaining -= text.length;
+            }
+            return selected;
+        };
+        // `auto` without a usable model: Gezel's own bar, in fused order.
+        const unjudged = () => c.json(AppKnowledgeRetrievalSchema.parse({
+            reranked: false,
+            passages: fit(candidates
+                .filter(({ hit }) => admitsUnjudgedKnowledge(hit))
+                .map(({ passage }) => passage)),
+        }));
+        if (!activeModel)
+            return unjudged();
+        if (!candidates.length)
+            return c.json({ reranked: true, passages: [] });
+        const result = await ctx.relevance.scorer.score({
+            model: activeModel.model,
+            query: query.query,
+            passages: candidates.map(({ passage }) => passage.text),
+            waitForLoad: true,
+            budgetMs: 30_000,
+        });
+        c.req.raw.signal.throwIfAborted();
+        if (result.status !== 'scored' ||
+            result.scores?.length !== candidates.length ||
+            result.scores.some((score) => score === null || !Number.isFinite(score))) {
+            if (!required)
+                return unjudged();
+            return c.json({
+                error: {
+                    code: 'reranker_required',
+                    message: 'The relevance model could not score every passage. Retry the request.',
+                },
+            }, 409);
+        }
+        const ordered = candidates
+            .map(({ passage }, index) => ({ passage, score: result.scores[index] }))
+            .filter(({ score }) => !activeModel.thresholds || score >= activeModel.thresholds.keep)
+            .sort((a, b) => b.score - a.score);
+        return c.json(AppKnowledgeRetrievalSchema.parse({
+            reranked: true,
+            passages: fit(ordered.map(({ passage }) => passage)),
+        }));
+    });
+    return app;
+}

package/dist/http/routes/v1-models-ensure.d.ts

@@ -0,0 +1,32 @@
+import { Hono } from 'hono';
+import type { InferenceContext } from '../inference-context.js';
+type EngineContext = Pick<InferenceContext, 'ensureModel'>;
+/**
+ * `/v1/models/ensure` — "make this local model downloaded and ready to
+ * serve."
+ *
+ *   POST /v1/models/ensure                    Body: `{model: "<backend>:<id>"}`.
+ *                                             Returns either
+ *                                             `{status:'ready', model_id}` or
+ *                                             `{status:'downloading',
+ *                                              model_id, job_id}`.
+ *
+ *   GET  /v1/models/ensure/:jobId              Snapshot of an in-flight or
+ *                                             recently-completed install job.
+ *                                             `events[]` carries everything
+ *                                             that's been published so far —
+ *                                             useful for callers that prefer
+ *                                             polling to SSE.
+ *
+ *   GET  /v1/models/ensure/:jobId/events       SSE: progress + terminal events
+ *                                             for an in-flight install. Each
+ *                                             frame is `data: <JSON>` where
+ *                                             the payload is the same shape
+ *                                             as the polled snapshot's
+ *                                             `events[]` entries.
+ *
+ * Auth: mounted under `bearerAuth + requireScope('openai')` like the
+ * rest of `/v1/*`.
+ */
+export declare function v1ModelsEnsureRoutes(ctx: EngineContext): Hono;
+export {};

package/dist/http/routes/v1-models-ensure.js

@@ -0,0 +1,122 @@
+import { Hono } from 'hono';
+import { streamSSE } from 'hono/streaming';
+import { z } from 'zod';
+import { KnownEnsureError } from '../../models/ensure.js';
+const EnsureRequestSchema = z.object({
+    model: z.string().min(1, 'model is required'),
+});
+/**
+ * `/v1/models/ensure` — "make this local model downloaded and ready to
+ * serve."
+ *
+ *   POST /v1/models/ensure                    Body: `{model: "<backend>:<id>"}`.
+ *                                             Returns either
+ *                                             `{status:'ready', model_id}` or
+ *                                             `{status:'downloading',
+ *                                              model_id, job_id}`.
+ *
+ *   GET  /v1/models/ensure/:jobId              Snapshot of an in-flight or
+ *                                             recently-completed install job.
+ *                                             `events[]` carries everything
+ *                                             that's been published so far —
+ *                                             useful for callers that prefer
+ *                                             polling to SSE.
+ *
+ *   GET  /v1/models/ensure/:jobId/events       SSE: progress + terminal events
+ *                                             for an in-flight install. Each
+ *                                             frame is `data: <JSON>` where
+ *                                             the payload is the same shape
+ *                                             as the polled snapshot's
+ *                                             `events[]` entries.
+ *
+ * Auth: mounted under `bearerAuth + requireScope('openai')` like the
+ * rest of `/v1/*`.
+ */
+export function v1ModelsEnsureRoutes(ctx) {
+    const app = new Hono();
+    app.post('/', async (c) => {
+        const parsed = EnsureRequestSchema.parse(await c.req.json());
+        try {
+            const result = await ctx.ensureModel.ensure(parsed.model);
+            if (result.status === 'ready') {
+                return c.json({ status: 'ready', model_id: result.modelId });
+            }
+            return c.json({
+                status: 'downloading',
+                model_id: result.modelId,
+                job_id: result.jobId,
+            }, 202);
+        }
+        catch (err) {
+            if (err instanceof KnownEnsureError) {
+                const status = err.code === 'unknown_model'
+                    ? 404
+                    : err.code === 'no_source_for_backend'
+                        ? 400
+                        : err.code === 'ambiguous_model'
+                            ? 400
+                            : 400;
+                return c.json({ error: { message: err.message, type: 'invalid_request_error', code: err.code } }, status);
+            }
+            const message = err instanceof Error ? err.message : String(err);
+            return c.json({ error: { message, type: 'server_error', code: 'ensure_failed' } }, 500);
+        }
+    });
+    app.get('/:jobId', (c) => {
+        const snapshot = ctx.ensureModel.getJob(c.req.param('jobId'));
+        if (!snapshot) {
+            return c.json({
+                error: { message: 'job not found', type: 'invalid_request_error', code: 'job_not_found' },
+            }, 404);
+        }
+        return c.json({
+            job_id: snapshot.jobId,
+            model_id: snapshot.modelId,
+            status: snapshot.status,
+            started_at: snapshot.startedAt,
+            events: snapshot.events,
+        });
+    });
+    app.get('/:jobId/events', (c) => {
+        const jobId = c.req.param('jobId');
+        const snapshot = ctx.ensureModel.getJob(jobId);
+        if (!snapshot) {
+            return c.json({
+                error: { message: 'job not found', type: 'invalid_request_error', code: 'job_not_found' },
+            }, 404);
+        }
+        return streamSSE(c, async (stream) => {
+            let closed = false;
+            // The orchestrator replays buffered events synchronously on
+            // subscribe so the SSE client sees the full history before live
+            // events. We forward each into the stream.
+            const unsubscribe = ctx.ensureModel.subscribeJob(jobId, (event) => {
+                void stream
+                    .writeSSE({ data: JSON.stringify(event) })
+                    .then(() => {
+                    if (event.type === 'done' || event.type === 'error') {
+                        closed = true;
+                    }
+                })
+                    .catch(() => {
+                    closed = true;
+                });
+            });
+            stream.onAbort(() => {
+                closed = true;
+                unsubscribe();
+            });
+            while (!closed) {
+                await stream.sleep(1000);
+                try {
+                    await stream.writeSSE({ event: 'ping', data: '' });
+                }
+                catch {
+                    closed = true;
+                }
+            }
+            unsubscribe();
+        });
+    });
+    return app;
+}

package/dist/http/routes/v1-models.d.ts

@@ -0,0 +1,8 @@
+import { Hono } from 'hono';
+import type { InferenceContext as ServiceContext } from '../inference-context.js';
+interface V1ModelsRouteOptions {
+    /** List only on-device engines plus downloadable on-device catalog entries. */
+    localOnly?: boolean;
+}
+export declare function v1ModelsRoutes(ctx: ServiceContext, options?: V1ModelsRouteOptions): Hono;
+export {};