@constructive-io/graphql-server 4.32.0 → 4.33.0

package/esm/middleware/llm-api.js

@@ -23,39 +23,10 @@
 import express, { Router } from 'express';
 import { Logger } from '@pgpmjs/logger';
 import { getPgPool } from 'pg-cache';
-import OllamaClient from '@agentic-kit/ollama';
+import { OllamaAdapter } from '@agentic-kit/ollama';
 import { ModuleConfigCache } from 'graphile-cache';
 import { getLlmEnvOptions, getAgentDiscovery, getLlmBillingConfig, } from 'graphile-llm';
 const log = new Logger('llm-api');
-/**
- * Placeholder: replace with actual provider token counts once generateWithUsage() is approved.
- * Estimates ~4 chars per token for English text.
- */
-function placeholderAmountTokens(text) {
-    return Math.ceil(text.length / 4);
-}
-/**
- * Call generate() and estimate token counts from text length.
- * When a provider-native token counting API is approved, swap the
- * estimation logic here without changing call sites.
- */
-async function callWithUsage(client, input, onChunk) {
-    const promptText = input.messages.map((m) => m.content).join(' ');
-    let content;
-    if (onChunk || input.stream) {
-        await client.generate(input, onChunk);
-        content = '';
-    }
-    else {
-        content = await client.generate(input);
-    }
-    const inputTokens = placeholderAmountTokens(promptText);
-    const outputTokens = placeholderAmountTokens(content);
-    return {
-        content,
-        usage: { input: inputTokens, output: outputTokens, totalTokens: inputTokens + outputTokens },
-    };
-}
 // ─── Helpers ────────────────────────────────────────────────────────────────
 function getPgSettings(req) {
     const settings = {};
@@ -90,12 +61,13 @@ async function withRlsClient(pool, pgSettings, fn) {
         client.release();
     }
 }
-function resolveOllamaClient() {
+function resolveOllamaAdapter() {
     const { chat } = getLlmEnvOptions();
     if (chat.provider === 'ollama') {
         return {
-            client: new OllamaClient(chat.baseUrl),
+            adapter: new OllamaAdapter(chat.baseUrl),
             model: chat.model,
+            baseUrl: chat.baseUrl,
         };
     }
     return null;
@@ -280,7 +252,7 @@ async function handleSendMessage(req, res, entityId) {
         ? await resolveBilling(pool, pgSettings, databaseId)
         : null;
     const inferenceLog = await getInferenceLogInfo(pool, dbname);
-    const ollama = resolveOllamaClient();
+    const ollama = resolveOllamaAdapter();
     if (!ollama) {
         res.status(503).json({ error: 'No LLM provider configured' });
         return;
@@ -343,7 +315,7 @@ async function handleSendMessage(req, res, entityId) {
     const shouldStream = body.stream !== false;
     const startTime = Date.now();
     if (shouldStream) {
-        // ── SSE Streaming ──────────────────────────────────────────────────
+        // ── SSE Streaming via OllamaAdapter ─────────────────────────────────
         res.writeHead(200, {
             'Content-Type': 'text/event-stream',
             'Cache-Control': 'no-cache',
@@ -352,32 +324,49 @@ async function handleSendMessage(req, res, entityId) {
         });
         const messageId = `msg_${Date.now()}`;
         try {
-            let streamedContent = '';
-            const result = await callWithUsage(ollama.client, {
-                model,
-                messages: llmMessages,
-                stream: true,
+            const systemMsg = llmMessages.find(m => m.role === 'system');
+            const nonSystem = llmMessages.filter(m => m.role !== 'system');
+            const modelDesc = ollama.adapter.createModel(model, {
+                maxOutputTokens: undefined,
+            });
+            const context = {
+                systemPrompt: systemMsg?.content,
+                messages: nonSystem.map((m) => ({
+                    role: m.role,
+                    content: m.content,
+                    timestamp: Date.now(),
+                })),
+            };
+            const stream = ollama.adapter.stream(modelDesc, context, {
                 temperature: body.temperature,
-            }, (chunk) => {
-                streamedContent += chunk;
-                const event = {
-                    id: messageId,
-                    choices: [{
-                            index: 0,
-                            delta: { content: chunk, role: 'assistant' },
-                            finish_reason: null,
-                        }],
-                    model,
-                };
-                res.write(`data: ${JSON.stringify(event)}\n\n`);
             });
-            // Streaming generate() returns void; use accumulated chunks
+            let streamedContent = '';
+            for await (const event of stream) {
+                if (event.type === 'text_delta') {
+                    streamedContent += event.delta;
+                    const sseEvent = {
+                        id: messageId,
+                        choices: [{
+                                index: 0,
+                                delta: { content: event.delta, role: 'assistant' },
+                                finish_reason: null,
+                            }],
+                        model,
+                    };
+                    res.write(`data: ${JSON.stringify(sseEvent)}\n\n`);
+                }
+            }
+            const result = await stream.result();
             const content = streamedContent;
-            const promptText = llmMessages.map(m => m.content).join(' ');
             const latencyMs = Date.now() - startTime;
-            const inputTokens = placeholderAmountTokens(promptText);
-            const outputTokens = placeholderAmountTokens(content);
-            const totalTokens = inputTokens + outputTokens;
+            const usage = {
+                input: result.usage.input,
+                output: result.usage.output,
+                reasoning: result.usage.reasoning,
+                cacheRead: result.usage.cacheRead,
+                cacheWrite: result.usage.cacheWrite,
+                totalTokens: result.usage.totalTokens,
+            };
             // Send [DONE] marker
             res.write('data: [DONE]\n\n');
             res.end();
@@ -398,10 +387,12 @@ async function handleSendMessage(req, res, entityId) {
                 });
             }
             // 7. Record billing usage (fire-and-forget)
-            if (billing && totalTokens > 0) {
-                recordUsage(pool, pgSettings, billing, entityId, meterSlug, totalTokens, {
-                    input_tokens: inputTokens,
-                    output_tokens: outputTokens,
+            if (billing && usage.totalTokens > 0) {
+                recordUsage(pool, pgSettings, billing, entityId, meterSlug, usage.totalTokens, {
+                    input_tokens: usage.input,
+                    output_tokens: usage.output,
+                    cache_read_tokens: usage.cacheRead,
+                    cache_write_tokens: usage.cacheWrite,
                     model,
                     latency_ms: latencyMs,
                     stream: true,
@@ -416,9 +407,9 @@ async function handleSendMessage(req, res, entityId) {
                     provider: 'ollama',
                     service: 'llm',
                     operation: 'chat',
-                    inputTokens,
-                    outputTokens,
-                    totalTokens,
+                    inputTokens: usage.input,
+                    outputTokens: usage.output,
+                    totalTokens: usage.totalTokens,
                     latencyMs,
                     status: 'ok',
                 }).catch(() => { });
@@ -433,17 +424,37 @@ async function handleSendMessage(req, res, entityId) {
         }
     }
     else {
-        // ── Non-streaming (batch) ──────────────────────────────────────────
-        const result = await callWithUsage(ollama.client, {
-            model,
-            messages: llmMessages,
-            stream: false,
+        // ── Non-streaming (batch) via OllamaAdapter ─────────────────────────
+        const systemMsg = llmMessages.find(m => m.role === 'system');
+        const nonSystem = llmMessages.filter(m => m.role !== 'system');
+        const modelDesc = ollama.adapter.createModel(model, {
+            maxOutputTokens: undefined,
+        });
+        const context = {
+            systemPrompt: systemMsg?.content,
+            messages: nonSystem.map((m) => ({
+                role: m.role,
+                content: m.content,
+                timestamp: Date.now(),
+            })),
+        };
+        const stream = ollama.adapter.stream(modelDesc, context, {
             temperature: body.temperature,
         });
+        const result = await stream.result();
+        const content = result.content
+            .filter((block) => block.type === 'text')
+            .map((block) => block.text)
+            .join('');
         const latencyMs = Date.now() - startTime;
-        const inputTokens = result.usage.input;
-        const outputTokens = result.usage.output;
-        const totalTokens = result.usage.totalTokens;
+        const usage = {
+            input: result.usage.input,
+            output: result.usage.output,
+            reasoning: result.usage.reasoning,
+            cacheRead: result.usage.cacheRead,
+            cacheWrite: result.usage.cacheWrite,
+            totalTokens: result.usage.totalTokens,
+        };
         // Persist assistant message with model
         await withRlsClient(pool, pgSettings, async (client) => {
             await client.query(`INSERT INTO "${msgTable.schemaName}"."${msgTable.tableName}"
@@ -452,15 +463,17 @@ async function handleSendMessage(req, res, entityId) {
                 threadId,
                 userId,
                 'assistant',
-                JSON.stringify([{ type: 'text', text: result.content }]),
+                JSON.stringify([{ type: 'text', text: content }]),
                 model,
             ]);
         });
         // Record billing usage
-        if (billing && totalTokens > 0) {
-            recordUsage(pool, pgSettings, billing, entityId, meterSlug, totalTokens, {
-                input_tokens: inputTokens,
-                output_tokens: outputTokens,
+        if (billing && usage.totalTokens > 0) {
+            recordUsage(pool, pgSettings, billing, entityId, meterSlug, usage.totalTokens, {
+                input_tokens: usage.input,
+                output_tokens: usage.output,
+                cache_read_tokens: usage.cacheRead,
+                cache_write_tokens: usage.cacheWrite,
                 model,
                 latency_ms: latencyMs,
                 stream: false,
@@ -475,9 +488,9 @@ async function handleSendMessage(req, res, entityId) {
                 provider: 'ollama',
                 service: 'llm',
                 operation: 'chat',
-                inputTokens,
-                outputTokens,
-                totalTokens,
+                inputTokens: usage.input,
+                outputTokens: usage.output,
+                totalTokens: usage.totalTokens,
                 latencyMs,
                 status: 'ok',
             }).catch(() => { });
@@ -486,14 +499,14 @@ async function handleSendMessage(req, res, entityId) {
             id: `msg_${Date.now()}`,
             choices: [{
                     index: 0,
-                    message: { role: 'assistant', content: result.content },
+                    message: { role: 'assistant', content },
                     finish_reason: 'stop',
                 }],
             model,
             usage: {
-                prompt_tokens: inputTokens,
-                completion_tokens: outputTokens,
-                total_tokens: totalTokens,
+                prompt_tokens: usage.input,
+                completion_tokens: usage.output,
+                total_tokens: usage.totalTokens,
             },
         });
     }

package/middleware/llm-api.js

@@ -54,47 +54,15 @@ var __importStar = (this && this.__importStar) || (function () {
         return result;
     };
 })();
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createLlmApiRouter = createLlmApiRouter;
 const express_1 = __importStar(require("express"));
 const logger_1 = require("@pgpmjs/logger");
 const pg_cache_1 = require("pg-cache");
-const ollama_1 = __importDefault(require("@agentic-kit/ollama"));
+const ollama_1 = require("@agentic-kit/ollama");
 const graphile_cache_1 = require("graphile-cache");
 const graphile_llm_1 = require("graphile-llm");
 const log = new logger_1.Logger('llm-api');
-/**
- * Placeholder: replace with actual provider token counts once generateWithUsage() is approved.
- * Estimates ~4 chars per token for English text.
- */
-function placeholderAmountTokens(text) {
-    return Math.ceil(text.length / 4);
-}
-/**
- * Call generate() and estimate token counts from text length.
- * When a provider-native token counting API is approved, swap the
- * estimation logic here without changing call sites.
- */
-async function callWithUsage(client, input, onChunk) {
-    const promptText = input.messages.map((m) => m.content).join(' ');
-    let content;
-    if (onChunk || input.stream) {
-        await client.generate(input, onChunk);
-        content = '';
-    }
-    else {
-        content = await client.generate(input);
-    }
-    const inputTokens = placeholderAmountTokens(promptText);
-    const outputTokens = placeholderAmountTokens(content);
-    return {
-        content,
-        usage: { input: inputTokens, output: outputTokens, totalTokens: inputTokens + outputTokens },
-    };
-}
 // ─── Helpers ────────────────────────────────────────────────────────────────
 function getPgSettings(req) {
     const settings = {};
@@ -129,12 +97,13 @@ async function withRlsClient(pool, pgSettings, fn) {
         client.release();
     }
 }
-function resolveOllamaClient() {
+function resolveOllamaAdapter() {
     const { chat } = (0, graphile_llm_1.getLlmEnvOptions)();
     if (chat.provider === 'ollama') {
         return {
-            client: new ollama_1.default(chat.baseUrl),
+            adapter: new ollama_1.OllamaAdapter(chat.baseUrl),
             model: chat.model,
+            baseUrl: chat.baseUrl,
         };
     }
     return null;
@@ -319,7 +288,7 @@ async function handleSendMessage(req, res, entityId) {
         ? await resolveBilling(pool, pgSettings, databaseId)
         : null;
     const inferenceLog = await getInferenceLogInfo(pool, dbname);
-    const ollama = resolveOllamaClient();
+    const ollama = resolveOllamaAdapter();
     if (!ollama) {
         res.status(503).json({ error: 'No LLM provider configured' });
         return;
@@ -382,7 +351,7 @@ async function handleSendMessage(req, res, entityId) {
     const shouldStream = body.stream !== false;
     const startTime = Date.now();
     if (shouldStream) {
-        // ── SSE Streaming ──────────────────────────────────────────────────
+        // ── SSE Streaming via OllamaAdapter ─────────────────────────────────
         res.writeHead(200, {
             'Content-Type': 'text/event-stream',
             'Cache-Control': 'no-cache',
@@ -391,32 +360,49 @@ async function handleSendMessage(req, res, entityId) {
         });
         const messageId = `msg_${Date.now()}`;
         try {
-            let streamedContent = '';
-            const result = await callWithUsage(ollama.client, {
-                model,
-                messages: llmMessages,
-                stream: true,
+            const systemMsg = llmMessages.find(m => m.role === 'system');
+            const nonSystem = llmMessages.filter(m => m.role !== 'system');
+            const modelDesc = ollama.adapter.createModel(model, {
+                maxOutputTokens: undefined,
+            });
+            const context = {
+                systemPrompt: systemMsg?.content,
+                messages: nonSystem.map((m) => ({
+                    role: m.role,
+                    content: m.content,
+                    timestamp: Date.now(),
+                })),
+            };
+            const stream = ollama.adapter.stream(modelDesc, context, {
                 temperature: body.temperature,
-            }, (chunk) => {
-                streamedContent += chunk;
-                const event = {
-                    id: messageId,
-                    choices: [{
-                            index: 0,
-                            delta: { content: chunk, role: 'assistant' },
-                            finish_reason: null,
-                        }],
-                    model,
-                };
-                res.write(`data: ${JSON.stringify(event)}\n\n`);
             });
-            // Streaming generate() returns void; use accumulated chunks
+            let streamedContent = '';
+            for await (const event of stream) {
+                if (event.type === 'text_delta') {
+                    streamedContent += event.delta;
+                    const sseEvent = {
+                        id: messageId,
+                        choices: [{
+                                index: 0,
+                                delta: { content: event.delta, role: 'assistant' },
+                                finish_reason: null,
+                            }],
+                        model,
+                    };
+                    res.write(`data: ${JSON.stringify(sseEvent)}\n\n`);
+                }
+            }
+            const result = await stream.result();
             const content = streamedContent;
-            const promptText = llmMessages.map(m => m.content).join(' ');
             const latencyMs = Date.now() - startTime;
-            const inputTokens = placeholderAmountTokens(promptText);
-            const outputTokens = placeholderAmountTokens(content);
-            const totalTokens = inputTokens + outputTokens;
+            const usage = {
+                input: result.usage.input,
+                output: result.usage.output,
+                reasoning: result.usage.reasoning,
+                cacheRead: result.usage.cacheRead,
+                cacheWrite: result.usage.cacheWrite,
+                totalTokens: result.usage.totalTokens,
+            };
             // Send [DONE] marker
             res.write('data: [DONE]\n\n');
             res.end();
@@ -437,10 +423,12 @@ async function handleSendMessage(req, res, entityId) {
                 });
             }
             // 7. Record billing usage (fire-and-forget)
-            if (billing && totalTokens > 0) {
-                recordUsage(pool, pgSettings, billing, entityId, meterSlug, totalTokens, {
-                    input_tokens: inputTokens,
-                    output_tokens: outputTokens,
+            if (billing && usage.totalTokens > 0) {
+                recordUsage(pool, pgSettings, billing, entityId, meterSlug, usage.totalTokens, {
+                    input_tokens: usage.input,
+                    output_tokens: usage.output,
+                    cache_read_tokens: usage.cacheRead,
+                    cache_write_tokens: usage.cacheWrite,
                     model,
                     latency_ms: latencyMs,
                     stream: true,
@@ -455,9 +443,9 @@ async function handleSendMessage(req, res, entityId) {
                     provider: 'ollama',
                     service: 'llm',
                     operation: 'chat',
-                    inputTokens,
-                    outputTokens,
-                    totalTokens,
+                    inputTokens: usage.input,
+                    outputTokens: usage.output,
+                    totalTokens: usage.totalTokens,
                     latencyMs,
                     status: 'ok',
                 }).catch(() => { });
@@ -472,17 +460,37 @@ async function handleSendMessage(req, res, entityId) {
         }
     }
     else {
-        // ── Non-streaming (batch) ──────────────────────────────────────────
-        const result = await callWithUsage(ollama.client, {
-            model,
-            messages: llmMessages,
-            stream: false,
+        // ── Non-streaming (batch) via OllamaAdapter ─────────────────────────
+        const systemMsg = llmMessages.find(m => m.role === 'system');
+        const nonSystem = llmMessages.filter(m => m.role !== 'system');
+        const modelDesc = ollama.adapter.createModel(model, {
+            maxOutputTokens: undefined,
+        });
+        const context = {
+            systemPrompt: systemMsg?.content,
+            messages: nonSystem.map((m) => ({
+                role: m.role,
+                content: m.content,
+                timestamp: Date.now(),
+            })),
+        };
+        const stream = ollama.adapter.stream(modelDesc, context, {
             temperature: body.temperature,
         });
+        const result = await stream.result();
+        const content = result.content
+            .filter((block) => block.type === 'text')
+            .map((block) => block.text)
+            .join('');
         const latencyMs = Date.now() - startTime;
-        const inputTokens = result.usage.input;
-        const outputTokens = result.usage.output;
-        const totalTokens = result.usage.totalTokens;
+        const usage = {
+            input: result.usage.input,
+            output: result.usage.output,
+            reasoning: result.usage.reasoning,
+            cacheRead: result.usage.cacheRead,
+            cacheWrite: result.usage.cacheWrite,
+            totalTokens: result.usage.totalTokens,
+        };
         // Persist assistant message with model
         await withRlsClient(pool, pgSettings, async (client) => {
             await client.query(`INSERT INTO "${msgTable.schemaName}"."${msgTable.tableName}"
@@ -491,15 +499,17 @@ async function handleSendMessage(req, res, entityId) {
                 threadId,
                 userId,
                 'assistant',
-                JSON.stringify([{ type: 'text', text: result.content }]),
+                JSON.stringify([{ type: 'text', text: content }]),
                 model,
             ]);
         });
         // Record billing usage
-        if (billing && totalTokens > 0) {
-            recordUsage(pool, pgSettings, billing, entityId, meterSlug, totalTokens, {
-                input_tokens: inputTokens,
-                output_tokens: outputTokens,
+        if (billing && usage.totalTokens > 0) {
+            recordUsage(pool, pgSettings, billing, entityId, meterSlug, usage.totalTokens, {
+                input_tokens: usage.input,
+                output_tokens: usage.output,
+                cache_read_tokens: usage.cacheRead,
+                cache_write_tokens: usage.cacheWrite,
                 model,
                 latency_ms: latencyMs,
                 stream: false,
@@ -514,9 +524,9 @@ async function handleSendMessage(req, res, entityId) {
                 provider: 'ollama',
                 service: 'llm',
                 operation: 'chat',
-                inputTokens,
-                outputTokens,
-                totalTokens,
+                inputTokens: usage.input,
+                outputTokens: usage.output,
+                totalTokens: usage.totalTokens,
                 latencyMs,
                 status: 'ok',
             }).catch(() => { });
@@ -525,14 +535,14 @@ async function handleSendMessage(req, res, entityId) {
             id: `msg_${Date.now()}`,
             choices: [{
                     index: 0,
-                    message: { role: 'assistant', content: result.content },
+                    message: { role: 'assistant', content },
                     finish_reason: 'stop',
                 }],
             model,
             usage: {
-                prompt_tokens: inputTokens,
-                completion_tokens: outputTokens,
-                total_tokens: totalTokens,
+                prompt_tokens: usage.input,
+                completion_tokens: usage.output,
+                total_tokens: usage.totalTokens,
             },
         });
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@constructive-io/graphql-server",
-  "version": "4.32.0",
+  "version": "4.33.0",
   "author": "Constructive <developers@constructive.io>",
   "description": "Constructive GraphQL Server",
   "main": "index.js",
@@ -41,7 +41,7 @@
     "backend"
   ],
   "dependencies": {
-    "@agentic-kit/ollama": "^1.2.1",
+    "@agentic-kit/ollama": "^2.0.0",
     "@constructive-io/csrf": "^0.14.0",
     "@constructive-io/graphql-env": "^3.11.1",
     "@constructive-io/graphql-types": "^3.10.1",
@@ -64,7 +64,7 @@
     "graphile-build-pg": "5.0.2",
     "graphile-cache": "^3.11.2",
     "graphile-config": "1.0.1",
-    "graphile-llm": "^0.8.0",
+    "graphile-llm": "^0.9.0",
     "graphile-settings": "^5.2.4",
     "graphile-utils": "5.0.1",
     "graphql": "16.13.0",
@@ -94,5 +94,5 @@
     "nodemon": "^3.1.14",
     "ts-node": "^10.9.2"
   },
-  "gitHead": "030e1144acbd4e288ee74eff2ac0021ca0382ef7"
+  "gitHead": "f3ea414974306e3c0d1d68edc93b4cdd8fa6e806"
 }