@constructive-io/graphql-server 4.31.5 → 4.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/middleware/graphile.js +11 -4
- package/esm/middleware/llm-api.js +575 -0
- package/esm/server.js +4 -0
- package/middleware/graphile.js +11 -4
- package/middleware/llm-api.d.ts +24 -0
- package/middleware/llm-api.js +611 -0
- package/package.json +22 -20
- package/server.js +4 -0
|
@@ -0,0 +1,611 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* LLM API Router
|
|
4
|
+
*
|
|
5
|
+
* Express router providing REST streaming endpoints for AI agent conversations.
|
|
6
|
+
* Uses the agent tables (agent_thread, agent_message) discovered from the
|
|
7
|
+
* agent_chat_module config table at runtime.
|
|
8
|
+
*
|
|
9
|
+
* Hybrid architecture:
|
|
10
|
+
* - GraphQL handles CRUD (threads, messages, tasks) via PostGraphile
|
|
11
|
+
* - REST handles SSE streaming for chat completions (what GraphQL can't do)
|
|
12
|
+
*
|
|
13
|
+
* Routes (entity-scoped):
|
|
14
|
+
* POST /v1/orgs/:entity_id/threads → create thread
|
|
15
|
+
* POST /v1/orgs/:entity_id/threads/:thread_id/messages → send message + stream response
|
|
16
|
+
*
|
|
17
|
+
* Routes (global — bills to actor_id from JWT):
|
|
18
|
+
* POST /v1/threads → create thread (entity_id = user_id)
|
|
19
|
+
* POST /v1/threads/:thread_id/messages → send message + stream response
|
|
20
|
+
*
|
|
21
|
+
* Auth: JWT from the auth middleware (req.token) → pg SET LOCAL context for RLS
|
|
22
|
+
* Metering: check_billing_quota → LLM call → record_usage with real token counts
|
|
23
|
+
*/
|
|
24
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
25
|
+
if (k2 === undefined) k2 = k;
|
|
26
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
27
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
28
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
29
|
+
}
|
|
30
|
+
Object.defineProperty(o, k2, desc);
|
|
31
|
+
}) : (function(o, m, k, k2) {
|
|
32
|
+
if (k2 === undefined) k2 = k;
|
|
33
|
+
o[k2] = m[k];
|
|
34
|
+
}));
|
|
35
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
36
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
37
|
+
}) : function(o, v) {
|
|
38
|
+
o["default"] = v;
|
|
39
|
+
});
|
|
40
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
41
|
+
var ownKeys = function(o) {
|
|
42
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
43
|
+
var ar = [];
|
|
44
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
45
|
+
return ar;
|
|
46
|
+
};
|
|
47
|
+
return ownKeys(o);
|
|
48
|
+
};
|
|
49
|
+
return function (mod) {
|
|
50
|
+
if (mod && mod.__esModule) return mod;
|
|
51
|
+
var result = {};
|
|
52
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
53
|
+
__setModuleDefault(result, mod);
|
|
54
|
+
return result;
|
|
55
|
+
};
|
|
56
|
+
})();
|
|
57
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
58
|
+
exports.createLlmApiRouter = createLlmApiRouter;
|
|
59
|
+
const express_1 = __importStar(require("express"));
|
|
60
|
+
const logger_1 = require("@pgpmjs/logger");
|
|
61
|
+
const pg_cache_1 = require("pg-cache");
|
|
62
|
+
const ollama_1 = require("@agentic-kit/ollama");
|
|
63
|
+
const graphile_cache_1 = require("graphile-cache");
|
|
64
|
+
const graphile_llm_1 = require("graphile-llm");
|
|
65
|
+
const log = new logger_1.Logger('llm-api');
|
|
66
|
+
// ─── Helpers ────────────────────────────────────────────────────────────────
|
|
67
|
+
function getPgSettings(req) {
|
|
68
|
+
const settings = {};
|
|
69
|
+
if (req.token?.user_id) {
|
|
70
|
+
settings['jwt.claims.user_id'] = req.token.user_id;
|
|
71
|
+
settings['role'] = 'authenticated';
|
|
72
|
+
}
|
|
73
|
+
if (req.databaseId) {
|
|
74
|
+
settings['jwt.claims.database_id'] = req.databaseId;
|
|
75
|
+
}
|
|
76
|
+
if (req.requestId) {
|
|
77
|
+
settings['request.id'] = req.requestId;
|
|
78
|
+
}
|
|
79
|
+
return settings;
|
|
80
|
+
}
|
|
81
|
+
async function withRlsClient(pool, pgSettings, fn) {
|
|
82
|
+
const client = await pool.connect();
|
|
83
|
+
try {
|
|
84
|
+
await client.query('BEGIN');
|
|
85
|
+
for (const [key, value] of Object.entries(pgSettings)) {
|
|
86
|
+
await client.query('SELECT set_config($1, $2, true)', [key, value]);
|
|
87
|
+
}
|
|
88
|
+
const result = await fn(client);
|
|
89
|
+
await client.query('COMMIT');
|
|
90
|
+
return result;
|
|
91
|
+
}
|
|
92
|
+
catch (err) {
|
|
93
|
+
await client.query('ROLLBACK').catch(() => { });
|
|
94
|
+
throw err;
|
|
95
|
+
}
|
|
96
|
+
finally {
|
|
97
|
+
client.release();
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
function resolveOllamaAdapter() {
|
|
101
|
+
const { chat } = (0, graphile_llm_1.getLlmEnvOptions)();
|
|
102
|
+
if (chat.provider === 'ollama') {
|
|
103
|
+
return {
|
|
104
|
+
adapter: new ollama_1.OllamaAdapter(chat.baseUrl),
|
|
105
|
+
model: chat.model,
|
|
106
|
+
baseUrl: chat.baseUrl,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
// ─── Billing Helpers ────────────────────────────────────────────────────────
|
|
112
|
+
async function checkQuota(pool, pgSettings, billing, entityId, meterSlug) {
|
|
113
|
+
try {
|
|
114
|
+
return await withRlsClient(pool, pgSettings, async (client) => {
|
|
115
|
+
const sql = `SELECT "${billing.privateSchema}"."${billing.checkBillingQuotaFunction}"($1, $2::uuid, $3) AS allowed`;
|
|
116
|
+
const result = await client.query(sql, [meterSlug, entityId, 1]);
|
|
117
|
+
return result.rows[0]?.allowed !== false;
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
catch (e) {
|
|
121
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
122
|
+
log.warn(`[llm-api] check_billing_quota failed (allowing): ${message}`);
|
|
123
|
+
return true;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
async function recordUsage(pool, pgSettings, billing, entityId, meterSlug, amount, metadata) {
|
|
127
|
+
try {
|
|
128
|
+
await withRlsClient(pool, pgSettings, async (client) => {
|
|
129
|
+
const sql = `SELECT "${billing.privateSchema}"."${billing.recordUsageFunction}"($1, $2::uuid, $3, $4::jsonb)`;
|
|
130
|
+
await client.query(sql, [meterSlug, entityId, amount, JSON.stringify(metadata)]);
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
catch (e) {
|
|
134
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
135
|
+
log.warn(`[llm-api] record_usage failed (non-fatal): ${message}`);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
async function resolveBilling(pool, pgSettings, databaseId) {
|
|
139
|
+
try {
|
|
140
|
+
let billing = null;
|
|
141
|
+
await withRlsClient(pool, pgSettings, async (client) => {
|
|
142
|
+
const entry = await (0, graphile_llm_1.getLlmBillingConfig)(client, databaseId);
|
|
143
|
+
billing = entry.billing;
|
|
144
|
+
});
|
|
145
|
+
return billing;
|
|
146
|
+
}
|
|
147
|
+
catch {
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
const INFERENCE_LOG_DISCOVERY_SQL = `
|
|
152
|
+
SELECT s.schema_name, ilm.inference_log_table_name
|
|
153
|
+
FROM metaschema_modules_public.inference_log_module ilm
|
|
154
|
+
JOIN metaschema_public.schema s ON s.id = ilm.schema_id
|
|
155
|
+
LIMIT 1
|
|
156
|
+
`;
|
|
157
|
+
const inferenceLogCache = new graphile_cache_1.ModuleConfigCache({
|
|
158
|
+
name: 'inference-log',
|
|
159
|
+
ttlMs: 60_000,
|
|
160
|
+
});
|
|
161
|
+
async function getInferenceLogInfo(pool, dbname) {
|
|
162
|
+
const cached = inferenceLogCache.get(dbname);
|
|
163
|
+
if (cached !== undefined)
|
|
164
|
+
return cached;
|
|
165
|
+
let info = null;
|
|
166
|
+
try {
|
|
167
|
+
const { rows } = await pool.query(INFERENCE_LOG_DISCOVERY_SQL);
|
|
168
|
+
if (rows.length > 0) {
|
|
169
|
+
info = {
|
|
170
|
+
schemaName: rows[0].schema_name,
|
|
171
|
+
tableName: rows[0].inference_log_table_name,
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
catch {
|
|
176
|
+
// Module not provisioned
|
|
177
|
+
}
|
|
178
|
+
inferenceLogCache.set(dbname, info);
|
|
179
|
+
return info;
|
|
180
|
+
}
|
|
181
|
+
async function logInference(pool, pgSettings, logInfo, data) {
|
|
182
|
+
try {
|
|
183
|
+
await withRlsClient(pool, pgSettings, async (client) => {
|
|
184
|
+
await client.query(`INSERT INTO "${logInfo.schemaName}"."${logInfo.tableName}"
|
|
185
|
+
(entity_id, actor_id, model, provider, service, operation, input_tokens, output_tokens, total_tokens, latency_ms, status)
|
|
186
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, [
|
|
187
|
+
data.entityId,
|
|
188
|
+
data.actorId,
|
|
189
|
+
data.model,
|
|
190
|
+
data.provider,
|
|
191
|
+
data.service,
|
|
192
|
+
data.operation,
|
|
193
|
+
data.inputTokens,
|
|
194
|
+
data.outputTokens,
|
|
195
|
+
data.totalTokens,
|
|
196
|
+
data.latencyMs,
|
|
197
|
+
data.status,
|
|
198
|
+
]);
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
catch (e) {
|
|
202
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
203
|
+
log.warn(`[llm-api] inference log INSERT failed (non-fatal): ${message}`);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
// ─── Route Handlers ─────────────────────────────────────────────────────────
|
|
207
|
+
async function handleCreateThread(req, res, entityId) {
|
|
208
|
+
if (!req.token?.user_id) {
|
|
209
|
+
res.status(401).json({ error: 'Authentication required' });
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
const dbname = req.api?.dbname;
|
|
213
|
+
if (!dbname) {
|
|
214
|
+
res.status(400).json({ error: 'Database not resolved' });
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
const pool = (0, pg_cache_1.getPgPool)({ database: dbname });
|
|
218
|
+
const discovery = await (0, graphile_llm_1.getAgentDiscovery)(pool, dbname);
|
|
219
|
+
if (!discovery?.thread) {
|
|
220
|
+
res.status(404).json({ error: 'Agent module not provisioned for this database' });
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
const body = req.body || {};
|
|
224
|
+
const { thread } = discovery;
|
|
225
|
+
const pgSettings = getPgSettings(req);
|
|
226
|
+
const result = await withRlsClient(pool, pgSettings, async (client) => {
|
|
227
|
+
const { rows } = await client.query(`INSERT INTO "${thread.schemaName}"."${thread.tableName}"
|
|
228
|
+
(entity_id, owner_id, mode, model, system_prompt, title)
|
|
229
|
+
VALUES ($1, $2, $3, $4, $5, $6)
|
|
230
|
+
RETURNING id, mode, model, system_prompt, status, created_at`, [
|
|
231
|
+
entityId,
|
|
232
|
+
req.token.user_id,
|
|
233
|
+
body.mode ?? 'ask',
|
|
234
|
+
body.model ?? null,
|
|
235
|
+
body.system_prompt ?? null,
|
|
236
|
+
body.title ?? null,
|
|
237
|
+
]);
|
|
238
|
+
return rows[0];
|
|
239
|
+
});
|
|
240
|
+
res.status(201).json({
|
|
241
|
+
id: result.id,
|
|
242
|
+
mode: result.mode,
|
|
243
|
+
model: result.model,
|
|
244
|
+
system_prompt: result.system_prompt,
|
|
245
|
+
status: result.status,
|
|
246
|
+
created_at: result.created_at,
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
async function handleSendMessage(req, res, entityId) {
|
|
250
|
+
if (!req.token?.user_id) {
|
|
251
|
+
res.status(401).json({ error: 'Authentication required' });
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
const dbname = req.api?.dbname;
|
|
255
|
+
if (!dbname) {
|
|
256
|
+
res.status(400).json({ error: 'Database not resolved' });
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
const pool = (0, pg_cache_1.getPgPool)({ database: dbname });
|
|
260
|
+
const discovery = await (0, graphile_llm_1.getAgentDiscovery)(pool, dbname);
|
|
261
|
+
if (!discovery?.thread || !discovery?.message) {
|
|
262
|
+
res.status(404).json({ error: 'Agent module not provisioned for this database' });
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
const body = req.body || {};
|
|
266
|
+
if (!body.messages?.length) {
|
|
267
|
+
res.status(400).json({ error: 'messages[] is required and must not be empty' });
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
const { thread, message: msgTable } = discovery;
|
|
271
|
+
const pgSettings = getPgSettings(req);
|
|
272
|
+
const threadId = req.params.thread_id;
|
|
273
|
+
const userId = req.token.user_id;
|
|
274
|
+
const databaseId = req.databaseId;
|
|
275
|
+
// 1. Verify thread exists and user owns it (RLS enforced)
|
|
276
|
+
const threadRow = await withRlsClient(pool, pgSettings, async (client) => {
|
|
277
|
+
const { rows } = await client.query(`SELECT id, mode, model, system_prompt, status
|
|
278
|
+
FROM "${thread.schemaName}"."${thread.tableName}"
|
|
279
|
+
WHERE id = $1`, [threadId]);
|
|
280
|
+
return rows[0];
|
|
281
|
+
});
|
|
282
|
+
if (!threadRow) {
|
|
283
|
+
res.status(404).json({ error: 'Thread not found' });
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
// 2. Resolve billing config + inference log discovery
|
|
287
|
+
const billing = databaseId
|
|
288
|
+
? await resolveBilling(pool, pgSettings, databaseId)
|
|
289
|
+
: null;
|
|
290
|
+
const inferenceLog = await getInferenceLogInfo(pool, dbname);
|
|
291
|
+
const ollama = resolveOllamaAdapter();
|
|
292
|
+
if (!ollama) {
|
|
293
|
+
res.status(503).json({ error: 'No LLM provider configured' });
|
|
294
|
+
return;
|
|
295
|
+
}
|
|
296
|
+
const model = body.model ?? threadRow.model ?? ollama.model;
|
|
297
|
+
const meterSlug = model;
|
|
298
|
+
if (billing) {
|
|
299
|
+
const allowed = await checkQuota(pool, pgSettings, billing, entityId, meterSlug);
|
|
300
|
+
if (!allowed) {
|
|
301
|
+
res.status(429).json({
|
|
302
|
+
error: 'Token quota exceeded',
|
|
303
|
+
meter: meterSlug,
|
|
304
|
+
entity_id: entityId,
|
|
305
|
+
});
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
// 3. Persist user message(s)
|
|
310
|
+
await withRlsClient(pool, pgSettings, async (client) => {
|
|
311
|
+
for (const msg of body.messages) {
|
|
312
|
+
if (msg.role === 'user') {
|
|
313
|
+
await client.query(`INSERT INTO "${msgTable.schemaName}"."${msgTable.tableName}"
|
|
314
|
+
(thread_id, owner_id, entity_id, author_role, parts)
|
|
315
|
+
VALUES ($1, $2, (SELECT entity_id FROM "${thread.schemaName}"."${thread.tableName}" WHERE id = $1), $3, $4)`, [
|
|
316
|
+
threadId,
|
|
317
|
+
userId,
|
|
318
|
+
'user',
|
|
319
|
+
JSON.stringify([{ type: 'text', text: msg.content }]),
|
|
320
|
+
]);
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
});
|
|
324
|
+
// 4. Load full thread history for context
|
|
325
|
+
const history = await withRlsClient(pool, pgSettings, async (client) => {
|
|
326
|
+
const { rows } = await client.query(`SELECT author_role, parts, created_at
|
|
327
|
+
FROM "${msgTable.schemaName}"."${msgTable.tableName}"
|
|
328
|
+
WHERE thread_id = $1
|
|
329
|
+
ORDER BY created_at ASC`, [threadId]);
|
|
330
|
+
return rows;
|
|
331
|
+
});
|
|
332
|
+
const llmMessages = [];
|
|
333
|
+
const systemPrompt = threadRow.system_prompt;
|
|
334
|
+
if (systemPrompt) {
|
|
335
|
+
llmMessages.push({ role: 'system', content: systemPrompt });
|
|
336
|
+
}
|
|
337
|
+
for (const row of history) {
|
|
338
|
+
const parts = Array.isArray(row.parts) ? row.parts : [];
|
|
339
|
+
const textContent = parts
|
|
340
|
+
.filter((p) => p.type === 'text')
|
|
341
|
+
.map((p) => p.text)
|
|
342
|
+
.join('');
|
|
343
|
+
if (textContent) {
|
|
344
|
+
llmMessages.push({
|
|
345
|
+
role: row.author_role === 'user' ? 'user' : 'assistant',
|
|
346
|
+
content: textContent,
|
|
347
|
+
});
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
// 5. Call LLM with token usage tracking
|
|
351
|
+
const shouldStream = body.stream !== false;
|
|
352
|
+
const startTime = Date.now();
|
|
353
|
+
if (shouldStream) {
|
|
354
|
+
// ── SSE Streaming via OllamaAdapter ─────────────────────────────────
|
|
355
|
+
res.writeHead(200, {
|
|
356
|
+
'Content-Type': 'text/event-stream',
|
|
357
|
+
'Cache-Control': 'no-cache',
|
|
358
|
+
'Connection': 'keep-alive',
|
|
359
|
+
'X-Accel-Buffering': 'no',
|
|
360
|
+
});
|
|
361
|
+
const messageId = `msg_${Date.now()}`;
|
|
362
|
+
try {
|
|
363
|
+
const systemMsg = llmMessages.find(m => m.role === 'system');
|
|
364
|
+
const nonSystem = llmMessages.filter(m => m.role !== 'system');
|
|
365
|
+
const modelDesc = ollama.adapter.createModel(model, {
|
|
366
|
+
maxOutputTokens: undefined,
|
|
367
|
+
});
|
|
368
|
+
const context = {
|
|
369
|
+
systemPrompt: systemMsg?.content,
|
|
370
|
+
messages: nonSystem.map((m) => ({
|
|
371
|
+
role: m.role,
|
|
372
|
+
content: m.content,
|
|
373
|
+
timestamp: Date.now(),
|
|
374
|
+
})),
|
|
375
|
+
};
|
|
376
|
+
const stream = ollama.adapter.stream(modelDesc, context, {
|
|
377
|
+
temperature: body.temperature,
|
|
378
|
+
});
|
|
379
|
+
let streamedContent = '';
|
|
380
|
+
for await (const event of stream) {
|
|
381
|
+
if (event.type === 'text_delta') {
|
|
382
|
+
streamedContent += event.delta;
|
|
383
|
+
const sseEvent = {
|
|
384
|
+
id: messageId,
|
|
385
|
+
choices: [{
|
|
386
|
+
index: 0,
|
|
387
|
+
delta: { content: event.delta, role: 'assistant' },
|
|
388
|
+
finish_reason: null,
|
|
389
|
+
}],
|
|
390
|
+
model,
|
|
391
|
+
};
|
|
392
|
+
res.write(`data: ${JSON.stringify(sseEvent)}\n\n`);
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
const result = await stream.result();
|
|
396
|
+
const content = streamedContent;
|
|
397
|
+
const latencyMs = Date.now() - startTime;
|
|
398
|
+
const usage = {
|
|
399
|
+
input: result.usage.input,
|
|
400
|
+
output: result.usage.output,
|
|
401
|
+
reasoning: result.usage.reasoning,
|
|
402
|
+
cacheRead: result.usage.cacheRead,
|
|
403
|
+
cacheWrite: result.usage.cacheWrite,
|
|
404
|
+
totalTokens: result.usage.totalTokens,
|
|
405
|
+
};
|
|
406
|
+
// Send [DONE] marker
|
|
407
|
+
res.write('data: [DONE]\n\n');
|
|
408
|
+
res.end();
|
|
409
|
+
// 6. Persist assistant message with model (fire-and-forget)
|
|
410
|
+
if (content) {
|
|
411
|
+
withRlsClient(pool, pgSettings, async (client) => {
|
|
412
|
+
await client.query(`INSERT INTO "${msgTable.schemaName}"."${msgTable.tableName}"
|
|
413
|
+
(thread_id, owner_id, entity_id, author_role, parts, model)
|
|
414
|
+
VALUES ($1, $2, (SELECT entity_id FROM "${thread.schemaName}"."${thread.tableName}" WHERE id = $1), $3, $4, $5)`, [
|
|
415
|
+
threadId,
|
|
416
|
+
userId,
|
|
417
|
+
'assistant',
|
|
418
|
+
JSON.stringify([{ type: 'text', text: content }]),
|
|
419
|
+
model,
|
|
420
|
+
]);
|
|
421
|
+
}).catch((err) => {
|
|
422
|
+
log.error('[llm-api] Failed to persist assistant message:', err);
|
|
423
|
+
});
|
|
424
|
+
}
|
|
425
|
+
// 7. Record billing usage (fire-and-forget)
|
|
426
|
+
if (billing && usage.totalTokens > 0) {
|
|
427
|
+
recordUsage(pool, pgSettings, billing, entityId, meterSlug, usage.totalTokens, {
|
|
428
|
+
input_tokens: usage.input,
|
|
429
|
+
output_tokens: usage.output,
|
|
430
|
+
cache_read_tokens: usage.cacheRead,
|
|
431
|
+
cache_write_tokens: usage.cacheWrite,
|
|
432
|
+
model,
|
|
433
|
+
latency_ms: latencyMs,
|
|
434
|
+
stream: true,
|
|
435
|
+
}).catch(() => { });
|
|
436
|
+
}
|
|
437
|
+
// 8. Inference log (fire-and-forget)
|
|
438
|
+
if (inferenceLog) {
|
|
439
|
+
logInference(pool, pgSettings, inferenceLog, {
|
|
440
|
+
entityId,
|
|
441
|
+
actorId: userId,
|
|
442
|
+
model,
|
|
443
|
+
provider: 'ollama',
|
|
444
|
+
service: 'llm',
|
|
445
|
+
operation: 'chat',
|
|
446
|
+
inputTokens: usage.input,
|
|
447
|
+
outputTokens: usage.output,
|
|
448
|
+
totalTokens: usage.totalTokens,
|
|
449
|
+
latencyMs,
|
|
450
|
+
status: 'ok',
|
|
451
|
+
}).catch(() => { });
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
catch (streamErr) {
|
|
455
|
+
log.error('[llm-api] Streaming error:', streamErr);
|
|
456
|
+
const errorEvent = { error: { message: streamErr.message, type: 'stream_error' } };
|
|
457
|
+
res.write(`data: ${JSON.stringify(errorEvent)}\n\n`);
|
|
458
|
+
res.write('data: [DONE]\n\n');
|
|
459
|
+
res.end();
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
else {
|
|
463
|
+
// ── Non-streaming (batch) via OllamaAdapter ─────────────────────────
|
|
464
|
+
const systemMsg = llmMessages.find(m => m.role === 'system');
|
|
465
|
+
const nonSystem = llmMessages.filter(m => m.role !== 'system');
|
|
466
|
+
const modelDesc = ollama.adapter.createModel(model, {
|
|
467
|
+
maxOutputTokens: undefined,
|
|
468
|
+
});
|
|
469
|
+
const context = {
|
|
470
|
+
systemPrompt: systemMsg?.content,
|
|
471
|
+
messages: nonSystem.map((m) => ({
|
|
472
|
+
role: m.role,
|
|
473
|
+
content: m.content,
|
|
474
|
+
timestamp: Date.now(),
|
|
475
|
+
})),
|
|
476
|
+
};
|
|
477
|
+
const stream = ollama.adapter.stream(modelDesc, context, {
|
|
478
|
+
temperature: body.temperature,
|
|
479
|
+
});
|
|
480
|
+
const result = await stream.result();
|
|
481
|
+
const content = result.content
|
|
482
|
+
.filter((block) => block.type === 'text')
|
|
483
|
+
.map((block) => block.text)
|
|
484
|
+
.join('');
|
|
485
|
+
const latencyMs = Date.now() - startTime;
|
|
486
|
+
const usage = {
|
|
487
|
+
input: result.usage.input,
|
|
488
|
+
output: result.usage.output,
|
|
489
|
+
reasoning: result.usage.reasoning,
|
|
490
|
+
cacheRead: result.usage.cacheRead,
|
|
491
|
+
cacheWrite: result.usage.cacheWrite,
|
|
492
|
+
totalTokens: result.usage.totalTokens,
|
|
493
|
+
};
|
|
494
|
+
// Persist assistant message with model
|
|
495
|
+
await withRlsClient(pool, pgSettings, async (client) => {
|
|
496
|
+
await client.query(`INSERT INTO "${msgTable.schemaName}"."${msgTable.tableName}"
|
|
497
|
+
(thread_id, owner_id, entity_id, author_role, parts, model)
|
|
498
|
+
VALUES ($1, $2, (SELECT entity_id FROM "${thread.schemaName}"."${thread.tableName}" WHERE id = $1), $3, $4, $5)`, [
|
|
499
|
+
threadId,
|
|
500
|
+
userId,
|
|
501
|
+
'assistant',
|
|
502
|
+
JSON.stringify([{ type: 'text', text: content }]),
|
|
503
|
+
model,
|
|
504
|
+
]);
|
|
505
|
+
});
|
|
506
|
+
// Record billing usage
|
|
507
|
+
if (billing && usage.totalTokens > 0) {
|
|
508
|
+
recordUsage(pool, pgSettings, billing, entityId, meterSlug, usage.totalTokens, {
|
|
509
|
+
input_tokens: usage.input,
|
|
510
|
+
output_tokens: usage.output,
|
|
511
|
+
cache_read_tokens: usage.cacheRead,
|
|
512
|
+
cache_write_tokens: usage.cacheWrite,
|
|
513
|
+
model,
|
|
514
|
+
latency_ms: latencyMs,
|
|
515
|
+
stream: false,
|
|
516
|
+
}).catch(() => { });
|
|
517
|
+
}
|
|
518
|
+
// Inference log
|
|
519
|
+
if (inferenceLog) {
|
|
520
|
+
logInference(pool, pgSettings, inferenceLog, {
|
|
521
|
+
entityId,
|
|
522
|
+
actorId: userId,
|
|
523
|
+
model,
|
|
524
|
+
provider: 'ollama',
|
|
525
|
+
service: 'llm',
|
|
526
|
+
operation: 'chat',
|
|
527
|
+
inputTokens: usage.input,
|
|
528
|
+
outputTokens: usage.output,
|
|
529
|
+
totalTokens: usage.totalTokens,
|
|
530
|
+
latencyMs,
|
|
531
|
+
status: 'ok',
|
|
532
|
+
}).catch(() => { });
|
|
533
|
+
}
|
|
534
|
+
res.json({
|
|
535
|
+
id: `msg_${Date.now()}`,
|
|
536
|
+
choices: [{
|
|
537
|
+
index: 0,
|
|
538
|
+
message: { role: 'assistant', content },
|
|
539
|
+
finish_reason: 'stop',
|
|
540
|
+
}],
|
|
541
|
+
model,
|
|
542
|
+
usage: {
|
|
543
|
+
prompt_tokens: usage.input,
|
|
544
|
+
completion_tokens: usage.output,
|
|
545
|
+
total_tokens: usage.totalTokens,
|
|
546
|
+
},
|
|
547
|
+
});
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
// ─── Router Factory ─────────────────────────────────────────────────────────
|
|
551
|
+
function createLlmApiRouter() {
|
|
552
|
+
const router = (0, express_1.Router)();
|
|
553
|
+
router.use(express_1.default.json());
|
|
554
|
+
// ── Entity-scoped routes ─────────────────────────────────────────────────
|
|
555
|
+
router.post('/v1/orgs/:entity_id/threads', async (req, res) => {
|
|
556
|
+
try {
|
|
557
|
+
await handleCreateThread(req, res, req.params.entity_id);
|
|
558
|
+
}
|
|
559
|
+
catch (err) {
|
|
560
|
+
log.error('[llm-api] Error creating thread:', err);
|
|
561
|
+
if (!res.headersSent) {
|
|
562
|
+
res.status(500).json({ error: 'Internal server error' });
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
});
|
|
566
|
+
router.post('/v1/orgs/:entity_id/threads/:thread_id/messages', async (req, res) => {
|
|
567
|
+
try {
|
|
568
|
+
await handleSendMessage(req, res, req.params.entity_id);
|
|
569
|
+
}
|
|
570
|
+
catch (err) {
|
|
571
|
+
log.error('[llm-api] Error in messages endpoint:', err);
|
|
572
|
+
if (!res.headersSent) {
|
|
573
|
+
res.status(500).json({ error: 'Internal server error' });
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
});
|
|
577
|
+
// ── Global routes (no entity_id — bills to actor_id from JWT) ────────────
|
|
578
|
+
router.post('/v1/threads', async (req, res) => {
|
|
579
|
+
try {
|
|
580
|
+
const userId = req.token?.user_id;
|
|
581
|
+
if (!userId) {
|
|
582
|
+
res.status(401).json({ error: 'Authentication required' });
|
|
583
|
+
return;
|
|
584
|
+
}
|
|
585
|
+
await handleCreateThread(req, res, userId);
|
|
586
|
+
}
|
|
587
|
+
catch (err) {
|
|
588
|
+
log.error('[llm-api] Error creating thread:', err);
|
|
589
|
+
if (!res.headersSent) {
|
|
590
|
+
res.status(500).json({ error: 'Internal server error' });
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
});
|
|
594
|
+
router.post('/v1/threads/:thread_id/messages', async (req, res) => {
|
|
595
|
+
try {
|
|
596
|
+
const userId = req.token?.user_id;
|
|
597
|
+
if (!userId) {
|
|
598
|
+
res.status(401).json({ error: 'Authentication required' });
|
|
599
|
+
return;
|
|
600
|
+
}
|
|
601
|
+
await handleSendMessage(req, res, userId);
|
|
602
|
+
}
|
|
603
|
+
catch (err) {
|
|
604
|
+
log.error('[llm-api] Error in messages endpoint:', err);
|
|
605
|
+
if (!res.headersSent) {
|
|
606
|
+
res.status(500).json({ error: 'Internal server error' });
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
});
|
|
610
|
+
return router;
|
|
611
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@constructive-io/graphql-server",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.33.0",
|
|
4
4
|
"author": "Constructive <developers@constructive.io>",
|
|
5
5
|
"description": "Constructive GraphQL Server",
|
|
6
6
|
"main": "index.js",
|
|
@@ -41,10 +41,11 @@
|
|
|
41
41
|
"backend"
|
|
42
42
|
],
|
|
43
43
|
"dependencies": {
|
|
44
|
+
"@agentic-kit/ollama": "^2.0.0",
|
|
44
45
|
"@constructive-io/csrf": "^0.14.0",
|
|
45
|
-
"@constructive-io/graphql-env": "^3.11.
|
|
46
|
-
"@constructive-io/graphql-types": "^3.10.
|
|
47
|
-
"@constructive-io/s3-utils": "^2.17.
|
|
46
|
+
"@constructive-io/graphql-env": "^3.11.1",
|
|
47
|
+
"@constructive-io/graphql-types": "^3.10.1",
|
|
48
|
+
"@constructive-io/s3-utils": "^2.17.1",
|
|
48
49
|
"@constructive-io/upload-names": "^2.16.0",
|
|
49
50
|
"@constructive-io/url-domains": "^2.16.0",
|
|
50
51
|
"@graphile-contrib/pg-many-to-many": "2.0.0-rc.2",
|
|
@@ -57,40 +58,41 @@
|
|
|
57
58
|
"deepmerge": "^4.3.1",
|
|
58
59
|
"express": "^5.2.1",
|
|
59
60
|
"gql-ast": "^3.10.0",
|
|
60
|
-
"grafast": "1.0.
|
|
61
|
+
"grafast": "1.0.2",
|
|
61
62
|
"grafserv": "1.0.0",
|
|
62
|
-
"graphile-build": "5.0.
|
|
63
|
-
"graphile-build-pg": "5.0.
|
|
64
|
-
"graphile-cache": "^3.11.
|
|
65
|
-
"graphile-config": "1.0.
|
|
66
|
-
"graphile-
|
|
67
|
-
"graphile-
|
|
63
|
+
"graphile-build": "5.0.2",
|
|
64
|
+
"graphile-build-pg": "5.0.2",
|
|
65
|
+
"graphile-cache": "^3.11.2",
|
|
66
|
+
"graphile-config": "1.0.1",
|
|
67
|
+
"graphile-llm": "^0.9.0",
|
|
68
|
+
"graphile-settings": "^5.2.4",
|
|
69
|
+
"graphile-utils": "5.0.1",
|
|
68
70
|
"graphql": "16.13.0",
|
|
69
71
|
"graphql-upload": "^13.0.0",
|
|
70
72
|
"lru-cache": "^11.2.7",
|
|
71
73
|
"multer": "^2.1.1",
|
|
72
|
-
"pg": "^8.
|
|
73
|
-
"pg-cache": "^3.10.
|
|
74
|
+
"pg": "^8.21.0",
|
|
75
|
+
"pg-cache": "^3.10.1",
|
|
74
76
|
"pg-env": "^1.14.0",
|
|
75
|
-
"pg-query-context": "^2.15.
|
|
76
|
-
"pg-sql2": "5.0.
|
|
77
|
-
"postgraphile": "5.0.
|
|
77
|
+
"pg-query-context": "^2.15.1",
|
|
78
|
+
"pg-sql2": "5.0.1",
|
|
79
|
+
"postgraphile": "5.0.3",
|
|
78
80
|
"request-ip": "^3.3.0"
|
|
79
81
|
},
|
|
80
82
|
"devDependencies": {
|
|
81
|
-
"@aws-sdk/client-s3": "^3.
|
|
83
|
+
"@aws-sdk/client-s3": "^3.1052.0",
|
|
82
84
|
"@types/cookie-parser": "^1.4.10",
|
|
83
85
|
"@types/cors": "^2.8.17",
|
|
84
86
|
"@types/express": "^5.0.6",
|
|
85
87
|
"@types/graphql-upload": "^8.0.12",
|
|
86
88
|
"@types/multer": "^2.1.0",
|
|
87
|
-
"@types/pg": "^8.
|
|
89
|
+
"@types/pg": "^8.20.0",
|
|
88
90
|
"@types/request-ip": "^0.0.41",
|
|
89
91
|
"cookie-parser": "^1.4.7",
|
|
90
|
-
"graphile-test": "4.15.
|
|
92
|
+
"graphile-test": "4.15.3",
|
|
91
93
|
"makage": "^0.3.0",
|
|
92
94
|
"nodemon": "^3.1.14",
|
|
93
95
|
"ts-node": "^10.9.2"
|
|
94
96
|
},
|
|
95
|
-
"gitHead": "
|
|
97
|
+
"gitHead": "f3ea414974306e3c0d1d68edc93b4cdd8fa6e806"
|
|
96
98
|
}
|