graphile-llm 0.8.0 → 0.10.0

package/plugins/llm-module-plugin.js

@@ -23,8 +23,8 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createLlmModulePlugin = createLlmModulePlugin;
-const embedder_1 = require("../embedder");
 const chat_1 = require("../chat");
+const embedder_1 = require("../embedder");
 const env_1 = require("../env");
 /**
  * Creates the LlmModulePlugin with the given options.
@@ -84,10 +84,10 @@ function createLlmModulePlugin(options = {}) {
                         llmEmbedder: embedder,
                         llmChatCompleter: chat,
                         llmEmbeddingModel: defaultEmbedder?.model ?? (0, env_1.getLlmEnvOptions)().embedding.model,
-                        llmChatModel: defaultChatCompleter?.model ?? (0, env_1.getLlmEnvOptions)().chat.model,
+                        llmChatModel: defaultChatCompleter?.model ?? (0, env_1.getLlmEnvOptions)().chat.model
                     }, 'LlmModulePlugin adding llmEmbedder, llmChatCompleter, and model names to build');
-                },
-            },
-        },
+                }
+            }
+        }
     };
 }

package/plugins/metering-plugin.js

@@ -34,8 +34,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createLlmMeteringPlugin = createLlmMeteringPlugin;
 const node_async_hooks_1 = require("node:async_hooks");
-const metering_1 = require("../metering");
 const config_cache_1 = require("../config-cache");
+const metering_1 = require("../metering");
 // ─── Request-scoped context via AsyncLocalStorage ───────────────────────────
 const meteringStore = new node_async_hooks_1.AsyncLocalStorage();
 // ─── Helpers ────────────────────────────────────────────────────────────────
@@ -75,7 +75,7 @@ async function buildMeteringContext(graphqlContext, resolveEntityId) {
         requestId,
         databaseId,
         actorId,
-        inferenceLog: inferenceLogConfig,
+        inferenceLog: inferenceLogConfig
     };
 }
 /**
@@ -92,10 +92,10 @@ function wrapEmbedderWithMetering(embedder, meteringOptions) {
         if (!ctx) {
             // No metering context in scope — call original embedder directly
             const startTime = Date.now();
-            const result = await embedder(text);
+            const { embedding } = await embedder(text);
             const latencyMs = Date.now() - startTime;
-            console.log(`[graphile-llm] Embed (unmetered): dims=${result?.length ?? 0}, latency=${latencyMs}ms`);
-            return result;
+            console.log(`[graphile-llm] Embed (unmetered): dims=${embedding?.length ?? 0}, latency=${latencyMs}ms`);
+            return embedding;
         }
         const result = await (0, metering_1.meteredEmbed)(embedder, text, ctx, meteringOptions);
         if (result.quotaExceeded) {
@@ -106,7 +106,7 @@ function wrapEmbedderWithMetering(embedder, meteringOptions) {
 }
 // ─── Plugin ─────────────────────────────────────────────────────────────────
 function createLlmMeteringPlugin(meteringConfig = {}) {
-    const { embeddingMeterSlug: configEmbeddingSlug, chatMeterSlug: configChatSlug, skipMetering, resolveEntityId = defaultResolveEntityId, } = meteringConfig;
+    const { embeddingMeterSlug: configEmbeddingSlug, chatMeterSlug: configChatSlug, skipMetering, resolveEntityId = defaultResolveEntityId } = meteringConfig;
     return {
         name: 'LlmMeteringPlugin',
         version: '0.2.0',
@@ -137,13 +137,13 @@ function createLlmMeteringPlugin(meteringConfig = {}) {
                         chatMeterSlug: chatSlug,
                         skipMetering,
                         embeddingModel: embeddingModel ?? undefined,
-                        chatModel: chatModel ?? undefined,
+                        chatModel: chatModel ?? undefined
                     };
                     // Replace the embedder with a metered version.
                     // Same signature except it can return null (quota exceeded).
                     const meteredEmbedder = wrapEmbedderWithMetering(originalEmbedder, meteringOptions);
                     return build.extend(build, {
-                        llmEmbedder: meteredEmbedder,
+                        llmEmbedder: meteredEmbedder
                     }, 'LlmMeteringPlugin replacing llmEmbedder with metered version');
                 },
                 /**
@@ -151,7 +151,7 @@ function createLlmMeteringPlugin(meteringConfig = {}) {
                  * request-scoped metering context via AsyncLocalStorage.
                  */
                 GraphQLObjectType_fields_field(field, build, context) {
-                    const { scope: { isRootQuery, isRootMutation }, } = context;
+                    const { scope: { isRootQuery, isRootMutation } } = context;
                     if (!isRootQuery && !isRootMutation)
                         return field;
                     // Only wrap if we actually replaced the embedder
@@ -169,10 +169,10 @@ function createLlmMeteringPlugin(meteringConfig = {}) {
                             return meteringStore.run(ctx, () => {
                                 return oldResolve(source, args, graphqlContext, info);
                             });
-                        },
+                        }
                     };
-                },
-            },
-        },
+                }
+            }
+        }
     };
 }

package/plugins/rag-plugin.js

@@ -65,7 +65,7 @@ function parseHasChunksTag(raw, codec) {
         parentFkField: parsed.parentFk || 'parent_id',
         parentPkField: parsed.parentPk || 'id',
         embeddingField: parsed.embeddingField || 'embedding',
-        contentField: parsed.contentField || 'content',
+        contentField: parsed.contentField || 'content'
     };
 }
 /**
@@ -225,10 +225,10 @@ function createLlmRagPlugin(ragDefaults = {}) {
                             minSimilarity: $minSimilarity,
                             systemPrompt: $systemPrompt,
                             withPgClient: $withPgClient,
-                            pgSettings: $pgSettings,
+                            pgSettings: $pgSettings
                         });
                         return (0, grafast_1.lambda)($combined, async (input) => {
-                            const { prompt, contextLimit: queryContextLimit, minSimilarity: queryMinSimilarity, systemPrompt: querySystemPrompt, withPgClient, pgSettings, } = input;
+                            const { prompt, contextLimit: queryContextLimit, minSimilarity: queryMinSimilarity, systemPrompt: querySystemPrompt, withPgClient, pgSettings } = input;
                             if (!prompt || typeof prompt !== 'string') {
                                 throw new Error('RAG_INVALID_PROMPT: prompt is required');
                             }
@@ -247,7 +247,7 @@ function createLlmRagPlugin(ragDefaults = {}) {
                             const systemPromptTemplate = querySystemPrompt ?? ragDefaults.systemPrompt ?? DEFAULT_SYSTEM_PROMPT;
                             // Step 1: Embed the prompt
                             const startEmbed = Date.now();
-                            const vector = await embedder(prompt);
+                            const { embedding: vector } = await embedder(prompt);
                             const embedLatency = Date.now() - startEmbed;
                             const vectorString = `[${vector.join(',')}]`;
                             console.log(`[graphile-llm] RAG embed: dims=${vector.length}, latency=${embedLatency}ms`);
@@ -263,7 +263,7 @@ function createLlmRagPlugin(ragDefaults = {}) {
                                                 content: row.content,
                                                 parent_id: row.parent_id,
                                                 distance: parseFloat(row.distance),
-                                                table_name: table.parentCodecName,
+                                                table_name: table.parentCodecName
                                             });
                                         }
                                     }
@@ -277,31 +277,31 @@ function createLlmRagPlugin(ragDefaults = {}) {
                                     answer: 'No relevant context found for your query. ' +
                                         'Try broadening your search or lowering the minimum similarity threshold.',
                                     sources: [],
-                                    tokensUsed: null,
+                                    tokensUsed: null
                                 };
                             }
                             // Step 3: Assemble context
                             const contextText = assembleContext(topChunks);
                             // Step 4: Call chat completion
                             const startChat = Date.now();
-                            const answer = await chatCompleter([
+                            const chatResult = await chatCompleter([
                                 { role: 'system', content: systemPromptTemplate + contextText },
-                                { role: 'user', content: prompt },
+                                { role: 'user', content: prompt }
                             ], {
-                                maxTokens: ragDefaults.maxTokens ?? DEFAULT_MAX_TOKENS,
+                                maxTokens: ragDefaults.maxTokens ?? DEFAULT_MAX_TOKENS
                             });
                             const chatLatency = Date.now() - startChat;
-                            console.log(`[graphile-llm] RAG chat: sources=${topChunks.length}, latency=${chatLatency}ms`);
+                            console.log(`[graphile-llm] RAG chat: sources=${topChunks.length}, tokens=${chatResult.usage.totalTokens}, latency=${chatLatency}ms`);
                             // Step 5: Return response
                             return {
-                                answer,
+                                answer: chatResult.content,
                                 sources: topChunks.map((chunk) => ({
                                     content: chunk.content,
                                     similarity: 1 - chunk.distance,
                                     tableName: chunk.table_name,
-                                    parentId: chunk.parent_id,
+                                    parentId: chunk.parent_id
                                 })),
-                                tokensUsed: null, // Deferred to metering system
+                                tokensUsed: chatResult.usage.totalTokens
                             };
                         });
                     },
@@ -316,17 +316,17 @@ function createLlmRagPlugin(ragDefaults = {}) {
                                     'to use embedText. Set defaultEmbedder in GraphileLlmPreset options.');
                             }
                             const startTime = Date.now();
-                            const vector = await embedder(text);
+                            const { embedding: vector } = await embedder(text);
                             const latencyMs = Date.now() - startTime;
                             console.log(`[graphile-llm] embedText: dims=${vector.length}, latency=${latencyMs}ms`);
                             return {
                                 vector,
-                                dimensions: vector.length,
+                                dimensions: vector.length
                             };
                         });
-                    },
-                },
-            },
+                    }
+                }
+            }
         };
     });
     return {
@@ -338,7 +338,7 @@ function createLlmRagPlugin(ragDefaults = {}) {
         after: [
             'LlmModulePlugin',
             'UnifiedSearchPlugin',
-            'VectorCodecPlugin',
-        ],
+            'VectorCodecPlugin'
+        ]
     };
 }

package/plugins/text-mutation-plugin.js

@@ -48,7 +48,7 @@ function getTextToVectorMapping(pgCodec, build) {
         if (isVectorCodec(attribute.codec)) {
             const fieldName = build.inflection.attribute({
                 codec: pgCodec,
-                attributeName,
+                attributeName
             });
             mapping[`${fieldName}Text`] = fieldName;
         }
@@ -76,7 +76,7 @@ function createLlmTextMutationPlugin() {
             'PgAttributesPlugin',
             'PgMutationCreatePlugin',
             'PgMutationUpdateDeletePlugin',
-            'VectorCodecPlugin',
+            'VectorCodecPlugin'
         ],
         schema: {
             hooks: {
@@ -85,12 +85,12 @@ function createLlmTextMutationPlugin() {
                  * for tables that have vector columns.
                  */
                 GraphQLInputObjectType_fields(fields, build, context) {
-                    const { scope: { isPgPatch, isPgBaseInput, isMutationInput, pgCodec, }, } = context;
+                    const { scope: { isPgPatch, isPgBaseInput, isMutationInput, pgCodec } } = context;
                     // Only intercept create/update input types for table rows
                     if (!pgCodec?.attributes || (!isPgPatch && !isPgBaseInput && !isMutationInput)) {
                         return fields;
                     }
-                    const { graphql: { GraphQLString }, } = build;
+                    const { graphql: { GraphQLString } } = build;
                     // Find vector columns on this table
                     const vectorColumns = [];
                     for (const [attributeName, attribute] of Object.entries(pgCodec.attributes)) {
@@ -106,7 +106,7 @@ function createLlmTextMutationPlugin() {
                         // Convert snake_case column name to camelCase field name
                         const fieldName = build.inflection.attribute({
                             codec: pgCodec,
-                            attributeName: columnName,
+                            attributeName: columnName
                         });
                         const textFieldName = `${fieldName}Text`;
                         newFields = build.extend(newFields, {
@@ -114,8 +114,8 @@ function createLlmTextMutationPlugin() {
                                 type: GraphQLString,
                                 description: `Natural language text to embed server-side into the \`${fieldName}\` vector column. ` +
                                     `Mutually exclusive with \`${fieldName}\` — provide one or the other. ` +
-                                    'Requires the LLM plugin to be configured with an embedding provider.',
-                            },
+                                    'Requires the LLM plugin to be configured with an embedding provider.'
+                            }
                         }, `LlmTextMutationPlugin adding ${textFieldName} companion field for vector column '${columnName}'`);
                     }
                     return newFields;
@@ -132,7 +132,7 @@ function createLlmTextMutationPlugin() {
                  * If the embedder returns null (e.g. quota exceeded), throws an error.
                  */
                 GraphQLObjectType_fields_field(field, build, context) {
-                    const { scope: { isRootMutation, fieldName, pgCodec }, } = context;
+                    const { scope: { isRootMutation, fieldName, pgCodec } } = context;
                     // Only wrap root mutation fields on tables with attributes
                     if (!isRootMutation || !pgCodec || !pgCodec.attributes) {
                         return field;
@@ -195,10 +195,10 @@ function createLlmTextMutationPlugin() {
                             }
                             await embedTextFields(args);
                             return oldResolve(source, args, graphqlContext, info);
-                        },
+                        }
                     };
-                },
-            },
-        },
+                }
+            }
+        }
     };
 }

package/plugins/text-search-plugin.js

@@ -105,7 +105,7 @@ function createLlmTextSearchPlugin() {
         after: [
             'LlmModulePlugin',
             'UnifiedSearchPlugin',
-            'VectorCodecPlugin',
+            'VectorCodecPlugin'
         ],
         schema: {
             hooks: {
@@ -116,18 +116,18 @@ function createLlmTextSearchPlugin() {
                  * The field is optional — clients provide either `text` or `vector`.
                  */
                 GraphQLInputObjectType_fields(fields, build, context) {
-                    const { scope: { inputObjectTypeName }, } = context;
+                    const { scope: { inputObjectTypeName } } = context;
                     if (inputObjectTypeName !== 'VectorNearbyInput') {
                         return fields;
                     }
-                    const { graphql: { GraphQLString }, } = build;
+                    const { graphql: { GraphQLString } } = build;
                     return build.extend(fields, {
                         text: {
                             type: GraphQLString,
                             description: 'Natural language text to embed server-side for similarity search. ' +
                                 'Mutually exclusive with `vector` — provide one or the other. ' +
-                                'Requires the LLM plugin to be configured with an embedding provider.',
-                        },
+                                'Requires the LLM plugin to be configured with an embedding provider.'
+                        }
                     }, 'LlmTextSearchPlugin adding text field to VectorNearbyInput');
                 },
                 /**
@@ -139,7 +139,7 @@ function createLlmTextSearchPlugin() {
                  * and graphile-bucket-provisioner-plugin.
                  */
                 GraphQLObjectType_fields_field(field, build, context) {
-                    const { scope: { isRootQuery, pgCodec }, } = context;
+                    const { scope: { isRootQuery, pgCodec } } = context;
                     // Only wrap root query fields on tables with vector columns
                     if (!isRootQuery || !pgCodec || !hasVectorColumns(pgCodec)) {
                         return field;
@@ -161,7 +161,7 @@ function createLlmTextSearchPlugin() {
                                 await embedTextInWhere(args.filter, embedder);
                             }
                             return oldResolve(source, args, graphqlContext, info);
-                        },
+                        }
                     };
                 },
                 finalize(schema, build) {
@@ -171,8 +171,8 @@ function createLlmTextSearchPlugin() {
                             'will return errors if used. Configure an embedding provider to enable.');
                     }
                     return schema;
-                },
-            },
-        },
+                }
+            }
+        }
     };
 }

package/preset.js

@@ -67,10 +67,10 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.GraphileLlmPreset = GraphileLlmPreset;
 const llm_module_plugin_1 = require("./plugins/llm-module-plugin");
-const text_search_plugin_1 = require("./plugins/text-search-plugin");
-const text_mutation_plugin_1 = require("./plugins/text-mutation-plugin");
-const rag_plugin_1 = require("./plugins/rag-plugin");
 const metering_plugin_1 = require("./plugins/metering-plugin");
+const rag_plugin_1 = require("./plugins/rag-plugin");
+const text_mutation_plugin_1 = require("./plugins/text-mutation-plugin");
+const text_search_plugin_1 = require("./plugins/text-search-plugin");
 /**
  * Creates a preset that includes all LLM plugins.
  *
@@ -78,9 +78,9 @@ const metering_plugin_1 = require("./plugins/metering-plugin");
  * @returns A GraphileConfig.Preset to add to your extends array
  */
 function GraphileLlmPreset(options = {}) {
-    const { enableTextSearch = true, enableTextMutations = true, enableRag = false, ragDefaults, metering, } = options;
+    const { enableTextSearch = true, enableTextMutations = true, enableRag = false, ragDefaults, metering } = options;
     const plugins = [
-        (0, llm_module_plugin_1.createLlmModulePlugin)(options),
+        (0, llm_module_plugin_1.createLlmModulePlugin)(options)
     ];
     // Metering is opt-in: only loaded when metering is truthy
     // (true, or a MeteringConfig object)
@@ -98,7 +98,7 @@ function GraphileLlmPreset(options = {}) {
         plugins.push((0, rag_plugin_1.createLlmRagPlugin)(ragDefaults));
     }
     return {
-        plugins,
+        plugins
     };
 }
 exports.default = GraphileLlmPreset;

package/types.d.ts

@@ -4,9 +4,18 @@
  * Shared type definitions for the LLM plugin.
  */
 /**
- * A function that converts text into a vector embedding.
+ * Result from an embedding call, including real token usage from the provider.
  */
-export type EmbedderFunction = (text: string) => Promise<number[]>;
+export interface EmbeddingResult {
+    /** The vector embedding */
+    embedding: number[];
+    /** Number of prompt tokens consumed (from provider; 0 if unavailable) */
+    promptTokens: number;
+}
+/**
+ * A function that converts text into a vector embedding with token usage.
+ */
+export type EmbedderFunction = (text: string) => Promise<EmbeddingResult>;
 /**
  * Configuration for an embedding provider.
  */
@@ -18,6 +27,24 @@ export interface EmbedderConfig {
     /** Base URL for the provider (e.g. 'http://localhost:11434' for Ollama) */
     baseUrl?: string;
 }
+/**
+ * Token usage metadata returned by LLM providers.
+ * Maps to the billing schema's inference_log columns.
+ */
+export interface LlmUsage {
+    /** Prompt / input tokens consumed */
+    input: number;
+    /** Completion / output tokens generated (includes reasoning for providers that count it) */
+    output: number;
+    /** Reasoning tokens (subset of output — not additive) */
+    reasoning: number;
+    /** Tokens served from prompt cache (zero cost) */
+    cacheRead: number;
+    /** Tokens written to prompt cache */
+    cacheWrite: number;
+    /** input + output + cacheRead + cacheWrite */
+    totalTokens: number;
+}
 /**
  * A single message in a chat conversation.
  */
@@ -35,9 +62,17 @@ export interface ChatOptions {
     temperature?: number;
 }
 /**
- * A function that sends messages to a chat completion provider and returns the response.
+ * Result from a chat completion call, including real token usage.
+ */
+export interface ChatResult {
+    content: string;
+    usage: LlmUsage;
+}
+/**
+ * A function that sends messages to a chat completion provider
+ * and returns the response with token usage metadata.
  */
-export type ChatFunction = (messages: ChatMessage[], options?: ChatOptions) => Promise<string>;
+export type ChatFunction = (messages: ChatMessage[], options?: ChatOptions) => Promise<ChatResult>;
 /**
  * Configuration for a chat completion provider.
  */