@formio/uag 2.0.0-rc.6 → 2.0.0-rc.7

package/lib/modules/types.d.ts

@@ -0,0 +1,68 @@
+import type { Request } from 'express';
+import type { ModuleApi, ModuleRequest, ModuleRoute } from '@formio/appserver';
+import type { Converter, ConverterOutput, Representation } from '../documents/types';
+import type { FileRoute, Pins, AcceptEntry } from '../documents/routes';
+/** A module route's request in the UAG: also its request id. */
+export type UAGModuleRequest = ModuleRequest & {
+    requestId: string;
+};
+/** The names of the UAG's own rate-limited operations, which module routes may not take. */
+export declare const UAG_OPERATIONS: string[];
+/** An agent engine a module may use: its name and model, and generate, which checks the license first. */
+export type GeneratingEngine = {
+    name: string;
+    model: string;
+    accepts: Representation[];
+    generate(request: any): Promise<any>;
+};
+/**
+ * The module API the UAG gives a module (version 1): appserver's, with the
+ * license check, engines, converters and the UAG's helpers. The project and
+ * the engines themselves are never handed out.
+ */
+export type UAGModuleApi<Settings extends Record<string, any> = Record<string, any>> = ModuleApi<Settings> & {
+    /** Throws 403 license_required unless Agentic Formflow is allowed. */
+    requireLicense(): Promise<void>;
+    converters: {
+        list(): Converter[];
+        identify(buffer: Buffer, hint: {
+            name: string;
+            type?: string;
+        }): string;
+        route(type: string, inputs: Representation[][], pins?: Pins): FileRoute | null;
+        accept(inputs: Representation[][]): AcceptEntry[];
+        run(steps: Converter[], input: {
+            buffer: Buffer;
+            type: string;
+            name: string;
+        }, to: Representation, options: {
+            signal: AbortSignal;
+        }): Promise<ConverterOutput>;
+        stepName(converter: {
+            name: string;
+            version?: string;
+        }): string;
+    };
+    engines: {
+        /**
+         * The named engine, or the first agent engine that generates and reads every
+         * representation in `needs`; throws 503 no_engine. Its `generate` checks the
+         * license first, so no module calls a model outside it.
+         */
+        select(options: {
+            name?: string;
+            needs?: Representation[];
+        }): GeneratingEngine;
+    };
+    markdown: {
+        sanitize(text: string): string;
+        fromHtml(html: string): string;
+    };
+    pdf: {
+        pageCount(pdf: Buffer): Promise<number | undefined>;
+    };
+    /** Who the request is from, for limits: the user or token, or, for an anonymous caller, `anonymous:<ip>`. */
+    callerKey(req: Request): string;
+};
+/** A form route of a UAG module. */
+export type UAGModuleRoute = ModuleRoute<UAGModuleApi>;

package/lib/modules/types.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UAG_OPERATIONS = void 0;
+/** The names of the UAG's own rate-limited operations, which module routes may not take. */
+exports.UAG_OPERATIONS = ['extract', 'decide', 'generate', 'chat', 'capabilities'];

package/lib/pluginFiles.d.ts

@@ -0,0 +1,16 @@
+import type { UAGServer } from './index';
+/** Where the curated image keeps its plugins file. */
+export declare const DEFAULT_PLUGINS_FILE = "/app/plugins/index.mjs";
+/**
+ * The plugins file to load: UAG_PLUGINS (a path, or "off"), or else the
+ * image's default when it exists. Undefined when there is none.
+ */
+export declare function pluginsFile(env?: Record<string, string | undefined>, defaultFile?: string): string | undefined;
+/**
+ * Load the plugins file, if any, and use it, named by its path in error
+ * messages. Its default export is a plugin or a list of plugins; or a module
+ * (an extra, with its plugins), or a list mixing modules and
+ * plugins, each module used in turn. Plugins are trusted code: they run in
+ * the UAG process.
+ */
+export declare function usePluginFiles(server: Pick<UAGServer, 'use'>, env?: Record<string, string | undefined>, defaultFile?: string): Promise<string | undefined>;

package/lib/pluginFiles.js

@@ -0,0 +1,62 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_PLUGINS_FILE = void 0;
+exports.pluginsFile = pluginsFile;
+exports.usePluginFiles = usePluginFiles;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const url_1 = require("url");
+const appserver_1 = require("@formio/appserver");
+/** Where the curated image keeps its plugins file. */
+exports.DEFAULT_PLUGINS_FILE = '/app/plugins/index.mjs';
+/**
+ * The plugins file to load: UAG_PLUGINS (a path, or "off"), or else the
+ * image's default when it exists. Undefined when there is none.
+ */
+function pluginsFile(env = process.env, defaultFile = exports.DEFAULT_PLUGINS_FILE) {
+    const setting = String(env.UAG_PLUGINS || '').trim();
+    if (setting.toLowerCase() === 'off')
+        return undefined;
+    if (setting)
+        return path_1.default.resolve(setting);
+    return fs_1.default.existsSync(defaultFile) ? defaultFile : undefined;
+}
+/** A real dynamic import(), kept one after compilation to CommonJS. */
+const dynamicImport = new Function('specifier', 'return import(specifier)');
+/**
+ * The keys only a module has: an entry with one of them (and no plugin `type`)
+ * is a module, anything else a plugin. Plugins have a `name` and `settings`
+ * too, so a name says nothing, and `settings` counts only as a function.
+ * (`uag` too, so a module written for the old shape is refused saying what to
+ * change, not read as a plugin.)
+ */
+const MODULE_KEYS = ['uag', 'routes', 'components', 'initialize', 'plugins', 'pluginTypes', 'config', 'formRoutes', 'actions', 'hooks', 'providers', 'processors'];
+const isModule = (entry) => !!entry && typeof entry === 'object' && !Array.isArray(entry) && typeof entry.type !== 'string'
+    && (MODULE_KEYS.some((key) => key in entry) || typeof entry.settings === 'function');
+/**
+ * Load the plugins file, if any, and use it, named by its path in error
+ * messages. Its default export is a plugin or a list of plugins; or a module
+ * (an extra, with its plugins), or a list mixing modules and
+ * plugins, each module used in turn. Plugins are trusted code: they run in
+ * the UAG process.
+ */
+async function usePluginFiles(server, env = process.env, defaultFile = exports.DEFAULT_PLUGINS_FILE) {
+    const file = pluginsFile(env, defaultFile);
+    if (!file)
+        return undefined;
+    const exported = (await dynamicImport((0, url_1.pathToFileURL)(file).href))?.default;
+    const entries = [].concat(exported ?? []);
+    if (entries.some(isModule)) {
+        const plugins = entries.filter((entry) => !isModule(entry));
+        server.use([...entries.filter(isModule), ...(plugins.length ? [{ plugins }] : [])], file);
+        console.log(`[formio-uag] Plugins and modules loaded from ${file}.`);
+        return file;
+    }
+    const plugins = await (0, appserver_1.loadPluginFile)(file);
+    server.use({ plugins }, file);
+    console.log(`[formio-uag] Plugins loaded from ${file}.`);
+    return file;
+}

package/lib/plugins.d.ts

@@ -0,0 +1,17 @@
+import type { PluginLog, PluginRegistry, PluginType } from '@formio/appserver';
+import type { UAGProjectInterface } from './UAGProjectInterface';
+/**
+ * Engines and MCP tools, listed in the plugin registry (phase 1). The UAG keeps
+ * loading them its own way (UAG_ENGINES, config.tools); the registry only lists
+ * them, so they appear in GET {project}/plugins. Settings values are never listed.
+ */
+export declare const enginePluginType: PluginType;
+export declare const toolPluginType: PluginType;
+/** List each loaded engine type: its instances, kinds, what agent instances read, and whether they generate. */
+export declare function listEngines(registry: PluginRegistry, project: UAGProjectInterface): void;
+/**
+ * List every MCP tool the project serves (built-in, then config.tools). A name
+ * served twice is listed once, with a warning: listing never refuses what the
+ * tool loader allows.
+ */
+export declare function listTools(registry: PluginRegistry, project: UAGProjectInterface, log?: PluginLog): Promise<void>;

package/lib/plugins.js

@@ -0,0 +1,93 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.toolPluginType = exports.enginePluginType = void 0;
+exports.listEngines = listEngines;
+exports.listTools = listTools;
+const uag_sdk_1 = require("@formio/uag-sdk");
+const tools_1 = require("./agent/tools");
+const engines_1 = require("./agent/engines");
+const version_1 = require("./version");
+/**
+ * Engines and MCP tools, listed in the plugin registry (phase 1). The UAG keeps
+ * loading them its own way (UAG_ENGINES, config.tools); the registry only lists
+ * them, so they appear in GET {project}/plugins. Settings values are never listed.
+ */
+exports.enginePluginType = {
+    type: 'engine',
+    title: 'Engine',
+    apiVersions: [1],
+    editForm: 'none',
+    // A misconfigured engine stops startup.
+    unavailable: 'error',
+    registration: 'internal',
+    stability: 'stable',
+    validate: (plugin) => uag_sdk_1.ENGINE_NAME.test(plugin.name) ? [] : [`engine name "${plugin.name}" must be lowercase letters, digits and "-", starting with a letter`],
+    manifest: (plugin) => ({
+        kinds: plugin.kinds,
+        accepts: plugin.accepts,
+        generate: plugin.generate,
+        instances: plugin.instances
+    })
+};
+exports.toolPluginType = {
+    type: 'tool',
+    title: 'MCP tool',
+    apiVersions: [1],
+    editForm: 'none',
+    unavailable: 'error',
+    registration: 'internal',
+    stability: 'stable',
+    validate: (plugin) => /^[A-Za-z][A-Za-z0-9_-]*$/.test(plugin.name) ? [] : [`tool name "${plugin.name}" must start with a letter and hold only letters, digits, "_" and "-"`],
+    manifest: (plugin) => ({ annotations: plugin.annotations ?? null })
+};
+const semver = (version) => (version && /^\d+\.\d+\.\d+/.test(version)) ? version : version_1.UAG_VERSION;
+/** List each loaded engine type: its instances, kinds, what agent instances read, and whether they generate. */
+function listEngines(registry, project) {
+    const engines = project.formflowEngines();
+    const entries = (0, engines_1.engineTypesOf)(engines).map((type) => {
+        const instances = type.instances.map((name) => engines.get(name)).filter(Boolean);
+        const agents = instances.filter((engine) => engine.kind === 'agent');
+        const accepts = [...new Set(agents.flatMap((engine) => Array.isArray(engine.accepts) ? engine.accepts : ['text']))];
+        return {
+            type: 'engine',
+            name: type.name,
+            title: type.title,
+            version: semver(type.version),
+            apiVersion: 1,
+            kinds: type.kinds,
+            accepts,
+            generate: agents.some((engine) => typeof engine.generate === 'function'),
+            instances: type.instances
+        };
+    });
+    registry.addType(exports.enginePluginType, '@formio/uag');
+    registry.add(entries, 'UAG_ENGINES', { server: true });
+}
+/**
+ * List every MCP tool the project serves (built-in, then config.tools). A name
+ * served twice is listed once, with a warning: listing never refuses what the
+ * tool loader allows.
+ */
+async function listTools(registry, project, log = console) {
+    const seen = new Set();
+    const entries = [];
+    for (const tool of await (0, tools_1.allTools)(project)) {
+        const name = String(tool.name);
+        if (seen.has(name)) {
+            log.warn(`[plugins] tool:${name} is defined more than once; it is listed once`);
+            continue;
+        }
+        seen.add(name);
+        entries.push({
+            type: 'tool',
+            name,
+            title: tool.title || name,
+            description: tool.description,
+            version: version_1.UAG_VERSION,
+            apiVersion: 1,
+            annotations: tool.annotations
+        });
+    }
+    registry.addType(exports.toolPluginType, '@formio/uag');
+    registry.add(entries, 'config.tools', { server: true });
+}

package/lib/richText.d.ts

@@ -0,0 +1,20 @@
+import type { Component } from '@formio/core';
+/**
+ * HTML for rich text fields (textareas with the CKEditor or Quill editor).
+ *
+ * What an agent engine writes into such a field is reduced to the markup the
+ * editors keep, because the engine's input can be steered (a document it reads
+ * is someone else's text), and the field is later rendered for reviewers. The
+ * HTML is parsed and written out again: allowed elements with their allowed
+ * attributes, text escaped, everything else dropped. An element that is not
+ * allowed is dropped and its children kept, except for elements whose content
+ * is never text to show (scripts, styles, frames, embedded documents).
+ */
+/** Allowed elements, each with the attributes it may keep. */
+export declare const RICH_TEXT_TAGS: Record<string, string[]>;
+/** Whether a component is a rich text field, whose value is HTML. */
+export declare function isRichText(component: Component | undefined): boolean;
+/** Reduce HTML to the markup rich text editors keep (RICH_TEXT_TAGS). */
+export declare function sanitizeRichText(html: string): string;
+/** The rule agents are given for a rich text field's value. */
+export declare const RICH_TEXT_RULE: string;

package/lib/richText.js

@@ -0,0 +1,99 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RICH_TEXT_RULE = exports.RICH_TEXT_TAGS = void 0;
+exports.isRichText = isRichText;
+exports.sanitizeRichText = sanitizeRichText;
+const node_html_parser_1 = require("node-html-parser");
+/**
+ * HTML for rich text fields (textareas with the CKEditor or Quill editor).
+ *
+ * What an agent engine writes into such a field is reduced to the markup the
+ * editors keep, because the engine's input can be steered (a document it reads
+ * is someone else's text), and the field is later rendered for reviewers. The
+ * HTML is parsed and written out again: allowed elements with their allowed
+ * attributes, text escaped, everything else dropped. An element that is not
+ * allowed is dropped and its children kept, except for elements whose content
+ * is never text to show (scripts, styles, frames, embedded documents).
+ */
+/** Allowed elements, each with the attributes it may keep. */
+exports.RICH_TEXT_TAGS = {
+    p: [], br: [], hr: [],
+    strong: [], b: [], em: [], i: [], u: [], s: [], sub: [], sup: [], code: [], pre: [],
+    h1: [], h2: [], h3: [], h4: [], h5: [], h6: [],
+    ul: [], ol: [], li: [],
+    blockquote: [],
+    a: ['href'],
+    figure: [], table: [], thead: [], tbody: [], tfoot: [], tr: [],
+    th: ['colspan', 'rowspan'], td: ['colspan', 'rowspan']
+};
+/** Elements dropped with everything inside them. */
+const DROPPED_WITH_CONTENT = new Set(['script', 'style', 'noscript', 'template', 'iframe', 'frame', 'frameset', 'object', 'embed', 'svg', 'math', 'head', 'title', 'textarea', 'select']);
+const VOID = new Set(['br', 'hr']);
+/** Whether a component is a rich text field, whose value is HTML. */
+function isRichText(component) {
+    const editor = component?.editor;
+    return component?.type === 'textarea' && (editor === 'ckeditor' || editor === 'quill');
+}
+function escapeText(text) {
+    return text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+}
+function escapeAttribute(value) {
+    return escapeText(value).replace(/"/g, '&quot;');
+}
+/** Only links to web pages and mail addresses: never `javascript:`, `data:` or the like. */
+function safeHref(value) {
+    const href = String(value || '').replace(/[\u0000-\u001F\u007F\s]+/g, '');
+    return /^(https?:\/\/|mailto:)/i.test(href) ? String(value).trim() : null;
+}
+function attributesOf(element, allowed) {
+    let out = '';
+    for (const name of allowed) {
+        const raw = element.getAttribute(name);
+        if (raw === undefined || raw === null)
+            continue;
+        let value = raw;
+        if (name === 'href')
+            value = safeHref(raw);
+        if ((name === 'colspan' || name === 'rowspan'))
+            value = /^\d{1,3}$/.test(raw.trim()) ? raw.trim() : null;
+        if (value === null)
+            continue;
+        out += ` ${name}="${escapeAttribute(value)}"`;
+    }
+    return out;
+}
+function write(node) {
+    if (node.nodeType === node_html_parser_1.NodeType.TEXT_NODE) {
+        // `text` is the decoded text; it is escaped once, here.
+        return escapeText(node.text);
+    }
+    if (node.nodeType !== node_html_parser_1.NodeType.ELEMENT_NODE)
+        return '';
+    const element = node;
+    const children = () => element.childNodes.map(write).join('');
+    const tag = String(element.rawTagName || '').toLowerCase();
+    if (!tag)
+        return children();
+    if (DROPPED_WITH_CONTENT.has(tag))
+        return '';
+    const allowed = exports.RICH_TEXT_TAGS[tag];
+    if (!allowed)
+        return children();
+    const open = `<${tag}${attributesOf(element, allowed)}>`;
+    return VOID.has(tag) ? open : `${open}${children()}</${tag}>`;
+}
+/** Reduce HTML to the markup rich text editors keep (RICH_TEXT_TAGS). */
+function sanitizeRichText(html) {
+    const root = (0, node_html_parser_1.parse)(String(html ?? ''), {
+        comment: false,
+        lowerCaseTagName: true,
+        // Read as raw text, so their content is never parsed as markup; they are dropped.
+        blockTextElements: { script: true, style: true, noscript: true }
+    });
+    return root.childNodes.map(write).join('').trim();
+}
+/** The rule agents are given for a rich text field's value. */
+exports.RICH_TEXT_RULE = 'The value is HTML for a rich text editor. Use only these elements: ' +
+    Object.keys(exports.RICH_TEXT_TAGS).join(', ') +
+    ', with no attributes except href on a (http, https or mailto links only) and colspan and rowspan on td and th. ' +
+    'No other elements, styles, classes, images or scripts, which are removed. Write HTML, not Markdown, and do not wrap it in a code fence.';

package/lib/router.d.ts

@@ -1,3 +1,12 @@
 import { Router } from 'express';
 import { UAGProjectInterface } from './UAGProjectInterface';
+import type { ScopedTokenClaims } from './formflow';
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+/**
+ * Hold a scoped token to its tools whatever built the server: a project
+ * interface a module replaces may override buildMcpServer() and register tools
+ * without the scope it was given. Every tool the token does not list, and every
+ * tool not annotated read-only, is disabled, so it is neither listed nor run.
+ */
+export declare function restrictToScope(server: McpServer, scope: Pick<ScopedTokenClaims, 'tools'>): void;
 export declare function UAGRouter(project: UAGProjectInterface): Router;

package/lib/router.js

@@ -1,5 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.restrictToScope = restrictToScope;
 exports.UAGRouter = UAGRouter;
 const express_1 = require("express");
 const streamableHttp_js_1 = require("@modelcontextprotocol/sdk/server/streamableHttp.js");
@@ -30,6 +31,22 @@ const handleResponseError = (error, res) => {
     res.status(500).json(createJsonRpcErrorResponse(-32603, "Internal Server Error"));
     return false;
 };
+/**
+ * Hold a scoped token to its tools whatever built the server: a project
+ * interface a module replaces may override buildMcpServer() and register tools
+ * without the scope it was given. Every tool the token does not list, and every
+ * tool not annotated read-only, is disabled, so it is neither listed nor run.
+ */
+function restrictToScope(server, scope) {
+    const allowed = new Set(scope.tools || []);
+    // The SDK keeps registered tools here; it has no public list of them.
+    const registered = server._registeredTools || {};
+    for (const [name, tool] of Object.entries(registered)) {
+        if (!allowed.has(name) || tool?.annotations?.readOnlyHint !== true) {
+            tool.disable?.();
+        }
+    }
+}
 /**
  * Build a per-request McpServer + transport (stateless mode) and tear both down
  * when the response closes. A fresh instance per request keeps concurrent requests
@@ -37,6 +54,8 @@ const handleResponseError = (error, res) => {
  */
 const connectMcpTransport = async (project, res, scope) => {
     const server = await project.buildMcpServer(scope);
+    if (scope)
+        restrictToScope(server, scope);
     const transport = new streamableHttp_js_1.StreamableHTTPServerTransport({
         sessionIdGenerator: undefined, // Use "undefined" to trigger stateless mode.
     });

package/lib/testing.d.ts

@@ -0,0 +1,74 @@
+/**
+ * @formio/uag/testing: test a module without a Form.io project or a running UAG.
+ *
+ *   const { createTestModuleApi, testForm, testAuth, mountModuleRoutes } = require('@formio/uag/testing');
+ *
+ *   const api = await createTestModuleApi(MyModule, { config: { myModule: { max: 2 } } });
+ *   const app = await mountModuleRoutes(MyModule, { form: testForm(definition), auth: testAuth() });
+ *   await supertest(app).post('/notes/my-module/check').send({ ... });
+ *
+ * The API is the UAG's own (errors, stream, forms, converters, engine handles,
+ * Markdown, callerKey), with stand-ins only for what needs a project: the
+ * caller's access (from `testAuth`), the license (`licensed`), engines
+ * (`engines`) and stored files (`testForm` reads base64 values; `files` for
+ * others). Settings go through the module's own `settings`, with the server's rules.
+ */
+import express from 'express';
+import type { AuthRequest, ModuleForm, ServerModule } from '@formio/appserver';
+import type { UAGModuleApi } from './modules/types';
+/** An engine for tests: an agent engine that generates. */
+export type TestEngine = {
+    name: string;
+    model?: string;
+    accepts?: Array<'text' | 'pdf'>;
+    generate(request: any): any;
+    kind?: 'agent';
+};
+export type TestModuleOptions = {
+    /** The settings as given, instead of running the module's `settings`. */
+    settings?: Record<string, any>;
+    /** The configuration and environment the module's `settings` gets ({} by default). */
+    config?: Record<string, any>;
+    env?: Record<string, string | undefined>;
+    /** Plugins to register beside the module's own (document plugins, converters). */
+    plugins?: any[];
+    /** The engines `api.engines.select()` chooses from. */
+    engines?: TestEngine[];
+    /** Whether Agentic Formflow is licensed (true): false makes `requireLicense` and engine handles refuse with 403. */
+    licensed?: boolean;
+    /** Reads a stored file value that is not base64 (the test form reads those). */
+    files?: (value: any, options: {
+        component?: any;
+        maxBytes?: number;
+    }) => Buffer | null | Promise<Buffer | null>;
+};
+/** A form for tests, as module routes get it: `definition` is the form's JSON. */
+export declare function testForm(definition: Record<string, any>, options?: {
+    id?: string;
+    files?: TestModuleOptions['files'];
+}): ModuleForm & {
+    readFile(value: any, auth: any, options?: any): Promise<Buffer | null>;
+};
+/**
+ * A caller for tests: a user with these permissions on every form (create and
+ * update by default), as a list (`['read']`) or as `formPermissions()` gives
+ * them (`{ read: true }`).
+ */
+export declare function testAuth(options?: {
+    user?: Record<string, any>;
+    permissions?: Array<'create' | 'read' | 'update' | 'delete'> | Partial<Record<'create' | 'read' | 'update' | 'delete', boolean>>;
+    admin?: boolean;
+}): AuthRequest;
+/** The UAG's module API for a module, in a test. */
+export declare function createTestModuleApi<Settings extends Record<string, any> = Record<string, any>>(module: ServerModule<any>, options?: TestModuleOptions): Promise<UAGModuleApi<Settings>>;
+/**
+ * The module's form routes on an Express app, as the UAG mounts them on a
+ * form (by its path, and by `form/<id>`), for supertest: with the form, the
+ * caller, a request id, the access check and the UAG's error shape. No rate
+ * limits. `api` is created with `options` unless given.
+ */
+export declare function mountModuleRoutes(module: ServerModule<any>, options: TestModuleOptions & {
+    form: ReturnType<typeof testForm>;
+    auth?: AuthRequest;
+    api?: UAGModuleApi;
+}): Promise<express.Express>;