@formio/uag 2.0.0-rc.5 → 2.0.0-rc.7

package/lib/testing.js

@@ -0,0 +1,165 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.testForm = testForm;
+exports.testAuth = testAuth;
+exports.createTestModuleApi = createTestModuleApi;
+exports.mountModuleRoutes = mountModuleRoutes;
+/**
+ * @formio/uag/testing: test a module without a Form.io project or a running UAG.
+ *
+ *   const { createTestModuleApi, testForm, testAuth, mountModuleRoutes } = require('@formio/uag/testing');
+ *
+ *   const api = await createTestModuleApi(MyModule, { config: { myModule: { max: 2 } } });
+ *   const app = await mountModuleRoutes(MyModule, { form: testForm(definition), auth: testAuth() });
+ *   await supertest(app).post('/notes/my-module/check').send({ ... });
+ *
+ * The API is the UAG's own (errors, stream, forms, converters, engine handles,
+ * Markdown, callerKey), with stand-ins only for what needs a project: the
+ * caller's access (from `testAuth`), the license (`licensed`), engines
+ * (`engines`) and stored files (`testForm` reads base64 values; `files` for
+ * others). Settings go through the module's own `settings`, with the server's rules.
+ */
+const express_1 = __importDefault(require("express"));
+const crypto_1 = require("crypto");
+const appserver_1 = require("@formio/appserver");
+const formflow_1 = require("./formflow");
+const plugins_1 = require("./documents/plugins");
+const errors_1 = require("./agent/errors");
+const version_1 = require("./version");
+const api_1 = require("./modules/api");
+const routes_1 = require("./modules/routes");
+/** A form for tests, as module routes get it: `definition` is the form's JSON. */
+function testForm(definition, options = {}) {
+    const form = {
+        id: options.id || String(definition._id || 'test-form'),
+        name: String(definition.name || definition.path || 'test'),
+        path: String(definition.path || definition.name || 'test'),
+        title: String(definition.title || definition.name || ''),
+        definition,
+        async readFile(value, _auth, readOptions = {}) {
+            const url = String(value?.url || '');
+            if (url.startsWith('data:')) {
+                const bytes = Buffer.from(url.slice(url.indexOf(',') + 1), url.includes(';base64,') ? 'base64' : 'utf8');
+                if (readOptions.maxBytes !== undefined && bytes.length > readOptions.maxBytes) {
+                    throw new Error(`The file "${value?.originalName || value?.name || 'unnamed'}" is larger than the ${readOptions.maxBytes} bytes that may be read.`);
+                }
+                return bytes;
+            }
+            if (options.files)
+                return options.files(value, readOptions);
+            throw new Error('testForm reads base64 (data: URL) files only: give createTestModuleApi or mountModuleRoutes a "files" function for others.');
+        }
+    };
+    return form;
+}
+/**
+ * A caller for tests: a user with these permissions on every form (create and
+ * update by default), as a list (`['read']`) or as `formPermissions()` gives
+ * them (`{ read: true }`).
+ */
+function testAuth(options = {}) {
+    const given = options.permissions || ['create', 'update'];
+    const granted = { create: false, read: false, update: false, delete: false };
+    if (Array.isArray(given)) {
+        for (const method of given)
+            granted[method] = true;
+    }
+    else if (given && typeof given === 'object') {
+        for (const method of Object.keys(granted))
+            granted[method] = given[method] === true;
+    }
+    else {
+        throw new Error('testAuth: "permissions" is a list of create, read, update and delete, or { read: true, ... }');
+    }
+    return {
+        user: options.user || { _id: 'test-user' },
+        admin: !!options.admin,
+        formPermissions: () => (options.admin ? { create: true, read: true, update: true, delete: true } : granted)
+    };
+}
+/** The module, checked for what a test needs: a name and an apiVersion. */
+function loadedOf(module, settings) {
+    if (!module || typeof module.name !== 'string' || module.apiVersion === undefined) {
+        throw new Error('The module needs a "name" and an "apiVersion" to be tested as a module.');
+    }
+    return { definition: module, name: module.name, source: 'test', settings };
+}
+/** The module's settings, as the server resolves them. */
+function settingsOf(module, options) {
+    if (options.settings)
+        return Object.freeze({ ...options.settings });
+    const settings = module.settings ? module.settings(options.config || {}, options.env || {}) : {};
+    if (settings && typeof settings.then === 'function')
+        throw new Error(`Module ${module.name}: "settings" returned a promise; it must return the settings themselves (no async)`);
+    return Object.freeze({ ...(settings || {}) });
+}
+/** The UAG's module API for a module, in a test. */
+async function createTestModuleApi(module, options = {}) {
+    const loaded = loadedOf(module, settingsOf(module, options));
+    const registry = new appserver_1.PluginRegistry();
+    (0, plugins_1.registerDocuments)(registry);
+    const licensed = options.licensed !== false;
+    const requireLicense = async () => {
+        if (!licensed)
+            throw new formflow_1.FormflowError(403, 'Agentic Formflow requires a license (the test API was created with licensed: false).', 'license_required');
+    };
+    const engines = (options.engines || []).map((engine) => ({ kind: 'agent', model: engine.name, accepts: ['text'], ...engine }));
+    const base = (0, appserver_1.createBaseModuleApi)(loaded, registry, { name: () => 'uag', version: () => version_1.UAG_VERSION, errorBody: routes_1.uagErrorBody });
+    const api = (0, api_1.uagModuleApi)(base, {
+        registry: () => registry,
+        engines: () => ({ get: (name) => engines.find((engine) => engine.name === name), agent: engines.filter((engine) => engine.kind === 'agent') }),
+        requireLicense
+    });
+    const pluginTypes = typeof module.pluginTypes === 'function' ? module.pluginTypes(api) : (module.pluginTypes || []);
+    for (const type of pluginTypes)
+        registry.addType(type, module.name);
+    if (module.plugins)
+        registry.add(module.plugins, module.name);
+    if (options.plugins?.length)
+        registry.add(options.plugins, 'test');
+    await registry.initialize({ server: {} });
+    return api;
+}
+/**
+ * The module's form routes on an Express app, as the UAG mounts them on a
+ * form (by its path, and by `form/<id>`), for supertest: with the form, the
+ * caller, a request id, the access check and the UAG's error shape. No rate
+ * limits. `api` is created with `options` unless given.
+ */
+async function mountModuleRoutes(module, options) {
+    const api = options.api || await createTestModuleApi(module, options);
+    const loaded = loadedOf(module, api.settings);
+    const app = (0, express_1.default)();
+    app.use(express_1.default.json({ limit: '50mb' }));
+    const host = {
+        errorBody: routes_1.uagErrorBody,
+        routeHandlers: (_project, route) => ({
+            before: [(req, res, next) => {
+                    req.requestId = req.get('x-request-id') || (0, crypto_1.randomUUID)();
+                    res.set('X-Request-Id', req.requestId);
+                    next();
+                }],
+            wrap: (run) => async (req, res) => {
+                try {
+                    await run(req, res);
+                }
+                catch (err) {
+                    const { status, body } = (0, errors_1.agentError)(req.requestId, err);
+                    res.status(status).json(body);
+                }
+            }
+        })
+    };
+    const router = (0, appserver_1.moduleFormRouter)({}, loaded, api, host);
+    const withForm = (req, res, next) => {
+        req.form = options.form;
+        req.auth = options.auth || testAuth();
+        next();
+    };
+    app.use(`/${options.form.path}`, withForm, router);
+    app.use(`/form/${options.form.id}`, withForm, router);
+    return app;
+}

package/lib/tools/index.d.ts

@@ -11,7 +11,8 @@ import { UAGProjectInterface } from "../UAGProjectInterface";
 import { agentProvide } from "./agentProvide";
 import { agentDecide } from "./agentDecide";
 import { fetchExternalData } from "./fetchExternalData";
+import { readDocument } from "./readDocument";
 export * from './utils';
 export * from './annotations';
-export { getForms, getFormFields, getFieldInfo, collectData, confirmSubmission, submitCompletedForm, findSubmission, submissionUpdate, fetchExternalData, agentProvide, agentDecide };
+export { getForms, getFormFields, getFieldInfo, collectData, confirmSubmission, submitCompletedForm, findSubmission, submissionUpdate, fetchExternalData, agentProvide, agentDecide, readDocument };
 export declare const getTools: (project: UAGProjectInterface) => Promise<ToolInfo[]>;

package/lib/tools/index.js

@@ -14,7 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getTools = exports.agentDecide = exports.agentProvide = exports.fetchExternalData = exports.submissionUpdate = exports.findSubmission = exports.submitCompletedForm = exports.confirmSubmission = exports.collectData = exports.getFieldInfo = exports.getFormFields = exports.getForms = void 0;
+exports.getTools = exports.readDocument = exports.agentDecide = exports.agentProvide = exports.fetchExternalData = exports.submissionUpdate = exports.findSubmission = exports.submitCompletedForm = exports.confirmSubmission = exports.collectData = exports.getFieldInfo = exports.getFormFields = exports.getForms = void 0;
 const getForms_1 = require("./getForms");
 Object.defineProperty(exports, "getForms", { enumerable: true, get: function () { return getForms_1.getForms; } });
 const getFormFields_1 = require("./getFormFields");
@@ -37,6 +37,8 @@ const agentDecide_1 = require("./agentDecide");
 Object.defineProperty(exports, "agentDecide", { enumerable: true, get: function () { return agentDecide_1.agentDecide; } });
 const fetchExternalData_1 = require("./fetchExternalData");
 Object.defineProperty(exports, "fetchExternalData", { enumerable: true, get: function () { return fetchExternalData_1.fetchExternalData; } });
+const readDocument_1 = require("./readDocument");
+Object.defineProperty(exports, "readDocument", { enumerable: true, get: function () { return readDocument_1.readDocument; } });
 __exportStar(require("./utils"), exports);
 __exportStar(require("./annotations"), exports);
 const getTools = async (project) => {
@@ -51,7 +53,8 @@ const getTools = async (project) => {
         (0, submissionUpdate_1.submissionUpdate)(project),
         (0, agentProvide_1.agentProvide)(project),
         (0, agentDecide_1.agentDecide)(project),
-        (0, fetchExternalData_1.fetchExternalData)(project)
+        (0, fetchExternalData_1.fetchExternalData)(project),
+        (0, readDocument_1.readDocument)(project)
     ]);
 };
 exports.getTools = getTools;

package/lib/tools/readDocument.d.ts

@@ -0,0 +1,17 @@
+import type { UAGProjectInterface } from '../UAGProjectInterface';
+import { ToolInfo } from './utils';
+/**
+ * read_document's limits: the largest file it reads (`documents.maxFileBytes`,
+ * else DOCUMENTS_MAX_FILE_BYTES, 25 MB), and the most characters of text it
+ * returns (`agentApi.files.maxChars`, as task inputs are cut, 200000).
+ */
+export declare function readDocumentLimits(config: any, env?: NodeJS.ProcessEnv): {
+    maxBytes: number;
+    maxChars: number;
+};
+/**
+ * read_document: the text of an uploaded document in a submission (a file in a
+ * file component, or in an AI Document's files), read through the installed
+ * converters with the caller's identity. Read-only.
+ */
+export declare const readDocument: (project: UAGProjectInterface) => Promise<ToolInfo>;

package/lib/tools/readDocument.js

@@ -0,0 +1,99 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readDocument = void 0;
+exports.readDocumentLimits = readDocumentLimits;
+const zod_1 = __importDefault(require("zod"));
+const lodash_1 = require("lodash");
+const core_1 = require("@formio/core");
+const template_1 = require("../template");
+const annotations_1 = require("./annotations");
+const readers_1 = require("../documents/readers");
+const DEFAULT_MAX_FILE_BYTES = 25 * 1024 * 1024;
+const DEFAULT_MAX_CHARS = 200000;
+const positive = (value) => (Number(value) > 0 ? Number(value) : undefined);
+/**
+ * read_document's limits: the largest file it reads (`documents.maxFileBytes`,
+ * else DOCUMENTS_MAX_FILE_BYTES, 25 MB), and the most characters of text it
+ * returns (`agentApi.files.maxChars`, as task inputs are cut, 200000).
+ */
+function readDocumentLimits(config, env = process.env) {
+    return {
+        maxBytes: positive(config?.documents?.maxFileBytes) || positive(env.DOCUMENTS_MAX_FILE_BYTES) || DEFAULT_MAX_FILE_BYTES,
+        maxChars: positive(config?.agentApi?.files?.maxChars) || DEFAULT_MAX_CHARS
+    };
+}
+/** The component at a data path, row indices removed (`rows[2].contract` is `rows.contract`). */
+function componentAt(components, dataPath) {
+    const path = dataPath.replace(/\[\d+\]/g, '');
+    let found;
+    // The data path: with every component included, the second argument also names layout components (panels, columns).
+    core_1.Utils.eachComponent(components, (component, componentPath, _components, _parent, paths) => {
+        if (!found && (paths?.path ?? componentPath) === path)
+            found = component;
+    }, true);
+    return found;
+}
+/**
+ * read_document: the text of an uploaded document in a submission (a file in a
+ * file component, or in an AI Document's files), read through the installed
+ * converters with the caller's identity. Read-only.
+ */
+const readDocument = async (project) => {
+    return (0, lodash_1.defaultsDeep)({}, project.config?.toolOverrides?.read_document, {
+        name: 'read_document',
+        title: 'Read Uploaded Document',
+        description: 'Read the text of a document uploaded to a submission: a file in a file field, or in an AI Document field. ' +
+            'Give the form, the submission id (from `find_submissions`) and the data path of the field (such as "contract" or "rows[0].attachment"; ' +
+            'add an index, such as "contract[1]", for another file than the first). Returns the file name, its type, its page count when known, and its text; ' +
+            'a long text is cut, and `truncated` is true.',
+        annotations: (0, annotations_1.reads)('Read an uploaded document'),
+        inputSchema: {
+            form_name: zod_1.default.string().describe('The name of the form, as `get_forms` lists it.'),
+            submission_id: zod_1.default.string().describe('The full ID of the submission, as `find_submissions` gives it.'),
+            data_path: zod_1.default.string().describe('The data path of the file field or AI Document field.')
+        },
+        execute: async ({ form_name, submission_id, data_path }, extra) => {
+            const form = await project.getForm(form_name);
+            if (!form) {
+                return project.mcpResponse(template_1.ResponseTemplate.formNotFound, { formName: form_name }, true);
+            }
+            const submission = await form.loadSubmission(submission_id, extra.authInfo);
+            if (!submission) {
+                return project.mcpResponse(template_1.ResponseTemplate.submissionNotFound, { submissionId: submission_id, form: form.form }, true);
+            }
+            // An index on the last segment picks a file of a multiple field: contract[1].
+            const index = /\[(\d+)\]$/.exec(data_path);
+            const fieldPath = index ? data_path.slice(0, index.index) : data_path;
+            const value = (0, lodash_1.get)(submission.data, fieldPath);
+            const isDocument = value && typeof value === 'object' && !Array.isArray(value) && Array.isArray(value.files);
+            const files = isDocument ? value.files : Array.isArray(value) ? value : (value && typeof value === 'object' ? [value] : []);
+            const file = files[index ? Number(index[1]) : 0];
+            const component = componentAt(form.form?.components || [], isDocument ? `${fieldPath}.files` : fieldPath);
+            if (!file || !component || component.type !== 'file') {
+                return project.mcpJSONResponse({ error: `No uploaded file at "${data_path}" in that submission. Give the data path of a file field or an AI Document field.` }, true);
+            }
+            if (!project.plugins) {
+                return project.mcpJSONResponse({ error: 'Documents cannot be read until the server has started.' }, true);
+            }
+            const limits = readDocumentLimits(project.config);
+            try {
+                const buffer = await form.readFile(file, extra.authInfo, { maxBytes: limits.maxBytes, component });
+                if (!buffer) {
+                    return project.mcpJSONResponse({ error: `The file is stored as ${file.storage || 'unknown'}, which cannot be read here.` }, true);
+                }
+                const name = String(file.originalName || file.name || 'file');
+                const read = await (0, readers_1.readAsText)(project.plugins, buffer, { name, type: typeof file.type === 'string' ? file.type : undefined }, { signal: extra?.signal });
+                const truncated = read.text.length > limits.maxChars;
+                return project.mcpJSONResponse({ file_name: name, type: read.type, pages: read.pages ?? null, text: truncated ? read.text.slice(0, limits.maxChars) : read.text, truncated });
+            }
+            catch (err) {
+                // The message names the file and the reason, never its content.
+                return project.mcpJSONResponse({ error: err?.message || 'The file could not be read.' }, true);
+            }
+        }
+    });
+};
+exports.readDocument = readDocument;

package/lib/tools/submissionUpdate.js

@@ -35,10 +35,11 @@ const submissionUpdate = async (project) => {
                 const updateSummary = [];
                 for (const update of updates) {
                     const previousValue = (0, lodash_1.get)(updatedData, update.data_path);
-                    (0, lodash_1.set)(updatedData, update.data_path, update.new_value);
+                    const newValue = form.normalizeWrittenValue(update.data_path, update.new_value);
+                    (0, lodash_1.set)(updatedData, update.data_path, newValue);
                     updateSummary.push({
                         data_path: update.data_path,
-                        new_value: update.new_value,
+                        new_value: newValue,
                         previous_value: previousValue || ''
                     });
                 }

package/lib/tools/utils.d.ts

@@ -1,5 +1,7 @@
 import { ZodRawShape } from "zod";
 import { ToolAnnotations } from "./annotations";
+import type { AuthRequest } from "@formio/appserver";
+import type { UAGProjectInterface } from "../UAGProjectInterface";
 /**
  * Escape a literal value so it can be carried in a `__regex` query parameter.
  *
@@ -20,14 +22,52 @@ import { ToolAnnotations } from "./annotations";
  * @returns The value as a regular expression pattern that matches it literally.
  */
 export declare const escapeSearchPattern: (value: string) => string;
+/**
+ * What a tool's `execute` receives besides its arguments, on every path that
+ * runs it: MCP clients on `/mcp`, and Agent API research in process.
+ */
+export type ToolExtra = {
+    /** The caller's identity: who the work runs as. Pass it to form methods so Form.io applies its permissions. */
+    authInfo: AuthRequest;
+    /** The project serving the request. */
+    project: UAGProjectInterface;
+    /** For a tool from a module with a name and an apiVersion: the module's API (its settings, engines, ...). */
+    module?: import('../modules').UAGModuleApi;
+    /** On `/mcp`, the rest of the MCP SDK's request context (signal, request id, ...). */
+    [key: string]: any;
+};
 export type ToolInfo = {
     name?: string;
     title?: string;
     description?: string;
     inputSchema?: ZodRawShape | undefined;
     annotations?: ToolAnnotations;
+    /**
+     * Runs the tool: always called with its arguments (an empty object when it
+     * takes none), then a `ToolExtra`. Typed loosely so existing tools and their
+     * callers keep compiling; annotate `extra` as `ToolExtra` in your own tools.
+     */
     execute?: any;
+    /**
+     * Builds the project-specific part of the tool (`inputSchema`, `annotations`,
+     * `execute`, ...) for a project, as the built-in tools are built. The static
+     * `name`, `title` and `description` are used unless `bind` returns its own,
+     * so the tool's name is known without a project. Called whenever the tool
+     * list is built (every `/mcp` request and every research plan): keep it
+     * cheap, and load libraries and other slow state once, outside it.
+     */
+    bind?: (project: UAGProjectInterface) => Partial<ToolInfo> | Promise<Partial<ToolInfo>>;
 };
+/** An entry of `config.tools`: a tool, optionally bound to the project with `bind`. */
+export type ToolEntry = ToolInfo;
+/** The shape a `config.tools` entry takes, shown when one is a bare function. */
+export declare const BOUND_TOOL_SHAPE = "{ name: 'my_tool', title: 'My Tool', description: '...', bind(project) { return { inputSchema, annotations, execute }; } }";
+/**
+ * Refuse `config.tools` entries that are bare functions (the unreleased factory
+ * form): a tool is an object, whose `bind(project)` builds its project-specific
+ * part. Throws naming the entry and showing the shape.
+ */
+export declare function assertToolEntries(entries: unknown[] | undefined): void;
 export type ParentInfo = {
     type: string;
     label?: string;

package/lib/tools/utils.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.escapeSearchPattern = void 0;
+exports.BOUND_TOOL_SHAPE = exports.escapeSearchPattern = void 0;
+exports.assertToolEntries = assertToolEntries;
 const lodash_1 = require("lodash");
 /**
  * Escape a literal value so it can be carried in a `__regex` query parameter.
@@ -23,3 +24,17 @@ const lodash_1 = require("lodash");
  */
 const escapeSearchPattern = (value) => (0, lodash_1.escapeRegExp)(value).replace(/\//g, '\\x2f');
 exports.escapeSearchPattern = escapeSearchPattern;
+/** The shape a `config.tools` entry takes, shown when one is a bare function. */
+exports.BOUND_TOOL_SHAPE = "{ name: 'my_tool', title: 'My Tool', description: '...', bind(project) { return { inputSchema, annotations, execute }; } }";
+/**
+ * Refuse `config.tools` entries that are bare functions (the unreleased factory
+ * form): a tool is an object, whose `bind(project)` builds its project-specific
+ * part. Throws naming the entry and showing the shape.
+ */
+function assertToolEntries(entries) {
+    (entries || []).forEach((entry, index) => {
+        if (typeof entry === 'function') {
+            throw new Error(`config.tools[${index}] is a function. A custom tool is an object; build its project-specific part with bind(project): ${exports.BOUND_TOOL_SHAPE}`);
+        }
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@formio/uag",
-  "version": "2.0.0-rc.5",
+  "version": "2.0.0-rc.7",
   "mcpName": "io.form/formio-uag",
   "description": "The Form.io Universal Agent Gateway (UAG).",
   "main": "lib/index.js",
@@ -10,7 +10,9 @@
   },
   "files": [
     "lib",
-    "bin"
+    "bin",
+    "testing.js",
+    "testing.d.ts"
   ],
   "workspaces": [
     "engines/*",
@@ -43,9 +45,9 @@
   "author": "Form.io Developers",
   "license": "MIT",
   "dependencies": {
-    "@formio/appserver": "^2.25.2",
+    "@formio/appserver": "^2.26.0",
     "@formio/core": "2.9.0",
-    "@formio/uag-sdk": "^2.0.0-rc.5",
+    "@formio/uag-sdk": "^2.0.0-rc.7",
     "@modelcontextprotocol/sdk": "1.32.2",
     "cors": "^2.8.6",
     "debug": "^4.4.3",
@@ -53,9 +55,13 @@
     "express": "^5.3.0",
     "jsonwebtoken": "^9.0.3",
     "lodash": "^4.18.1",
+    "markdown-it": "^15.0.3",
     "mongodb": "^7.7.0",
     "node-cache": "^5.1.2",
     "node-html-markdown": "^2.0.0",
+    "node-html-parser": "^9.0.5",
+    "pdf-lib": "^1.17.1",
+    "unpdf": "^2.1.0",
     "zod": "^4.6.7"
   },
   "resolutions": {
@@ -72,7 +78,6 @@
     "@types/chai": "^5.2.3",
     "@types/cors": "^2.8.19",
     "@types/debug": "^4.1.13",
-    "@types/dotenv": "^8.2.3",
     "@types/express": "^5.0.6",
     "@types/jsdom": "^30.0.0",
     "@types/jsonwebtoken": "^9.0.10",

package/testing.d.ts

@@ -0,0 +1 @@
+export * from './lib/testing';

package/testing.js

@@ -0,0 +1,2 @@
+// @formio/uag/testing: test a module without a Form.io project or a running UAG.
+module.exports = require('./lib/testing');