@aigne/agent-library 1.10.0 → 1.11.0

package/CHANGELOG.md

@@ -1,6 +1,9 @@
-## [1.2.0](https://github.com/AIGNE-io/aigne-framework/compare/agent-library-v1.1.0...agent-library-v1.2.0) (2025-03-18)
+## [1.11.0](https://github.com/AIGNE-io/aigne-framework/compare/agent-library-v1.10.0...agent-library-v1.11.0) (2025-05-27)
 
-- chore: release 1.2.0
+
+### Features
+
+* add schema transform ([#35](https://github.com/AIGNE-io/aigne-framework/issues/35)) ([c7d9a2c](https://github.com/AIGNE-io/aigne-framework/commit/c7d9a2c9fcab8d384d4198db5ff6ba4603846cdf))
 
 ## [1.10.0](https://github.com/AIGNE-io/aigne-framework/compare/agent-library-v1.9.0...agent-library-v1.10.0) (2025-05-25)
 

package/lib/cjs/data-mapper/agents/mapper.d.ts

@@ -0,0 +1,14 @@
+import { AIAgent } from "@aigne/core";
+declare const mapper: AIAgent<{
+    sourceData: string;
+    responseSchema: string;
+    sourceSchema?: string | undefined;
+    instruction?: string | undefined;
+    responseData?: string | undefined;
+    feedback?: string | undefined;
+}, {
+    jsonata: string;
+    confidence: number;
+    confidenceReasoning: string;
+}>;
+export default mapper;

package/lib/cjs/data-mapper/agents/mapper.js

@@ -0,0 +1,63 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const core_1 = require("@aigne/core");
+const zod_1 = require("zod");
+const prompts_js_1 = require("../prompts.js");
+const mapper = core_1.AIAgent.from({
+    subscribeTopic: [core_1.UserInputTopic, "mapping_request"],
+    publishTopic: "review_request",
+    inputSchema: zod_1.z.object({
+        sourceData: zod_1.z.string(),
+        sourceSchema: zod_1.z.string().optional(),
+        responseSchema: zod_1.z.string(),
+        instruction: zod_1.z.string().optional(),
+        responseData: zod_1.z.string().optional(),
+        feedback: zod_1.z.string().optional(),
+    }),
+    outputSchema: zod_1.z.object({
+        jsonata: zod_1.z.string().describe("JSONata expression"),
+        confidence: zod_1.z.number().describe(`Confidence score for the JSONata expression between 0 and 100.
+      Give a low confidence score if there are missing fields in the source data.
+      Give a low confidence score if there are multiple options for a field and it is unclear which one to choose.`),
+        confidenceReasoning: zod_1.z.string().describe("Reasoning for the confidence score"),
+    }),
+    includeInputInOutput: true,
+    instructions: core_1.PromptBuilder.from({
+        messages: [
+            {
+                role: "assistant",
+                content: {
+                    type: "text",
+                    text: prompts_js_1.PROMPT_MAPPING,
+                },
+            },
+            {
+                role: "user",
+                content: {
+                    type: "text",
+                    text: `Given a source data and structure, create a jsonata expression in JSON FORMAT.
+                Important: The output should be a jsonata expression creating an object that matches the following schema:
+                {{responseSchema}}
+
+                Pay special attention to the requirements in field descriptions
+
+                The instruction from the user is: {{instruction}}
+
+                ------
+
+                Source Data Structure:
+                {{sourceSchema}}
+
+                Source data Sample:
+                {{sourceData}}
+
+                ------
+
+                Previous feedback:
+                {{feedback}}`,
+                },
+            },
+        ],
+    }),
+});
+exports.default = mapper;

package/lib/cjs/data-mapper/agents/reviewer.d.ts

@@ -0,0 +1,19 @@
+import { FunctionAgent } from "@aigne/core";
+declare const reviewer: FunctionAgent<{
+    sourceData: string;
+    responseSchema: string;
+    jsonata: string;
+}, {
+    success: boolean;
+    data: null;
+    feedback: string;
+} | {
+    success: boolean;
+    data: unknown;
+    feedback?: undefined;
+} | {
+    success: false;
+    data: unknown;
+    feedback: string | undefined;
+}>;
+export default reviewer;

package/lib/cjs/data-mapper/agents/reviewer.js

@@ -0,0 +1,47 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const core_1 = require("@aigne/core");
+const zod_1 = require("zod");
+const tools_js_1 = require("../tools.js");
+const reviewer = core_1.FunctionAgent.from({
+    name: "check_mapping",
+    description: "Check the mapping result",
+    subscribeTopic: ["review_request"],
+    publishTopic: (output) => (output.success ? core_1.UserOutputTopic : "mapping_request"),
+    inputSchema: zod_1.z.object({
+        sourceData: zod_1.z.string(),
+        jsonata: zod_1.z.string(),
+        responseSchema: zod_1.z.string(),
+    }),
+    process: async ({ sourceData, jsonata, responseSchema, }) => {
+        let parsedSourceData = null;
+        let parsedResponseSchema = null;
+        try {
+            parsedSourceData = sourceData ? JSON.parse(sourceData) : null;
+            parsedResponseSchema = responseSchema ? JSON.parse(responseSchema) : null;
+        }
+        catch (parseError) {
+            // input data is not valid JSON, return success
+            return {
+                success: true,
+                data: null,
+                feedback: `JSON parsing failed: ${parseError.message}`,
+            };
+        }
+        const transformation = await (0, tools_js_1.applyJsonataWithValidation)(parsedSourceData, jsonata, parsedResponseSchema);
+        // if transformation is successful, return success
+        if (transformation.success) {
+            return {
+                success: true,
+                data: transformation.data,
+            };
+        }
+        return {
+            success: transformation.success,
+            data: transformation.data,
+            feedback: transformation.error,
+        };
+    },
+    includeInputInOutput: true,
+});
+exports.default = reviewer;

package/lib/cjs/data-mapper/index.d.ts

@@ -0,0 +1,18 @@
+import { type ChatModel } from "@aigne/core";
+export interface TransformInput {
+    responseSchema: string;
+    responseSampleData?: string;
+    sourceData?: string;
+    sourceSchema?: string;
+    instruction?: string;
+    [key: string]: unknown;
+}
+export declare function generateMapping({ input, model, }: {
+    input: TransformInput;
+    model: ChatModel;
+}): Promise<{
+    jsonata: string;
+    confidence: number;
+    confidenceReasoning: string;
+} | null>;
+export { applyJsonata } from "./tools.js";

package/lib/cjs/data-mapper/index.js

@@ -0,0 +1,29 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.applyJsonata = void 0;
+exports.generateMapping = generateMapping;
+const core_1 = require("@aigne/core");
+const mapper_js_1 = __importDefault(require("./agents/mapper.js"));
+const reviewer_js_1 = __importDefault(require("./agents/reviewer.js"));
+const tools_js_1 = require("./tools.js");
+async function generateMapping({ input, model, }) {
+    if (!model)
+        throw new Error("model is required to run data mapper");
+    // if sourceSchema is not provided, generate it from sourceData
+    if (!input.sourceSchema && input.sourceData) {
+        input.sourceSchema = JSON.stringify((0, tools_js_1.getSchemaFromData)(JSON.parse(input.sourceData)));
+    }
+    const aigne = new core_1.AIGNE({ model, agents: [mapper_js_1.default, reviewer_js_1.default] });
+    aigne.publish(core_1.UserInputTopic, input);
+    const { message } = await aigne.subscribe(core_1.UserOutputTopic);
+    return {
+        jsonata: message.jsonata || "",
+        confidence: message.confidence || 0,
+        confidenceReasoning: message.confidenceReasoning || "",
+    };
+}
+var tools_js_2 = require("./tools.js");
+Object.defineProperty(exports, "applyJsonata", { enumerable: true, get: function () { return tools_js_2.applyJsonata; } });

package/lib/cjs/data-mapper/prompts.d.ts

@@ -0,0 +1 @@
+export declare const PROMPT_MAPPING = "You are an AI that generates JSONata mapping expressions to transform source data structures into target structures.\n\nGuidelines for creating JSONata mappings:\n\n1. Source References:\n   - Use exact field paths from the source data, e.g. $.merchant_category\n   - For accessing fields with names containing spaces, use backticks, e.g. $.`merchant category`\n   - Jsonata will automatically extract all the fields from the current context. E.g. if you need all variants from all products, you can use $.products.variants. No need to do nested map reduce operations.\n   - $. The variable with no name refers to the context value at any point in the input JSON hierarchy. E.g. if the current context is products.price, then $.currency is products.price.currency\n   - %. The parent of the current context value. E.g. if the current context is products.variants.size and you want variant name, use %.name\n\n   - When multiple source fields could map to a target, use a maximum of 3 fallbacks:\n     GOOD: source1 ? source1 : source2 ? source2 : source3 ? source3 : 'default'\n     BAD: source1 ? source1 : source1 ? source1 : source1 (repeated fields)\n\n2. Expression Rules:\n   - Avoid unnecessary array/string operations\n   - Each mapping should be clear and concise\n   - Use proper JSONata syntax for coalesce operations\n   - Do not use ~> to execute functions. Use the functions directly with the correct arguments or use $map(arr, $function) to apply a function to each element of an array.\n\n3. Array Handling:\n   - For mapping to an array of objects, use the following patterns:\n     a) When in array scope, use $.{} to map each object:\n        Correct: [$.{\"id\": id, \"name\": name}]\n        Incorrect: [{\"id\": $.id}]\n     b) When outside array scope, include the source path:\n        Correct: [$.items.{\"id\": id, \"name\": name}]\n        Incorrect: [{\"id\": $.items.id}]\n     c) For nested arrays, chain the array operators:\n        Correct: [products.variants.{\"size\": size, \"color\": color}]\n        Incorrect: [products.[{\"size\": variants.size}]]\n     d) You need to use the square brackets [] to map to an array of objects, otherwise it might return an object and fail the validation.\n        Correct: variants: [variants.{\"size\": size, \"color\": color}]\n        Incorrect: variants: variants.{\"size\": variants.size}\n   - For array elements, use JSONata array operators like [0] for first element, [-1] for last element\n   - Square bracket notation [] can be used with predicates, e.g. items[type='book']\n\n4. Field Selection Priority:\n   - Prefer variant-specific fields over general fields (e.g., sizeVariants.description over sizes)\n   - Choose the most specific/detailed field available (e.g., type=\"shelf\" over category=\"furniture\")\n\n5. Filters:\n   - Pay special attention to filter statements in the instruction and the schema description. Add them to the generated jsonata expression.\n     Example: Get me all products with SKU 0406654608 or products: {\"type\": \"array\", description: \"only products with SKU 0406654608\"}\n     Generated jsonata expression: Account.Order.Product[SKU = \"0406654608\"].{\"Description\": Description}\n   - For filtering with arrays, you can use the \"in\" operator. E.g. library.books[\"Steven King\" in authors]\n\n6. Data Integrity:\n   - ONLY use fields that exist in the source data structure\n   - If no matching source field exists, leave the field undefined\n   - Never invent or assume the existence of fields not present in the source data\n\n7. Function Calls:\n   - You may use the following functions if prompted:\n      $string(arg) - Converts argument to string\n      $length(str) - Returns string length\n      $substring(str, start[, length]) - Extracts substring\n      $substringBefore(str, chars) - Gets substring before specified chars\n      $substringAfter(str, chars) - Gets substring after specified chars\n      $uppercase(str) - Converts to uppercase\n      $lowercase(str) - Converts to lowercase\n      $trim(str) - Removes whitespace from both ends\n      $pad(str, width[, char]) - Pads string to specified width\n      $contains(str, substring) - Tests if string contains substring\n      $toMillis(timestamp [, picture]) - Converts ISO 8601 timestamp to milliseconds. E.g. $toMillis(\"2017-11-07T15:07:54.972Z\") => 1510067274972\n      $toDate(str | number) - Converts any timestamp string to valid ISO 8601 date string. E.g. $toDate(\"Oct 15, 2024 12:00:00 AM UTC\") => \"2024-10-15T00:00:00.000Z\", $toDate(1728873600000) => \"2024-10-15T00:00:00.000Z\"\n      $dateMax(arr) - Returns the maximum date of an array of dates. E.g. $dateMax([\"2017-11-07T15:07:54.972Z\", \"Oct 15, 2012 12:00:00 AM UTC\"]) returns \"2017-11-07T15:07:54.972Z\".\n      $dateMin(arr) - Returns the minimum date of an array of dates. E.g. $dateMin($.variants.created_at) returns the minimum created_at date of all variants.\n      $dateDiff(date1, date2, unit: \"seconds\" | \"minutes\" | \"hours\" | \"days\") - Returns the difference between two dates in the specified unit. E.g. $dateDiff($.order.created_at, $.order.updated_at, \"days\") returns the number of days between the order created_at and updated_at.\n      $now([picture [, timezone]]) - Returns current date and time in ISO 8601 format. E.g. $now() => \"2017-05-15T15:12:59.152Z\"\n      $split(str[, separator][, limit]) - Splits string into array\n      $join(array[, separator]) - Joins array elements into string\n      $match(str, pattern[, limit]) - Returns array of regex matches\n      $replace(str, pattern, replacement) - Replaces all occurrences of pattern. E.g. $replace(\"abracadabra\", /a.*?a/, \"*\") returns \"ab*ad*bra\". $replace(\"John Smith\", \"John\", \"Marc\") returns Marc Smith.\n      $number(arg) - Converts an argument to a number.\n      $min(arr) - Returns minimum number of a number array. E.g. $min($map($.variants.price, $number)) returns the minimum price of all variants.\n      $max(arr) - Returns maximum number of a number array. E.g. $max($map($.variants.price, $number)) returns the maximum price of all variants.\n      $count(array) - Returns array length\n      $sort(array[, function]) - Sorts array\n      $distinct(array) - Removes duplicates\n      $map(array, function) - Applies function to each element\n      $filter(array, function) - Filters array based on predicate\n\n- Error handling:\n  - If you get an error like \"is not of a type(s) string/number/object\", try to convert the source field, but also consider that the original field or one of its parent might be null. In this case, add a default value.\n  - If the error is something like \"instance is not of a type(s) object\", make sure you REALLY create the target schema with the correct type.\n  - If the error is something like \"instance is not of a type(s) array or array/null\". In this case, wrap the source selector in an array to ensure it always returns an array. E.g. \"result\": [$.items]\n  - if an object is optional but its fields required, you can add a test and default to {}, but do not set the inner fields to default null.\n\nRemember: The goal is to create valid JSONata expressions that accurately transform the source data structure into the required target structure.";

package/lib/cjs/data-mapper/prompts.js

@@ -0,0 +1,94 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PROMPT_MAPPING = void 0;
+exports.PROMPT_MAPPING = `You are an AI that generates JSONata mapping expressions to transform source data structures into target structures.
+
+Guidelines for creating JSONata mappings:
+
+1. Source References:
+   - Use exact field paths from the source data, e.g. $.merchant_category
+   - For accessing fields with names containing spaces, use backticks, e.g. $.\`merchant category\`
+   - Jsonata will automatically extract all the fields from the current context. E.g. if you need all variants from all products, you can use $.products.variants. No need to do nested map reduce operations.
+   - $. The variable with no name refers to the context value at any point in the input JSON hierarchy. E.g. if the current context is products.price, then $.currency is products.price.currency
+   - %. The parent of the current context value. E.g. if the current context is products.variants.size and you want variant name, use %.name
+
+   - When multiple source fields could map to a target, use a maximum of 3 fallbacks:
+     GOOD: source1 ? source1 : source2 ? source2 : source3 ? source3 : 'default'
+     BAD: source1 ? source1 : source1 ? source1 : source1 (repeated fields)
+
+2. Expression Rules:
+   - Avoid unnecessary array/string operations
+   - Each mapping should be clear and concise
+   - Use proper JSONata syntax for coalesce operations
+   - Do not use ~> to execute functions. Use the functions directly with the correct arguments or use $map(arr, $function) to apply a function to each element of an array.
+
+3. Array Handling:
+   - For mapping to an array of objects, use the following patterns:
+     a) When in array scope, use $.{} to map each object:
+        Correct: [$.{"id": id, "name": name}]
+        Incorrect: [{"id": $.id}]
+     b) When outside array scope, include the source path:
+        Correct: [$.items.{"id": id, "name": name}]
+        Incorrect: [{"id": $.items.id}]
+     c) For nested arrays, chain the array operators:
+        Correct: [products.variants.{"size": size, "color": color}]
+        Incorrect: [products.[{"size": variants.size}]]
+     d) You need to use the square brackets [] to map to an array of objects, otherwise it might return an object and fail the validation.
+        Correct: variants: [variants.{"size": size, "color": color}]
+        Incorrect: variants: variants.{"size": variants.size}
+   - For array elements, use JSONata array operators like [0] for first element, [-1] for last element
+   - Square bracket notation [] can be used with predicates, e.g. items[type='book']
+
+4. Field Selection Priority:
+   - Prefer variant-specific fields over general fields (e.g., sizeVariants.description over sizes)
+   - Choose the most specific/detailed field available (e.g., type="shelf" over category="furniture")
+
+5. Filters:
+   - Pay special attention to filter statements in the instruction and the schema description. Add them to the generated jsonata expression.
+     Example: Get me all products with SKU 0406654608 or products: {"type": "array", description: "only products with SKU 0406654608"}
+     Generated jsonata expression: Account.Order.Product[SKU = "0406654608"].{"Description": Description}
+   - For filtering with arrays, you can use the "in" operator. E.g. library.books["Steven King" in authors]
+
+6. Data Integrity:
+   - ONLY use fields that exist in the source data structure
+   - If no matching source field exists, leave the field undefined
+   - Never invent or assume the existence of fields not present in the source data
+
+7. Function Calls:
+   - You may use the following functions if prompted:
+      $string(arg) - Converts argument to string
+      $length(str) - Returns string length
+      $substring(str, start[, length]) - Extracts substring
+      $substringBefore(str, chars) - Gets substring before specified chars
+      $substringAfter(str, chars) - Gets substring after specified chars
+      $uppercase(str) - Converts to uppercase
+      $lowercase(str) - Converts to lowercase
+      $trim(str) - Removes whitespace from both ends
+      $pad(str, width[, char]) - Pads string to specified width
+      $contains(str, substring) - Tests if string contains substring
+      $toMillis(timestamp [, picture]) - Converts ISO 8601 timestamp to milliseconds. E.g. $toMillis("2017-11-07T15:07:54.972Z") => 1510067274972
+      $toDate(str | number) - Converts any timestamp string to valid ISO 8601 date string. E.g. $toDate("Oct 15, 2024 12:00:00 AM UTC") => "2024-10-15T00:00:00.000Z", $toDate(1728873600000) => "2024-10-15T00:00:00.000Z"
+      $dateMax(arr) - Returns the maximum date of an array of dates. E.g. $dateMax(["2017-11-07T15:07:54.972Z", "Oct 15, 2012 12:00:00 AM UTC"]) returns "2017-11-07T15:07:54.972Z".
+      $dateMin(arr) - Returns the minimum date of an array of dates. E.g. $dateMin($.variants.created_at) returns the minimum created_at date of all variants.
+      $dateDiff(date1, date2, unit: "seconds" | "minutes" | "hours" | "days") - Returns the difference between two dates in the specified unit. E.g. $dateDiff($.order.created_at, $.order.updated_at, "days") returns the number of days between the order created_at and updated_at.
+      $now([picture [, timezone]]) - Returns current date and time in ISO 8601 format. E.g. $now() => "2017-05-15T15:12:59.152Z"
+      $split(str[, separator][, limit]) - Splits string into array
+      $join(array[, separator]) - Joins array elements into string
+      $match(str, pattern[, limit]) - Returns array of regex matches
+      $replace(str, pattern, replacement) - Replaces all occurrences of pattern. E.g. $replace("abracadabra", /a.*?a/, "*") returns "ab*ad*bra". $replace("John Smith", "John", "Marc") returns Marc Smith.
+      $number(arg) - Converts an argument to a number.
+      $min(arr) - Returns minimum number of a number array. E.g. $min($map($.variants.price, $number)) returns the minimum price of all variants.
+      $max(arr) - Returns maximum number of a number array. E.g. $max($map($.variants.price, $number)) returns the maximum price of all variants.
+      $count(array) - Returns array length
+      $sort(array[, function]) - Sorts array
+      $distinct(array) - Removes duplicates
+      $map(array, function) - Applies function to each element
+      $filter(array, function) - Filters array based on predicate
+
+- Error handling:
+  - If you get an error like "is not of a type(s) string/number/object", try to convert the source field, but also consider that the original field or one of its parent might be null. In this case, add a default value.
+  - If the error is something like "instance is not of a type(s) object", make sure you REALLY create the target schema with the correct type.
+  - If the error is something like "instance is not of a type(s) array or array/null". In this case, wrap the source selector in an array to ensure it always returns an array. E.g. "result": [$.items]
+  - if an object is optional but its fields required, you can add a test and default to {}, but do not set the inner fields to default null.
+
+Remember: The goal is to create valid JSONata expressions that accurately transform the source data structure into the required target structure.`;

package/lib/cjs/data-mapper/tools.d.ts

@@ -0,0 +1,17 @@
+import jsonata from "jsonata";
+import type { Schema } from "jsonschema";
+export interface TransformResult {
+    success: boolean;
+    data?: unknown;
+    error?: string;
+}
+/**
+ * Extract JSON Schema from data
+ * @param data Any data
+ * @returns JSON Schema or null
+ */
+export declare function getSchemaFromData(data: unknown): unknown;
+export declare function applyJsonataWithValidation(data: unknown, expr: string, schema: unknown): Promise<TransformResult>;
+export declare function applyJsonata(data: unknown, expr: string): Promise<unknown>;
+export declare function extendJsonata(expr: string): jsonata.Expression;
+export declare function addNullableToOptional(schema: Schema): Schema;

package/lib/cjs/data-mapper/tools.js

@@ -0,0 +1,177 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getSchemaFromData = getSchemaFromData;
+exports.applyJsonataWithValidation = applyJsonataWithValidation;
+exports.applyJsonata = applyJsonata;
+exports.extendJsonata = extendJsonata;
+exports.addNullableToOptional = addNullableToOptional;
+/* eslint-disable @typescript-eslint/no-explicit-any */
+const jsonata_1 = __importDefault(require("jsonata"));
+const jsonschema_1 = require("jsonschema");
+const to_json_schema_1 = __importDefault(require("to-json-schema"));
+/**
+ * Extract JSON Schema from data
+ * @param data Any data
+ * @returns JSON Schema or null
+ */
+function getSchemaFromData(data) {
+    if (!data)
+        return null;
+    return (0, to_json_schema_1.default)(data);
+}
+async function applyJsonataWithValidation(data, expr, schema) {
+    try {
+        const result = await applyJsonata(data, expr);
+        if (result === null ||
+            result === undefined ||
+            (Array.isArray(result) && result.length === 0) ||
+            (typeof result === "object" && Object.keys(result).length === 0)) {
+            return { success: false, error: "Result is empty" };
+        }
+        const validator = new jsonschema_1.Validator();
+        const optionalSchema = addNullableToOptional(schema);
+        const validation = validator.validate(result, optionalSchema);
+        if (!validation.valid) {
+            return {
+                success: false,
+                error: validation.errors
+                    .map((e) => `${e.stack}. Source: ${e.instance ? JSON.stringify(e.instance) : "undefined"}`)
+                    .join("\n")
+                    .slice(0, 5000),
+            };
+        }
+        return { success: true, data: result };
+    }
+    catch (error) {
+        return { success: false, error: `Validation failed: ${error.message}` };
+    }
+}
+async function applyJsonata(data, expr) {
+    try {
+        const expression = extendJsonata(expr);
+        const result = await expression.evaluate(data);
+        return result;
+    }
+    catch (error) {
+        throw new Error(`JSONata evaluation failed for expression "${expr}": ${error.message}`);
+    }
+}
+function extendJsonata(expr) {
+    const expression = (0, jsonata_1.default)(expr);
+    expression.registerFunction("max", (arr) => {
+        if (Array.isArray(arr)) {
+            return Math.max(...arr.map(Number).filter((n) => !Number.isNaN(n)));
+        }
+        return arr;
+    });
+    expression.registerFunction("min", (arr) => {
+        if (Array.isArray(arr)) {
+            return Math.min(...arr.map(Number).filter((n) => !Number.isNaN(n)));
+        }
+        return arr;
+    });
+    expression.registerFunction("number", (value) => Number.parseFloat(value));
+    expression.registerFunction("substring", (str, start, end) => String(str).substring(start, end));
+    expression.registerFunction("replace", (obj, pattern, replacement) => {
+        if (Array.isArray(obj)) {
+            return obj.map((item) => String(item).replace(pattern, replacement));
+        }
+        if (typeof obj === "object") {
+            return Object.fromEntries(Object.entries(obj || {}).map(([key, value]) => [
+                key,
+                String(value).replace(pattern, replacement),
+            ]));
+        }
+        return String(obj).replace(pattern, replacement);
+    });
+    expression.registerFunction("toDate", (date) => {
+        try {
+            // Handle numeric timestamps (milliseconds or seconds)
+            if (typeof date === "number" || /^\d+$/.test(date)) {
+                const timestamp = typeof date === "number" ? date : Number.parseInt(date, 10);
+                // If timestamp is in seconds (typically 10 digits), convert to milliseconds
+                const millisTimestamp = timestamp < 10000000000 ? timestamp * 1000 : timestamp;
+                return new Date(millisTimestamp).toISOString();
+            }
+            // Handle date strings in MM/DD/YYYY format
+            const match = String(date).match(/^(\d{2})\/(\d{2})\/(\d{4})(?:\s+(\d{2}):(\d{2}):(\d{2}))?$/);
+            if (match) {
+                // eslint-disable-next-line @typescript-eslint/naming-convention
+                const [_, month, day, year, hours = "00", minutes = "00", seconds = "00"] = match;
+                const isoDate = `${year}-${month}-${day}T${hours}:${minutes}:${seconds}.000Z`;
+                return new Date(isoDate).toISOString();
+            }
+            // Default case: try standard Date parsing
+            return new Date(date).toISOString();
+        }
+        catch (e) {
+            throw new Error(`Invalid date: ${e.message}`);
+        }
+    });
+    expression.registerFunction("dateMax", (dates) => dates.reduce((max, curr) => (new Date(max) > new Date(curr) ? max : curr)));
+    expression.registerFunction("dateMin", (dates) => dates.reduce((min, curr) => (new Date(min) < new Date(curr) ? min : curr)));
+    expression.registerFunction("dateDiff", (date1, date2, unit = "days") => {
+        const d1 = new Date(date1);
+        const d2 = new Date(date2);
+        const diff = Math.abs(d1.getTime() - d2.getTime());
+        switch (unit.toLowerCase()) {
+            case "seconds":
+                return Math.floor(diff / 1000);
+            case "minutes":
+                return Math.floor(diff / (1000 * 60));
+            case "hours":
+                return Math.floor(diff / (1000 * 60 * 60));
+            case "days":
+                return Math.floor(diff / (1000 * 60 * 60 * 24));
+            default:
+                return diff; // milliseconds
+        }
+    });
+    return expression;
+}
+function addNullableToOptional(schema) {
+    if (!schema || typeof schema !== "object")
+        return schema;
+    const newSchema = { ...schema };
+    if (schema.type === "object" && schema.properties) {
+        const required = new Set(Array.isArray(schema.required) ? schema.required : []);
+        newSchema.properties = Object.entries(schema.properties).reduce((acc, [key, value]) => ({
+            // biome-ignore lint/performance/noAccumulatingSpread: <explanation>
+            ...acc,
+            [key]: !required.has(key) ? makeNullable(value) : addNullableToOptional(value),
+        }), {});
+    }
+    if (schema.type === "array" && schema.items) {
+        newSchema.items = addNullableToOptional(schema.items);
+    }
+    return newSchema;
+}
+// biome-ignore lint/suspicious/noExplicitAny: <explanation>
+function makeNullable(schema) {
+    if (!schema || typeof schema !== "object")
+        return schema;
+    const newSchema = { ...schema };
+    if (Array.isArray(schema.type)) {
+        if (!schema.type.includes("null")) {
+            newSchema.type = [...schema.type, "null"];
+        }
+    }
+    else if (schema.type) {
+        newSchema.type = [schema.type, "null"];
+    }
+    // Recursively process nested properties
+    if (schema.properties) {
+        newSchema.properties = Object.entries(schema.properties).reduce((acc, [key, value]) => ({
+            // biome-ignore lint/performance/noAccumulatingSpread: <explanation>
+            ...acc,
+            [key]: makeNullable(value),
+        }), {});
+    }
+    if (schema.items) {
+        newSchema.items = makeNullable(schema.items);
+    }
+    return newSchema;
+}

package/lib/dts/data-mapper/agents/mapper.d.ts

@@ -0,0 +1,14 @@
+import { AIAgent } from "@aigne/core";
+declare const mapper: AIAgent<{
+    sourceData: string;
+    responseSchema: string;
+    sourceSchema?: string | undefined;
+    instruction?: string | undefined;
+    responseData?: string | undefined;
+    feedback?: string | undefined;
+}, {
+    jsonata: string;
+    confidence: number;
+    confidenceReasoning: string;
+}>;
+export default mapper;

package/lib/dts/data-mapper/agents/reviewer.d.ts

@@ -0,0 +1,19 @@
+import { FunctionAgent } from "@aigne/core";
+declare const reviewer: FunctionAgent<{
+    sourceData: string;
+    responseSchema: string;
+    jsonata: string;
+}, {
+    success: boolean;
+    data: null;
+    feedback: string;
+} | {
+    success: boolean;
+    data: unknown;
+    feedback?: undefined;
+} | {
+    success: false;
+    data: unknown;
+    feedback: string | undefined;
+}>;
+export default reviewer;

package/lib/dts/data-mapper/index.d.ts

@@ -0,0 +1,18 @@
+import { type ChatModel } from "@aigne/core";
+export interface TransformInput {
+    responseSchema: string;
+    responseSampleData?: string;
+    sourceData?: string;
+    sourceSchema?: string;
+    instruction?: string;
+    [key: string]: unknown;
+}
+export declare function generateMapping({ input, model, }: {
+    input: TransformInput;
+    model: ChatModel;
+}): Promise<{
+    jsonata: string;
+    confidence: number;
+    confidenceReasoning: string;
+} | null>;
+export { applyJsonata } from "./tools.js";

package/lib/dts/data-mapper/prompts.d.ts

@@ -0,0 +1 @@
+export declare const PROMPT_MAPPING = "You are an AI that generates JSONata mapping expressions to transform source data structures into target structures.\n\nGuidelines for creating JSONata mappings:\n\n1. Source References:\n   - Use exact field paths from the source data, e.g. $.merchant_category\n   - For accessing fields with names containing spaces, use backticks, e.g. $.`merchant category`\n   - Jsonata will automatically extract all the fields from the current context. E.g. if you need all variants from all products, you can use $.products.variants. No need to do nested map reduce operations.\n   - $. The variable with no name refers to the context value at any point in the input JSON hierarchy. E.g. if the current context is products.price, then $.currency is products.price.currency\n   - %. The parent of the current context value. E.g. if the current context is products.variants.size and you want variant name, use %.name\n\n   - When multiple source fields could map to a target, use a maximum of 3 fallbacks:\n     GOOD: source1 ? source1 : source2 ? source2 : source3 ? source3 : 'default'\n     BAD: source1 ? source1 : source1 ? source1 : source1 (repeated fields)\n\n2. Expression Rules:\n   - Avoid unnecessary array/string operations\n   - Each mapping should be clear and concise\n   - Use proper JSONata syntax for coalesce operations\n   - Do not use ~> to execute functions. Use the functions directly with the correct arguments or use $map(arr, $function) to apply a function to each element of an array.\n\n3. Array Handling:\n   - For mapping to an array of objects, use the following patterns:\n     a) When in array scope, use $.{} to map each object:\n        Correct: [$.{\"id\": id, \"name\": name}]\n        Incorrect: [{\"id\": $.id}]\n     b) When outside array scope, include the source path:\n        Correct: [$.items.{\"id\": id, \"name\": name}]\n        Incorrect: [{\"id\": $.items.id}]\n     c) For nested arrays, chain the array operators:\n        Correct: [products.variants.{\"size\": size, \"color\": color}]\n        Incorrect: [products.[{\"size\": variants.size}]]\n     d) You need to use the square brackets [] to map to an array of objects, otherwise it might return an object and fail the validation.\n        Correct: variants: [variants.{\"size\": size, \"color\": color}]\n        Incorrect: variants: variants.{\"size\": variants.size}\n   - For array elements, use JSONata array operators like [0] for first element, [-1] for last element\n   - Square bracket notation [] can be used with predicates, e.g. items[type='book']\n\n4. Field Selection Priority:\n   - Prefer variant-specific fields over general fields (e.g., sizeVariants.description over sizes)\n   - Choose the most specific/detailed field available (e.g., type=\"shelf\" over category=\"furniture\")\n\n5. Filters:\n   - Pay special attention to filter statements in the instruction and the schema description. Add them to the generated jsonata expression.\n     Example: Get me all products with SKU 0406654608 or products: {\"type\": \"array\", description: \"only products with SKU 0406654608\"}\n     Generated jsonata expression: Account.Order.Product[SKU = \"0406654608\"].{\"Description\": Description}\n   - For filtering with arrays, you can use the \"in\" operator. E.g. library.books[\"Steven King\" in authors]\n\n6. Data Integrity:\n   - ONLY use fields that exist in the source data structure\n   - If no matching source field exists, leave the field undefined\n   - Never invent or assume the existence of fields not present in the source data\n\n7. Function Calls:\n   - You may use the following functions if prompted:\n      $string(arg) - Converts argument to string\n      $length(str) - Returns string length\n      $substring(str, start[, length]) - Extracts substring\n      $substringBefore(str, chars) - Gets substring before specified chars\n      $substringAfter(str, chars) - Gets substring after specified chars\n      $uppercase(str) - Converts to uppercase\n      $lowercase(str) - Converts to lowercase\n      $trim(str) - Removes whitespace from both ends\n      $pad(str, width[, char]) - Pads string to specified width\n      $contains(str, substring) - Tests if string contains substring\n      $toMillis(timestamp [, picture]) - Converts ISO 8601 timestamp to milliseconds. E.g. $toMillis(\"2017-11-07T15:07:54.972Z\") => 1510067274972\n      $toDate(str | number) - Converts any timestamp string to valid ISO 8601 date string. E.g. $toDate(\"Oct 15, 2024 12:00:00 AM UTC\") => \"2024-10-15T00:00:00.000Z\", $toDate(1728873600000) => \"2024-10-15T00:00:00.000Z\"\n      $dateMax(arr) - Returns the maximum date of an array of dates. E.g. $dateMax([\"2017-11-07T15:07:54.972Z\", \"Oct 15, 2012 12:00:00 AM UTC\"]) returns \"2017-11-07T15:07:54.972Z\".\n      $dateMin(arr) - Returns the minimum date of an array of dates. E.g. $dateMin($.variants.created_at) returns the minimum created_at date of all variants.\n      $dateDiff(date1, date2, unit: \"seconds\" | \"minutes\" | \"hours\" | \"days\") - Returns the difference between two dates in the specified unit. E.g. $dateDiff($.order.created_at, $.order.updated_at, \"days\") returns the number of days between the order created_at and updated_at.\n      $now([picture [, timezone]]) - Returns current date and time in ISO 8601 format. E.g. $now() => \"2017-05-15T15:12:59.152Z\"\n      $split(str[, separator][, limit]) - Splits string into array\n      $join(array[, separator]) - Joins array elements into string\n      $match(str, pattern[, limit]) - Returns array of regex matches\n      $replace(str, pattern, replacement) - Replaces all occurrences of pattern. E.g. $replace(\"abracadabra\", /a.*?a/, \"*\") returns \"ab*ad*bra\". $replace(\"John Smith\", \"John\", \"Marc\") returns Marc Smith.\n      $number(arg) - Converts an argument to a number.\n      $min(arr) - Returns minimum number of a number array. E.g. $min($map($.variants.price, $number)) returns the minimum price of all variants.\n      $max(arr) - Returns maximum number of a number array. E.g. $max($map($.variants.price, $number)) returns the maximum price of all variants.\n      $count(array) - Returns array length\n      $sort(array[, function]) - Sorts array\n      $distinct(array) - Removes duplicates\n      $map(array, function) - Applies function to each element\n      $filter(array, function) - Filters array based on predicate\n\n- Error handling:\n  - If you get an error like \"is not of a type(s) string/number/object\", try to convert the source field, but also consider that the original field or one of its parent might be null. In this case, add a default value.\n  - If the error is something like \"instance is not of a type(s) object\", make sure you REALLY create the target schema with the correct type.\n  - If the error is something like \"instance is not of a type(s) array or array/null\". In this case, wrap the source selector in an array to ensure it always returns an array. E.g. \"result\": [$.items]\n  - if an object is optional but its fields required, you can add a test and default to {}, but do not set the inner fields to default null.\n\nRemember: The goal is to create valid JSONata expressions that accurately transform the source data structure into the required target structure.";

package/lib/dts/data-mapper/tools.d.ts

@@ -0,0 +1,17 @@
+import jsonata from "jsonata";
+import type { Schema } from "jsonschema";
+export interface TransformResult {
+    success: boolean;
+    data?: unknown;
+    error?: string;
+}
+/**
+ * Extract JSON Schema from data
+ * @param data Any data
+ * @returns JSON Schema or null
+ */
+export declare function getSchemaFromData(data: unknown): unknown;
+export declare function applyJsonataWithValidation(data: unknown, expr: string, schema: unknown): Promise<TransformResult>;
+export declare function applyJsonata(data: unknown, expr: string): Promise<unknown>;
+export declare function extendJsonata(expr: string): jsonata.Expression;
+export declare function addNullableToOptional(schema: Schema): Schema;

package/lib/esm/data-mapper/agents/mapper.d.ts

@@ -0,0 +1,14 @@
+import { AIAgent } from "@aigne/core";
+declare const mapper: AIAgent<{
+    sourceData: string;
+    responseSchema: string;
+    sourceSchema?: string | undefined;
+    instruction?: string | undefined;
+    responseData?: string | undefined;
+    feedback?: string | undefined;
+}, {
+    jsonata: string;
+    confidence: number;
+    confidenceReasoning: string;
+}>;
+export default mapper;

package/lib/esm/data-mapper/agents/mapper.js

@@ -0,0 +1,61 @@
+import { AIAgent, PromptBuilder, UserInputTopic } from "@aigne/core";
+import { z } from "zod";
+import { PROMPT_MAPPING } from "../prompts.js";
+const mapper = AIAgent.from({
+    subscribeTopic: [UserInputTopic, "mapping_request"],
+    publishTopic: "review_request",
+    inputSchema: z.object({
+        sourceData: z.string(),
+        sourceSchema: z.string().optional(),
+        responseSchema: z.string(),
+        instruction: z.string().optional(),
+        responseData: z.string().optional(),
+        feedback: z.string().optional(),
+    }),
+    outputSchema: z.object({
+        jsonata: z.string().describe("JSONata expression"),
+        confidence: z.number().describe(`Confidence score for the JSONata expression between 0 and 100.
+      Give a low confidence score if there are missing fields in the source data.
+      Give a low confidence score if there are multiple options for a field and it is unclear which one to choose.`),
+        confidenceReasoning: z.string().describe("Reasoning for the confidence score"),
+    }),
+    includeInputInOutput: true,
+    instructions: PromptBuilder.from({
+        messages: [
+            {
+                role: "assistant",
+                content: {
+                    type: "text",
+                    text: PROMPT_MAPPING,
+                },
+            },
+            {
+                role: "user",
+                content: {
+                    type: "text",
+                    text: `Given a source data and structure, create a jsonata expression in JSON FORMAT.
+                Important: The output should be a jsonata expression creating an object that matches the following schema:
+                {{responseSchema}}
+
+                Pay special attention to the requirements in field descriptions
+
+                The instruction from the user is: {{instruction}}
+
+                ------
+
+                Source Data Structure:
+                {{sourceSchema}}
+
+                Source data Sample:
+                {{sourceData}}
+
+                ------
+
+                Previous feedback:
+                {{feedback}}`,
+                },
+            },
+        ],
+    }),
+});
+export default mapper;

package/lib/esm/data-mapper/agents/reviewer.d.ts

@@ -0,0 +1,19 @@
+import { FunctionAgent } from "@aigne/core";
+declare const reviewer: FunctionAgent<{
+    sourceData: string;
+    responseSchema: string;
+    jsonata: string;
+}, {
+    success: boolean;
+    data: null;
+    feedback: string;
+} | {
+    success: boolean;
+    data: unknown;
+    feedback?: undefined;
+} | {
+    success: false;
+    data: unknown;
+    feedback: string | undefined;
+}>;
+export default reviewer;

package/lib/esm/data-mapper/agents/reviewer.js

@@ -0,0 +1,45 @@
+import { FunctionAgent, UserOutputTopic } from "@aigne/core";
+import { z } from "zod";
+import { applyJsonataWithValidation } from "../tools.js";
+const reviewer = FunctionAgent.from({
+    name: "check_mapping",
+    description: "Check the mapping result",
+    subscribeTopic: ["review_request"],
+    publishTopic: (output) => (output.success ? UserOutputTopic : "mapping_request"),
+    inputSchema: z.object({
+        sourceData: z.string(),
+        jsonata: z.string(),
+        responseSchema: z.string(),
+    }),
+    process: async ({ sourceData, jsonata, responseSchema, }) => {
+        let parsedSourceData = null;
+        let parsedResponseSchema = null;
+        try {
+            parsedSourceData = sourceData ? JSON.parse(sourceData) : null;
+            parsedResponseSchema = responseSchema ? JSON.parse(responseSchema) : null;
+        }
+        catch (parseError) {
+            // input data is not valid JSON, return success
+            return {
+                success: true,
+                data: null,
+                feedback: `JSON parsing failed: ${parseError.message}`,
+            };
+        }
+        const transformation = await applyJsonataWithValidation(parsedSourceData, jsonata, parsedResponseSchema);
+        // if transformation is successful, return success
+        if (transformation.success) {
+            return {
+                success: true,
+                data: transformation.data,
+            };
+        }
+        return {
+            success: transformation.success,
+            data: transformation.data,
+            feedback: transformation.error,
+        };
+    },
+    includeInputInOutput: true,
+});
+export default reviewer;

package/lib/esm/data-mapper/index.d.ts

@@ -0,0 +1,18 @@
+import { type ChatModel } from "@aigne/core";
+export interface TransformInput {
+    responseSchema: string;
+    responseSampleData?: string;
+    sourceData?: string;
+    sourceSchema?: string;
+    instruction?: string;
+    [key: string]: unknown;
+}
+export declare function generateMapping({ input, model, }: {
+    input: TransformInput;
+    model: ChatModel;
+}): Promise<{
+    jsonata: string;
+    confidence: number;
+    confidenceReasoning: string;
+} | null>;
+export { applyJsonata } from "./tools.js";

package/lib/esm/data-mapper/index.js

@@ -0,0 +1,21 @@
+import { AIGNE, UserInputTopic, UserOutputTopic } from "@aigne/core";
+import mapper from "./agents/mapper.js";
+import reviewer from "./agents/reviewer.js";
+import { getSchemaFromData } from "./tools.js";
+export async function generateMapping({ input, model, }) {
+    if (!model)
+        throw new Error("model is required to run data mapper");
+    // if sourceSchema is not provided, generate it from sourceData
+    if (!input.sourceSchema && input.sourceData) {
+        input.sourceSchema = JSON.stringify(getSchemaFromData(JSON.parse(input.sourceData)));
+    }
+    const aigne = new AIGNE({ model, agents: [mapper, reviewer] });
+    aigne.publish(UserInputTopic, input);
+    const { message } = await aigne.subscribe(UserOutputTopic);
+    return {
+        jsonata: message.jsonata || "",
+        confidence: message.confidence || 0,
+        confidenceReasoning: message.confidenceReasoning || "",
+    };
+}
+export { applyJsonata } from "./tools.js";

package/lib/esm/data-mapper/prompts.d.ts

@@ -0,0 +1 @@
+export declare const PROMPT_MAPPING = "You are an AI that generates JSONata mapping expressions to transform source data structures into target structures.\n\nGuidelines for creating JSONata mappings:\n\n1. Source References:\n   - Use exact field paths from the source data, e.g. $.merchant_category\n   - For accessing fields with names containing spaces, use backticks, e.g. $.`merchant category`\n   - Jsonata will automatically extract all the fields from the current context. E.g. if you need all variants from all products, you can use $.products.variants. No need to do nested map reduce operations.\n   - $. The variable with no name refers to the context value at any point in the input JSON hierarchy. E.g. if the current context is products.price, then $.currency is products.price.currency\n   - %. The parent of the current context value. E.g. if the current context is products.variants.size and you want variant name, use %.name\n\n   - When multiple source fields could map to a target, use a maximum of 3 fallbacks:\n     GOOD: source1 ? source1 : source2 ? source2 : source3 ? source3 : 'default'\n     BAD: source1 ? source1 : source1 ? source1 : source1 (repeated fields)\n\n2. Expression Rules:\n   - Avoid unnecessary array/string operations\n   - Each mapping should be clear and concise\n   - Use proper JSONata syntax for coalesce operations\n   - Do not use ~> to execute functions. Use the functions directly with the correct arguments or use $map(arr, $function) to apply a function to each element of an array.\n\n3. Array Handling:\n   - For mapping to an array of objects, use the following patterns:\n     a) When in array scope, use $.{} to map each object:\n        Correct: [$.{\"id\": id, \"name\": name}]\n        Incorrect: [{\"id\": $.id}]\n     b) When outside array scope, include the source path:\n        Correct: [$.items.{\"id\": id, \"name\": name}]\n        Incorrect: [{\"id\": $.items.id}]\n     c) For nested arrays, chain the array operators:\n        Correct: [products.variants.{\"size\": size, \"color\": color}]\n        Incorrect: [products.[{\"size\": variants.size}]]\n     d) You need to use the square brackets [] to map to an array of objects, otherwise it might return an object and fail the validation.\n        Correct: variants: [variants.{\"size\": size, \"color\": color}]\n        Incorrect: variants: variants.{\"size\": variants.size}\n   - For array elements, use JSONata array operators like [0] for first element, [-1] for last element\n   - Square bracket notation [] can be used with predicates, e.g. items[type='book']\n\n4. Field Selection Priority:\n   - Prefer variant-specific fields over general fields (e.g., sizeVariants.description over sizes)\n   - Choose the most specific/detailed field available (e.g., type=\"shelf\" over category=\"furniture\")\n\n5. Filters:\n   - Pay special attention to filter statements in the instruction and the schema description. Add them to the generated jsonata expression.\n     Example: Get me all products with SKU 0406654608 or products: {\"type\": \"array\", description: \"only products with SKU 0406654608\"}\n     Generated jsonata expression: Account.Order.Product[SKU = \"0406654608\"].{\"Description\": Description}\n   - For filtering with arrays, you can use the \"in\" operator. E.g. library.books[\"Steven King\" in authors]\n\n6. Data Integrity:\n   - ONLY use fields that exist in the source data structure\n   - If no matching source field exists, leave the field undefined\n   - Never invent or assume the existence of fields not present in the source data\n\n7. Function Calls:\n   - You may use the following functions if prompted:\n      $string(arg) - Converts argument to string\n      $length(str) - Returns string length\n      $substring(str, start[, length]) - Extracts substring\n      $substringBefore(str, chars) - Gets substring before specified chars\n      $substringAfter(str, chars) - Gets substring after specified chars\n      $uppercase(str) - Converts to uppercase\n      $lowercase(str) - Converts to lowercase\n      $trim(str) - Removes whitespace from both ends\n      $pad(str, width[, char]) - Pads string to specified width\n      $contains(str, substring) - Tests if string contains substring\n      $toMillis(timestamp [, picture]) - Converts ISO 8601 timestamp to milliseconds. E.g. $toMillis(\"2017-11-07T15:07:54.972Z\") => 1510067274972\n      $toDate(str | number) - Converts any timestamp string to valid ISO 8601 date string. E.g. $toDate(\"Oct 15, 2024 12:00:00 AM UTC\") => \"2024-10-15T00:00:00.000Z\", $toDate(1728873600000) => \"2024-10-15T00:00:00.000Z\"\n      $dateMax(arr) - Returns the maximum date of an array of dates. E.g. $dateMax([\"2017-11-07T15:07:54.972Z\", \"Oct 15, 2012 12:00:00 AM UTC\"]) returns \"2017-11-07T15:07:54.972Z\".\n      $dateMin(arr) - Returns the minimum date of an array of dates. E.g. $dateMin($.variants.created_at) returns the minimum created_at date of all variants.\n      $dateDiff(date1, date2, unit: \"seconds\" | \"minutes\" | \"hours\" | \"days\") - Returns the difference between two dates in the specified unit. E.g. $dateDiff($.order.created_at, $.order.updated_at, \"days\") returns the number of days between the order created_at and updated_at.\n      $now([picture [, timezone]]) - Returns current date and time in ISO 8601 format. E.g. $now() => \"2017-05-15T15:12:59.152Z\"\n      $split(str[, separator][, limit]) - Splits string into array\n      $join(array[, separator]) - Joins array elements into string\n      $match(str, pattern[, limit]) - Returns array of regex matches\n      $replace(str, pattern, replacement) - Replaces all occurrences of pattern. E.g. $replace(\"abracadabra\", /a.*?a/, \"*\") returns \"ab*ad*bra\". $replace(\"John Smith\", \"John\", \"Marc\") returns Marc Smith.\n      $number(arg) - Converts an argument to a number.\n      $min(arr) - Returns minimum number of a number array. E.g. $min($map($.variants.price, $number)) returns the minimum price of all variants.\n      $max(arr) - Returns maximum number of a number array. E.g. $max($map($.variants.price, $number)) returns the maximum price of all variants.\n      $count(array) - Returns array length\n      $sort(array[, function]) - Sorts array\n      $distinct(array) - Removes duplicates\n      $map(array, function) - Applies function to each element\n      $filter(array, function) - Filters array based on predicate\n\n- Error handling:\n  - If you get an error like \"is not of a type(s) string/number/object\", try to convert the source field, but also consider that the original field or one of its parent might be null. In this case, add a default value.\n  - If the error is something like \"instance is not of a type(s) object\", make sure you REALLY create the target schema with the correct type.\n  - If the error is something like \"instance is not of a type(s) array or array/null\". In this case, wrap the source selector in an array to ensure it always returns an array. E.g. \"result\": [$.items]\n  - if an object is optional but its fields required, you can add a test and default to {}, but do not set the inner fields to default null.\n\nRemember: The goal is to create valid JSONata expressions that accurately transform the source data structure into the required target structure.";

package/lib/esm/data-mapper/prompts.js

@@ -0,0 +1,91 @@
+export const PROMPT_MAPPING = `You are an AI that generates JSONata mapping expressions to transform source data structures into target structures.
+
+Guidelines for creating JSONata mappings:
+
+1. Source References:
+   - Use exact field paths from the source data, e.g. $.merchant_category
+   - For accessing fields with names containing spaces, use backticks, e.g. $.\`merchant category\`
+   - Jsonata will automatically extract all the fields from the current context. E.g. if you need all variants from all products, you can use $.products.variants. No need to do nested map reduce operations.
+   - $. The variable with no name refers to the context value at any point in the input JSON hierarchy. E.g. if the current context is products.price, then $.currency is products.price.currency
+   - %. The parent of the current context value. E.g. if the current context is products.variants.size and you want variant name, use %.name
+
+   - When multiple source fields could map to a target, use a maximum of 3 fallbacks:
+     GOOD: source1 ? source1 : source2 ? source2 : source3 ? source3 : 'default'
+     BAD: source1 ? source1 : source1 ? source1 : source1 (repeated fields)
+
+2. Expression Rules:
+   - Avoid unnecessary array/string operations
+   - Each mapping should be clear and concise
+   - Use proper JSONata syntax for coalesce operations
+   - Do not use ~> to execute functions. Use the functions directly with the correct arguments or use $map(arr, $function) to apply a function to each element of an array.
+
+3. Array Handling:
+   - For mapping to an array of objects, use the following patterns:
+     a) When in array scope, use $.{} to map each object:
+        Correct: [$.{"id": id, "name": name}]
+        Incorrect: [{"id": $.id}]
+     b) When outside array scope, include the source path:
+        Correct: [$.items.{"id": id, "name": name}]
+        Incorrect: [{"id": $.items.id}]
+     c) For nested arrays, chain the array operators:
+        Correct: [products.variants.{"size": size, "color": color}]
+        Incorrect: [products.[{"size": variants.size}]]
+     d) You need to use the square brackets [] to map to an array of objects, otherwise it might return an object and fail the validation.
+        Correct: variants: [variants.{"size": size, "color": color}]
+        Incorrect: variants: variants.{"size": variants.size}
+   - For array elements, use JSONata array operators like [0] for first element, [-1] for last element
+   - Square bracket notation [] can be used with predicates, e.g. items[type='book']
+
+4. Field Selection Priority:
+   - Prefer variant-specific fields over general fields (e.g., sizeVariants.description over sizes)
+   - Choose the most specific/detailed field available (e.g., type="shelf" over category="furniture")
+
+5. Filters:
+   - Pay special attention to filter statements in the instruction and the schema description. Add them to the generated jsonata expression.
+     Example: Get me all products with SKU 0406654608 or products: {"type": "array", description: "only products with SKU 0406654608"}
+     Generated jsonata expression: Account.Order.Product[SKU = "0406654608"].{"Description": Description}
+   - For filtering with arrays, you can use the "in" operator. E.g. library.books["Steven King" in authors]
+
+6. Data Integrity:
+   - ONLY use fields that exist in the source data structure
+   - If no matching source field exists, leave the field undefined
+   - Never invent or assume the existence of fields not present in the source data
+
+7. Function Calls:
+   - You may use the following functions if prompted:
+      $string(arg) - Converts argument to string
+      $length(str) - Returns string length
+      $substring(str, start[, length]) - Extracts substring
+      $substringBefore(str, chars) - Gets substring before specified chars
+      $substringAfter(str, chars) - Gets substring after specified chars
+      $uppercase(str) - Converts to uppercase
+      $lowercase(str) - Converts to lowercase
+      $trim(str) - Removes whitespace from both ends
+      $pad(str, width[, char]) - Pads string to specified width
+      $contains(str, substring) - Tests if string contains substring
+      $toMillis(timestamp [, picture]) - Converts ISO 8601 timestamp to milliseconds. E.g. $toMillis("2017-11-07T15:07:54.972Z") => 1510067274972
+      $toDate(str | number) - Converts any timestamp string to valid ISO 8601 date string. E.g. $toDate("Oct 15, 2024 12:00:00 AM UTC") => "2024-10-15T00:00:00.000Z", $toDate(1728873600000) => "2024-10-15T00:00:00.000Z"
+      $dateMax(arr) - Returns the maximum date of an array of dates. E.g. $dateMax(["2017-11-07T15:07:54.972Z", "Oct 15, 2012 12:00:00 AM UTC"]) returns "2017-11-07T15:07:54.972Z".
+      $dateMin(arr) - Returns the minimum date of an array of dates. E.g. $dateMin($.variants.created_at) returns the minimum created_at date of all variants.
+      $dateDiff(date1, date2, unit: "seconds" | "minutes" | "hours" | "days") - Returns the difference between two dates in the specified unit. E.g. $dateDiff($.order.created_at, $.order.updated_at, "days") returns the number of days between the order created_at and updated_at.
+      $now([picture [, timezone]]) - Returns current date and time in ISO 8601 format. E.g. $now() => "2017-05-15T15:12:59.152Z"
+      $split(str[, separator][, limit]) - Splits string into array
+      $join(array[, separator]) - Joins array elements into string
+      $match(str, pattern[, limit]) - Returns array of regex matches
+      $replace(str, pattern, replacement) - Replaces all occurrences of pattern. E.g. $replace("abracadabra", /a.*?a/, "*") returns "ab*ad*bra". $replace("John Smith", "John", "Marc") returns Marc Smith.
+      $number(arg) - Converts an argument to a number.
+      $min(arr) - Returns minimum number of a number array. E.g. $min($map($.variants.price, $number)) returns the minimum price of all variants.
+      $max(arr) - Returns maximum number of a number array. E.g. $max($map($.variants.price, $number)) returns the maximum price of all variants.
+      $count(array) - Returns array length
+      $sort(array[, function]) - Sorts array
+      $distinct(array) - Removes duplicates
+      $map(array, function) - Applies function to each element
+      $filter(array, function) - Filters array based on predicate
+
+- Error handling:
+  - If you get an error like "is not of a type(s) string/number/object", try to convert the source field, but also consider that the original field or one of its parent might be null. In this case, add a default value.
+  - If the error is something like "instance is not of a type(s) object", make sure you REALLY create the target schema with the correct type.
+  - If the error is something like "instance is not of a type(s) array or array/null". In this case, wrap the source selector in an array to ensure it always returns an array. E.g. "result": [$.items]
+  - if an object is optional but its fields required, you can add a test and default to {}, but do not set the inner fields to default null.
+
+Remember: The goal is to create valid JSONata expressions that accurately transform the source data structure into the required target structure.`;

package/lib/esm/data-mapper/tools.d.ts

@@ -0,0 +1,17 @@
+import jsonata from "jsonata";
+import type { Schema } from "jsonschema";
+export interface TransformResult {
+    success: boolean;
+    data?: unknown;
+    error?: string;
+}
+/**
+ * Extract JSON Schema from data
+ * @param data Any data
+ * @returns JSON Schema or null
+ */
+export declare function getSchemaFromData(data: unknown): unknown;
+export declare function applyJsonataWithValidation(data: unknown, expr: string, schema: unknown): Promise<TransformResult>;
+export declare function applyJsonata(data: unknown, expr: string): Promise<unknown>;
+export declare function extendJsonata(expr: string): jsonata.Expression;
+export declare function addNullableToOptional(schema: Schema): Schema;

package/lib/esm/data-mapper/tools.js

@@ -0,0 +1,167 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import jsonata from "jsonata";
+import { Validator } from "jsonschema";
+import toJsonSchema from "to-json-schema";
+/**
+ * Extract JSON Schema from data
+ * @param data Any data
+ * @returns JSON Schema or null
+ */
+export function getSchemaFromData(data) {
+    if (!data)
+        return null;
+    return toJsonSchema(data);
+}
+export async function applyJsonataWithValidation(data, expr, schema) {
+    try {
+        const result = await applyJsonata(data, expr);
+        if (result === null ||
+            result === undefined ||
+            (Array.isArray(result) && result.length === 0) ||
+            (typeof result === "object" && Object.keys(result).length === 0)) {
+            return { success: false, error: "Result is empty" };
+        }
+        const validator = new Validator();
+        const optionalSchema = addNullableToOptional(schema);
+        const validation = validator.validate(result, optionalSchema);
+        if (!validation.valid) {
+            return {
+                success: false,
+                error: validation.errors
+                    .map((e) => `${e.stack}. Source: ${e.instance ? JSON.stringify(e.instance) : "undefined"}`)
+                    .join("\n")
+                    .slice(0, 5000),
+            };
+        }
+        return { success: true, data: result };
+    }
+    catch (error) {
+        return { success: false, error: `Validation failed: ${error.message}` };
+    }
+}
+export async function applyJsonata(data, expr) {
+    try {
+        const expression = extendJsonata(expr);
+        const result = await expression.evaluate(data);
+        return result;
+    }
+    catch (error) {
+        throw new Error(`JSONata evaluation failed for expression "${expr}": ${error.message}`);
+    }
+}
+export function extendJsonata(expr) {
+    const expression = jsonata(expr);
+    expression.registerFunction("max", (arr) => {
+        if (Array.isArray(arr)) {
+            return Math.max(...arr.map(Number).filter((n) => !Number.isNaN(n)));
+        }
+        return arr;
+    });
+    expression.registerFunction("min", (arr) => {
+        if (Array.isArray(arr)) {
+            return Math.min(...arr.map(Number).filter((n) => !Number.isNaN(n)));
+        }
+        return arr;
+    });
+    expression.registerFunction("number", (value) => Number.parseFloat(value));
+    expression.registerFunction("substring", (str, start, end) => String(str).substring(start, end));
+    expression.registerFunction("replace", (obj, pattern, replacement) => {
+        if (Array.isArray(obj)) {
+            return obj.map((item) => String(item).replace(pattern, replacement));
+        }
+        if (typeof obj === "object") {
+            return Object.fromEntries(Object.entries(obj || {}).map(([key, value]) => [
+                key,
+                String(value).replace(pattern, replacement),
+            ]));
+        }
+        return String(obj).replace(pattern, replacement);
+    });
+    expression.registerFunction("toDate", (date) => {
+        try {
+            // Handle numeric timestamps (milliseconds or seconds)
+            if (typeof date === "number" || /^\d+$/.test(date)) {
+                const timestamp = typeof date === "number" ? date : Number.parseInt(date, 10);
+                // If timestamp is in seconds (typically 10 digits), convert to milliseconds
+                const millisTimestamp = timestamp < 10000000000 ? timestamp * 1000 : timestamp;
+                return new Date(millisTimestamp).toISOString();
+            }
+            // Handle date strings in MM/DD/YYYY format
+            const match = String(date).match(/^(\d{2})\/(\d{2})\/(\d{4})(?:\s+(\d{2}):(\d{2}):(\d{2}))?$/);
+            if (match) {
+                // eslint-disable-next-line @typescript-eslint/naming-convention
+                const [_, month, day, year, hours = "00", minutes = "00", seconds = "00"] = match;
+                const isoDate = `${year}-${month}-${day}T${hours}:${minutes}:${seconds}.000Z`;
+                return new Date(isoDate).toISOString();
+            }
+            // Default case: try standard Date parsing
+            return new Date(date).toISOString();
+        }
+        catch (e) {
+            throw new Error(`Invalid date: ${e.message}`);
+        }
+    });
+    expression.registerFunction("dateMax", (dates) => dates.reduce((max, curr) => (new Date(max) > new Date(curr) ? max : curr)));
+    expression.registerFunction("dateMin", (dates) => dates.reduce((min, curr) => (new Date(min) < new Date(curr) ? min : curr)));
+    expression.registerFunction("dateDiff", (date1, date2, unit = "days") => {
+        const d1 = new Date(date1);
+        const d2 = new Date(date2);
+        const diff = Math.abs(d1.getTime() - d2.getTime());
+        switch (unit.toLowerCase()) {
+            case "seconds":
+                return Math.floor(diff / 1000);
+            case "minutes":
+                return Math.floor(diff / (1000 * 60));
+            case "hours":
+                return Math.floor(diff / (1000 * 60 * 60));
+            case "days":
+                return Math.floor(diff / (1000 * 60 * 60 * 24));
+            default:
+                return diff; // milliseconds
+        }
+    });
+    return expression;
+}
+export function addNullableToOptional(schema) {
+    if (!schema || typeof schema !== "object")
+        return schema;
+    const newSchema = { ...schema };
+    if (schema.type === "object" && schema.properties) {
+        const required = new Set(Array.isArray(schema.required) ? schema.required : []);
+        newSchema.properties = Object.entries(schema.properties).reduce((acc, [key, value]) => ({
+            // biome-ignore lint/performance/noAccumulatingSpread: <explanation>
+            ...acc,
+            [key]: !required.has(key) ? makeNullable(value) : addNullableToOptional(value),
+        }), {});
+    }
+    if (schema.type === "array" && schema.items) {
+        newSchema.items = addNullableToOptional(schema.items);
+    }
+    return newSchema;
+}
+// biome-ignore lint/suspicious/noExplicitAny: <explanation>
+function makeNullable(schema) {
+    if (!schema || typeof schema !== "object")
+        return schema;
+    const newSchema = { ...schema };
+    if (Array.isArray(schema.type)) {
+        if (!schema.type.includes("null")) {
+            newSchema.type = [...schema.type, "null"];
+        }
+    }
+    else if (schema.type) {
+        newSchema.type = [schema.type, "null"];
+    }
+    // Recursively process nested properties
+    if (schema.properties) {
+        newSchema.properties = Object.entries(schema.properties).reduce((acc, [key, value]) => ({
+            // biome-ignore lint/performance/noAccumulatingSpread: <explanation>
+            ...acc,
+            [key]: makeNullable(value),
+        }), {});
+    }
+    if (schema.items) {
+        newSchema.items = makeNullable(schema.items);
+    }
+    return newSchema;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aigne/agent-library",
-  "version": "1.10.0",
+  "version": "1.11.0",
   "description": "Collection of agent libraries for AIGNE framework",
   "publishConfig": {
     "access": "public"
@@ -39,12 +39,17 @@
     }
   },
   "dependencies": {
+    "jsonata": "^2.0.6",
+    "jsonschema": "^1.5.0",
+    "to-json-schema": "^0.2.5",
     "fastq": "^1.19.1",
     "yaml": "^2.7.1",
     "zod": "^3.24.4",
-    "@aigne/core": "^1.17.0"
+    "@aigne/core": "^1.17.0",
+    "@aigne/openai": "^0.2.0"
   },
   "devDependencies": {
+    "@types/to-json-schema": "^0.2.4",
     "@types/bun": "^1.2.12",
     "npm-run-all": "^4.1.5",
     "rimraf": "^6.0.1",