@aigne/core 0.0.1

package/lib/esm/types/resource/index.js

@@ -0,0 +1,26 @@
+import Joi from 'joi';
+import { projectSettingsSchema } from './project';
+export * from './project';
+export const ResourceTypes = [
+    'application',
+    'tool',
+    'llm-adapter',
+    'aigc-adapter',
+    'knowledge',
+    'template',
+    'example',
+];
+const resourceProjectSchema = Joi.object({
+    project: projectSettingsSchema.required(),
+    agents: Joi.array().items(Joi.object()).required(),
+    config: Joi.object().default({}),
+    cron: Joi.object().default({}),
+    memory: Joi.object().default({}),
+}).options({ stripUnknown: true });
+export async function validateResourceProject(value) {
+    return resourceProjectSchema.validateAsync({
+        ...value,
+        agents: value.agents || value.assistants,
+        cron: value.cron || value.cronConfig,
+    });
+}

package/lib/esm/types/resource/project.js

@@ -0,0 +1,29 @@
+import Joi from 'joi';
+export const projectSettingsSchema = Joi.object({
+    id: Joi.string().required(),
+    name: Joi.string().empty(['', null]),
+    description: Joi.string().empty(['', null]),
+    model: Joi.string().empty(['', null]),
+    temperature: Joi.number().empty(['', null]),
+    topP: Joi.number().empty(['', null]),
+    presencePenalty: Joi.number().empty(['', null]),
+    frequencyPenalty: Joi.number().empty(['', null]),
+    maxTokens: Joi.number().empty(['', null]),
+    createdAt: Joi.alternatives(Joi.string().isoDate(), Joi.date().cast('string')).empty(['', null]),
+    updatedAt: Joi.alternatives(Joi.string().isoDate(), Joi.date().cast('string')).empty(['', null]),
+    createdBy: Joi.string().empty(['', null]),
+    updatedBy: Joi.string().empty(['', null]),
+    appearance: Joi.object({
+        primaryColor: Joi.string().empty(['', null]),
+        typography: Joi.object({
+            fontFamily: Joi.string().empty(['', null]),
+            heading: Joi.object({
+                fontFamily: Joi.string().empty(['', null]),
+            }).empty(['', null]),
+        }).empty(['', null]),
+    }).empty(['', null]),
+    readme: Joi.string().empty(['', null]).optional(),
+    executor: Joi.object().empty(['', null]).optional(),
+})
+    .rename('_id', 'id', { override: true, ignoreUndefined: true })
+    .options({ stripUnknown: true });

package/lib/esm/types/runtime/agent.js

@@ -0,0 +1 @@
+export {};

package/lib/esm/types/runtime/error.js

@@ -0,0 +1,14 @@
+export var RuntimeErrorType;
+(function (RuntimeErrorType) {
+    RuntimeErrorType["MissingSecretError"] = "MissingSecretError";
+    RuntimeErrorType["RequestExceededError"] = "RequestExceededError";
+    RuntimeErrorType["ProjectOwnerRequestExceededError"] = "ProjectOwnerRequestExceededError";
+    RuntimeErrorType["ProjectLimitExceededError"] = "ProjectLimitExceededError";
+})(RuntimeErrorType || (RuntimeErrorType = {}));
+export class RuntimeError extends Error {
+    type;
+    constructor(type, message) {
+        super(message);
+        this.type = type;
+    }
+}

package/lib/esm/types/runtime/index.js

@@ -0,0 +1,20 @@
+export * from './schema';
+export * from './error';
+export var ExecutionPhase;
+(function (ExecutionPhase) {
+    ExecutionPhase["EXECUTE_BLOCK_START"] = "EXECUTE_BLOCK_START";
+    ExecutionPhase["EXECUTE_SELECT_STOP"] = "EXECUTE_SELECT_STOP";
+    ExecutionPhase["EXECUTE_ASSISTANT_START"] = "EXECUTE_ASSISTANT_START";
+    ExecutionPhase["EXECUTE_ASSISTANT_RUNNING"] = "EXECUTE_BLOCK_RUNNING";
+    ExecutionPhase["EXECUTE_ASSISTANT_END"] = "EXECUTE_ASSISTANT_END";
+})(ExecutionPhase || (ExecutionPhase = {}));
+export var AssistantResponseType;
+(function (AssistantResponseType) {
+    AssistantResponseType["ERROR"] = "ERROR";
+    AssistantResponseType["LOG"] = "LOG";
+    AssistantResponseType["INPUT"] = "INPUT";
+    AssistantResponseType["CHUNK"] = "CHUNK";
+    AssistantResponseType["USAGE"] = "USAGE";
+    AssistantResponseType["EXECUTE"] = "EXECUTE";
+    AssistantResponseType["INPUT_PARAMETER"] = "INPUT_PARAMETER";
+})(AssistantResponseType || (AssistantResponseType = {}));

package/lib/esm/types/runtime/runtime-resource-blocklet-state.js

@@ -0,0 +1 @@
+export const RUNTIME_RESOURCE_BLOCKLET_STATE_GLOBAL_VARIABLE = '__AI_RUNTIME_RESOURCE_BLOCKLET_STATE__';

package/lib/esm/types/runtime/schema.js

@@ -0,0 +1,249 @@
+import Joi from 'joi';
+import omitBy from 'lodash/omitBy';
+import toLower from 'lodash/toLower';
+export const variableBlockListForAgent = {
+    prompt: {
+    // block: new Set(['$images']),
+    },
+    image: {
+    // allow: new Set(['$images']),
+    },
+};
+export const runtimeVariablesSchema = {
+    $text: {
+        type: 'string',
+        description: 'Text Stream',
+        faker: 'lorem.paragraph',
+    },
+    $images: {
+        type: 'array',
+        description: 'Generated Images',
+        element: {
+            id: '',
+            type: 'object',
+            properties: [
+                {
+                    id: '',
+                    type: 'string',
+                    name: 'url',
+                    description: 'Image Url',
+                    required: true,
+                    faker: 'image.url',
+                },
+            ],
+        },
+    },
+    '$suggested.questions': {
+        type: 'array',
+        description: 'Generate 3 questions for users to ask you based on answers and context',
+        element: {
+            id: '',
+            type: 'object',
+            properties: [
+                {
+                    id: '',
+                    type: 'string',
+                    name: 'question',
+                    required: true,
+                    faker: 'lorem.sentence',
+                },
+            ],
+        },
+        required: true,
+    },
+    '$reference.links': {
+        type: 'array',
+        description: 'List all referenced links in the generated text',
+        element: {
+            id: '',
+            type: 'object',
+            properties: [
+                {
+                    id: '',
+                    type: 'string',
+                    name: 'title',
+                    required: true,
+                    faker: 'lorem.sentence',
+                },
+                {
+                    id: '',
+                    type: 'string',
+                    name: 'url',
+                    required: true,
+                    faker: 'internet.url',
+                },
+            ],
+        },
+        required: true,
+    },
+};
+export function outputVariablesToJsonSchema(assistant, { variables, includeRuntimeOutputVariables, includeFaker, }) {
+    const variableToSchema = (variable) => {
+        if (variable.from?.type === 'input')
+            return undefined;
+        if (!includeRuntimeOutputVariables && ignoreJsonSchemaOutputs.has(variable.name))
+            return undefined;
+        if (variable.name && isRuntimeOutputVariable(variable.name)) {
+            const runtimeVariable = runtimeVariablesSchema[variable.name];
+            if (!runtimeVariable)
+                return undefined;
+            return variableToSchema({
+                ...runtimeVariable,
+                description: [runtimeVariable.description, variable.description].filter((i) => !!i).join('\n'),
+            });
+        }
+        if (variable.variable) {
+            const { key, scope } = variable.variable;
+            const v = variables.find((i) => toLower(i.key) === toLower(key) && i.scope === scope);
+            if (!v?.type)
+                throw new Error(`Variable ${key} not found from ${scope}`);
+            return variableToSchema(v.type);
+        }
+        return omitBy({
+            type: variable.type || 'string',
+            description: variable.description,
+            properties: variable.type === 'object' && variable.properties
+                ? Object.fromEntries(variable.properties
+                    .map((property) => [property.name, variableToSchema(property)])
+                    .filter((i) => i[0] && i[1]))
+                : undefined,
+            items: variable.type === 'array' && variable.element ? variableToSchema(variable.element) : undefined,
+            additionalProperties: variable.type === 'object' ? false : undefined,
+            required: variable.type === 'object' && variable.properties?.length
+                ? variable.properties
+                    .filter((i) => i.name &&
+                    (i.required ||
+                        (runtimeOutputVariableSet.has(i.name) &&
+                            runtimeVariablesSchema[i.name] &&
+                            !ignoreJsonSchemaOutputs.has(i.name))))
+                    .map((i) => i.name)
+                : undefined,
+            faker: includeFaker ? variable.faker : undefined,
+        }, (v) => v === undefined);
+    };
+    const outputVariables = (assistant.outputVariables ?? []).filter((i) => !i.hidden && i.from?.type !== 'callAgent' && !i.valueTemplate?.trim());
+    return variableToSchema({ type: 'object', properties: outputVariables });
+}
+export function outputVariablesToJoiSchema(assistant, { partial, variables }) {
+    const variableToSchema = (variable) => {
+        let schema;
+        if (variable.from?.type === 'input') {
+            const fromId = variable.from.id;
+            const input = assistant.parameters?.find((i) => i.id === fromId && !i.hidden);
+            if (input) {
+                return Joi.any();
+            }
+            return undefined;
+        }
+        if (variable.from?.type === 'output') {
+            return Joi.any();
+        }
+        if (variable.name && isRuntimeOutputVariable(variable.name)) {
+            if (variable.name === RuntimeOutputVariable.llmResponseStream) {
+                schema = Joi.any();
+            }
+            else {
+                const runtimeVariable = runtimeVariablesSchema[variable.name];
+                if (!runtimeVariable)
+                    return undefined;
+                schema = variableToSchema({ ...runtimeVariable });
+                if (schema) {
+                    schema = Joi.alternatives().try(schema, Joi.any().empty(Joi.any()));
+                }
+            }
+            return schema;
+        }
+        if (variable.variable) {
+            const { key, scope } = variable.variable;
+            const v = variables.find((i) => toLower(i.key) === toLower(key) && i.scope === scope);
+            if (!v?.type)
+                return undefined;
+            schema = variableToSchema(v.type);
+        }
+        else if (!variable.type || variable.type === 'string') {
+            schema = Joi.string().empty(['', null]);
+        }
+        else if (variable.type === 'number') {
+            schema = Joi.number().empty([null, '']);
+        }
+        else if (variable.type === 'boolean') {
+            schema = Joi.boolean().empty([null, '']);
+        }
+        else if (variable.type === 'object') {
+            if (variable.properties?.length) {
+                schema = Joi.object(Object.fromEntries((variable.properties ?? [])
+                    .map((property) => [property.name, variableToSchema(property)])
+                    .filter((i) => i[0] && i[1])))
+                    .empty([null, ''])
+                    .options({ stripUnknown: true });
+            }
+            else {
+                schema = Joi.any().empty(['', null]);
+            }
+        }
+        else if (variable.type === 'array') {
+            schema = Joi.array()
+                .empty([null, ''])
+                .items((variable.element && variableToSchema(variable.element)) || Joi.string().empty([null, '']));
+        }
+        if (!schema)
+            return undefined;
+        if ('defaultValue' in variable) {
+            schema = schema.default(variable.defaultValue);
+        }
+        if (variable.required) {
+            schema = schema.required();
+        }
+        return schema;
+    };
+    const outputVariables = (assistant.outputVariables ?? []).filter((i) => !i.hidden && i.from?.type !== 'callAgent' && !i.valueTemplate);
+    return variableToSchema({
+        type: 'object',
+        properties: partial ? outputVariables.map((i) => ({ ...i, required: false })) : outputVariables,
+    });
+}
+export var RuntimeOutputVariable;
+(function (RuntimeOutputVariable) {
+    RuntimeOutputVariable["llmResponseStream"] = "$llmResponseStream";
+    RuntimeOutputVariable["text"] = "$text";
+    RuntimeOutputVariable["images"] = "$images";
+    RuntimeOutputVariable["suggestedQuestions"] = "$suggested.questions";
+    RuntimeOutputVariable["referenceLinks"] = "$reference.links";
+    RuntimeOutputVariable["appearancePage"] = "$appearance.page";
+    RuntimeOutputVariable["appearanceInput"] = "$appearance.input";
+    RuntimeOutputVariable["appearanceOutput"] = "$appearance.output";
+    RuntimeOutputVariable["children"] = "$children";
+    RuntimeOutputVariable["share"] = "$share";
+    RuntimeOutputVariable["openingQuestions"] = "$openingQuestions";
+    RuntimeOutputVariable["openingMessage"] = "$openingMessage";
+    RuntimeOutputVariable["profile"] = "$profile";
+})(RuntimeOutputVariable || (RuntimeOutputVariable = {}));
+const runtimeOutputVariableSet = new Set(Object.values(RuntimeOutputVariable));
+const ignoreJsonSchemaOutputs = new Set([
+    RuntimeOutputVariable.text,
+    RuntimeOutputVariable.images,
+]);
+export function isRuntimeOutputVariable(variable) {
+    return Object.values(RuntimeOutputVariable).includes(variable);
+}
+export function jsonSchemaToOpenAIJsonSchema(schema) {
+    if (schema?.type === 'object') {
+        const { required, properties } = schema;
+        return {
+            ...schema,
+            properties: Object.fromEntries(Object.entries(properties).map(([key, value]) => {
+                const valueSchema = jsonSchemaToOpenAIJsonSchema(value);
+                // NOTE: All fields must be required https://platform.openai.com/docs/guides/structured-outputs/all-fields-must-be-required
+                return [key, required?.includes(key) ? valueSchema : { anyOf: [valueSchema, { type: ['null'] }] }];
+            })),
+            required: Object.keys(properties),
+        };
+    }
+    if (schema?.type === 'array') {
+        return {
+            ...schema,
+            items: jsonSchemaToOpenAIJsonSchema(schema.items),
+        };
+    }
+    return schema;
+}

package/lib/esm/utils/cron-job.js

@@ -0,0 +1,44 @@
+import { CronJob } from 'cron';
+export const CronJobDefaultGroup = 'default';
+export class CronJobManager {
+    groups = {};
+    resetJobs(jobs, { groupId = CronJobDefaultGroup } = {}) {
+        this.destroyGroup({ groupId });
+        for (const job of jobs) {
+            this.resetJob(job, { groupId });
+        }
+    }
+    resetJob(job, { groupId = CronJobDefaultGroup } = {}) {
+        if (!job.cronTime)
+            throw new Error('cronTime is required');
+        this.groups[groupId] ??= { jobs: {} };
+        const group = this.groups[groupId];
+        this.stopJob(job.id, { groupId });
+        const cronJob = CronJob.from({ cronTime: job.cronTime, onTick: job.onTick, utcOffset: 0, start: true });
+        group.jobs[job.id] = { cronJob, job };
+    }
+    stopJob(jobId, { groupId = CronJobDefaultGroup } = {}) {
+        const group = this.groups[groupId];
+        if (!group)
+            return;
+        const item = group.jobs[jobId];
+        if (!item)
+            return;
+        item.cronJob.stop();
+        delete group.jobs[jobId];
+    }
+    destroyGroup({ groupId }) {
+        const group = this.groups[groupId];
+        if (!group)
+            return;
+        for (const item of Object.values(group.jobs)) {
+            this.stopJob(item.job.id, { groupId });
+        }
+        delete this.groups[groupId];
+    }
+    destroy() {
+        for (const groupId of Object.keys(this.groups)) {
+            this.destroyGroup({ groupId });
+        }
+    }
+}

package/lib/esm/utils/extract-metadata-transform.js

@@ -0,0 +1,54 @@
+import { TransformStream } from 'stream/web';
+export class ExtractMetadataTransform extends TransformStream {
+    buffer = '';
+    cursor = 0;
+    state = 'none';
+    constructor({ start, end }) {
+        super({
+            transform: async (chunk, controller) => {
+                if (!start || !end) {
+                    controller.enqueue({ type: 'text', text: chunk });
+                    return;
+                }
+                this.buffer += chunk;
+                for (;;) {
+                    if (this.state === 'none') {
+                        const found = findMatchIndex(this.buffer, this.cursor, start);
+                        if (found.start > this.cursor) {
+                            const text = this.buffer.slice(this.cursor, found.start);
+                            this.cursor = found.start;
+                            controller.enqueue({ type: 'text', text });
+                        }
+                        if (found.end) {
+                            this.state = 'start';
+                            this.cursor = found.end;
+                        }
+                    }
+                    if (this.state === 'start') {
+                        const found = findMatchIndex(this.buffer, this.cursor, end);
+                        if (found.end) {
+                            const match = this.buffer.slice(this.cursor, found.start);
+                            controller.enqueue({ type: 'match', text: match });
+                            this.state = 'none';
+                            this.cursor = found.end;
+                            continue;
+                        }
+                    }
+                    break;
+                }
+            },
+        });
+    }
+}
+function findMatchIndex(str, position, match) {
+    const i = str.indexOf(match, position);
+    if (i >= 0)
+        return { start: i, end: i + match.length };
+    for (let i = match.length - 1; i > 0; i--) {
+        const m = match.slice(0, i);
+        if (str.endsWith(m)) {
+            return { start: str.length - m.length };
+        }
+    }
+    return { start: str.length };
+}

package/lib/esm/utils/extract-metadata-transform.test.js

@@ -0,0 +1,59 @@
+import { ReadableStream } from 'stream/web';
+import { describe, expect, test } from '@jest/globals';
+import { ExtractMetadataTransform } from './extract-metadata-transform';
+describe('ExtractMetadataTransform', () => {
+    async function run(str, start, end, step = 1) {
+        const s = new ReadableStream({
+            start(controller) {
+                let t = '';
+                for (const i of str) {
+                    t += i;
+                    if (t.length === step) {
+                        controller.enqueue(t);
+                        t = '';
+                    }
+                }
+                controller.enqueue(t);
+                controller.close();
+            },
+        }).pipeThrough(new ExtractMetadataTransform({ start, end }));
+        let text = '';
+        const matched = [];
+        for await (const i of s) {
+            if (i.type === 'text') {
+                text += i.text;
+            }
+            else if (i.type === 'match') {
+                matched.push(i.text);
+            }
+        }
+        return { text, matched };
+    }
+    const cases = [
+        {
+            input: 'hello<metadata>{"name":"foo","age":120}</metadata>world how<metadata>{"name":"bar","age":130}</metadata> to',
+            start: '<metadata>',
+            end: '</metadata>',
+            expected: { text: 'helloworld how to', matched: ['{"name":"foo","age":120}', '{"name":"bar","age":130}'] },
+        },
+        {
+            input: '```metadata{"name":"foo","age":120}```',
+            start: '```metadata',
+            end: '```',
+            expected: { text: '', matched: ['{"name":"foo","age":120}'] },
+        },
+        {
+            input: '```metadata{"name":"foo","age":120}``````metadata{"name":"bar","age":130}```',
+            start: '```metadata',
+            end: '```',
+            expected: { text: '', matched: ['{"name":"foo","age":120}', '{"name":"bar","age":130}'] },
+        },
+    ];
+    test('extract metadata transform', async () => {
+        for (const c of cases) {
+            for (let i = 1; i < c.input.length; i++) {
+                expect([await run(c.input, c.start, c.end, i), i]).toEqual([c.expected, i]);
+            }
+        }
+    });
+});

package/lib/esm/utils/fs.js

@@ -0,0 +1,41 @@
+import fs from 'fs';
+import path from 'path';
+export async function copyFile(src, dest, { skip } = {}) {
+    if (skip?.(src))
+        return;
+    await new Promise((resolve, reject) => {
+        const readStream = fs.createReadStream(src);
+        const writeStream = fs.createWriteStream(dest);
+        readStream.on('error', reject);
+        writeStream.on('error', reject);
+        writeStream.on('finish', resolve);
+        readStream.pipe(writeStream);
+    });
+}
+export async function copyDirectory(src, dest, { skip } = {}) {
+    if (skip?.(src))
+        return;
+    await fs.promises.mkdir(dest, { recursive: true });
+    const entries = await fs.promises.readdir(src, { withFileTypes: true });
+    for (const entry of entries) {
+        const srcPath = path.join(src, entry.name);
+        const destPath = path.join(dest, entry.name);
+        if (entry.isDirectory()) {
+            await copyDirectory(srcPath, destPath, { skip });
+        }
+        else {
+            await copyFile(srcPath, destPath, { skip });
+        }
+    }
+}
+export async function copyRecursive(src, dest, { skip } = {}) {
+    if (skip?.(src))
+        return;
+    const srcStats = await fs.promises.stat(src);
+    if (srcStats.isDirectory()) {
+        await copyDirectory(src, dest, { skip });
+    }
+    else {
+        await copyFile(src, dest, { skip });
+    }
+}