agentlang 0.7.2 → 0.7.4

package/src/runtime/loader.ts

@@ -77,6 +77,7 @@ import {
   isFqName,
   makeFqName,
   maybeExtends,
+  objectAsString,
   preprocessRawConfig,
   registerInitFunction,
   rootRef,
@@ -342,6 +343,40 @@ export async function flushAllAndLoad(
   return await load(fileName, fsOptions, callback);
 }
 
+async function evaluateConfigPatterns(cfgPats: string): Promise<any> {
+  const cfgWf = `workflow createConfig{\n${cfgPats}}`;
+  const wf = await parseWorkflow(cfgWf);
+  const cfgStmts = new Array<Statement>();
+  const initInsts = new Array<Statement>();
+  for (let i = 0; i < wf.statements.length; ++i) {
+    const stmt: Statement = wf.statements[i];
+    if (stmt.pattern.crudMap) {
+      initInsts.push(await makeDeleteAllConfigStatement(stmt.pattern.crudMap));
+      initInsts.push(stmt);
+    } else {
+      cfgStmts.push(stmt);
+    }
+  }
+  if (initInsts.length > 0) {
+    registerInitFunction(async () => {
+      const env = new Environment('config.insts.env');
+      try {
+        await evaluateStatements(initInsts, env);
+        await env.commitAllTransactions();
+      } catch (reason: any) {
+        await env.rollbackAllTransactions();
+        console.error(`Failed to initialize config instances: ${reason}`);
+      }
+    });
+  }
+  if (cfgStmts.length > 0) {
+    const env = new Environment('config.env');
+    await evaluateStatements(cfgStmts, env);
+    return env.getLastResult();
+  }
+  return undefined;
+}
+
 export async function loadAppConfig(configDir: string): Promise<Config> {
   let cfgObj: any = undefined;
   const fs = await getFileSystem();
@@ -349,39 +384,12 @@ export async function loadAppConfig(configDir: string): Promise<Config> {
   if (await fs.exists(alCfgFile)) {
     const cfgPats = await fs.readFile(alCfgFile);
     if (canParse(cfgPats)) {
-      const cfgWf = `workflow createConfig{\n${cfgPats}}`;
-      const wf = await parseWorkflow(cfgWf);
-      const env = new Environment('config.env');
-      const cfgStmts = new Array<Statement>();
-      const initInsts = new Array<Statement>();
-      for (let i = 0; i < wf.statements.length; ++i) {
-        const stmt: Statement = wf.statements[i];
-        if (stmt.pattern.crudMap) {
-          initInsts.push(await makeDeleteAllConfigStatement(stmt.pattern.crudMap));
-          initInsts.push(stmt);
-        } else {
-          cfgStmts.push(stmt);
-        }
-      }
-      if (initInsts.length > 0) {
-        registerInitFunction(async () => {
-          const env = new Environment('config.insts.env');
-          try {
-            await evaluateStatements(initInsts, env);
-            await env.commitAllTransactions();
-          } catch (reason: any) {
-            await env.rollbackAllTransactions();
-            console.error(`Failed to initialize config instances: ${reason}`);
-          }
-        });
-      }
-      await evaluateStatements(cfgStmts, env);
-      cfgObj = env.getLastResult();
+      cfgObj = await evaluateConfigPatterns(cfgPats);
     }
   }
   try {
     const cfg = cfgObj
-      ? configFromObject(cfgObj)
+      ? await configFromObject(cfgObj)
       : await loadRawConfig(`${configDir}${path.sep}app.config.json`);
     return setAppConfig(cfg);
   } catch (err: any) {
@@ -1178,10 +1186,31 @@ export async function loadRawConfig(
   }
 }
 
-export function configFromObject(cfgObj: any, validate: boolean = true): any {
+function filterConfigEntityInstances(rawConfig: any): [any, Map<string, any>] {
+  const cfg = new Map<string, any>();
+  const insts = new Map<string, any>();
+  Object.entries(rawConfig).forEach(([key, value]: [string, any]) => {
+    if (isFqName(key)) {
+      insts.set(key, value);
+    } else {
+      cfg.set(key, value);
+    }
+  });
+  return [Object.fromEntries(cfg), insts];
+}
+
+async function configFromObject(cfgObj: any, validate: boolean = true): Promise<any> {
   const rawConfig = preprocessRawConfig(cfgObj);
   if (validate) {
-    return ConfigSchema.parse(rawConfig);
+    const [cfg, insts] = filterConfigEntityInstances(rawConfig);
+    const pats = new Array<string>();
+    insts.forEach((v: any, k: string) => {
+      pats.push(`{${k} ${objectAsString(v)}}`);
+    });
+    if (pats.length > 0) {
+      await evaluateConfigPatterns(pats.join('\n'));
+    }
+    return ConfigSchema.parse(cfg);
   }
   return rawConfig;
 }

package/src/runtime/logger.ts

@@ -1,4 +1,5 @@
 import { isNodeEnv } from '../utils/runtime.js';
+import { AppConfig } from './state.js';
 
 let DailyRotateFile: any;
 let winston: any;
@@ -16,34 +17,58 @@ if (isNodeEnv) {
 
 export let logger: any;
 
-if (isNodeEnv) {
-  const fileTransport = new DailyRotateFile({
-    level: 'debug',
-    filename: 'logs/app-%DATE%.log',
-    datePattern: 'YYYY-MM-DD',
-    maxSize: '20m',
-    maxFiles: '7d',
-  });
+function getLogLevel(): string {
+  if (isNodeEnv && process.env && process.env.DEBUG) {
+    return 'debug';
+  }
+  return AppConfig?.logging?.level || 'info';
+}
 
-  logger = winston.createLogger({
-    format: winston.format.combine(
-      winston.format.timestamp(),
-      winston.format.printf(({ timestamp, level, message }: any) => {
-        return `[${timestamp}] ${level}: ${message}`;
-      })
-    ),
-    transports: [fileTransport],
-  });
-} else {
-  function mkLogger(tag: string): Function {
-    return (msg: string) => {
-      console.log(`${tag}: ${msg}`);
+export function initializeLogger() {
+  const logLevel = getLogLevel();
+
+  if (isNodeEnv) {
+    const fileTransport = new DailyRotateFile({
+      level: logLevel,
+      filename: 'logs/app-%DATE%.log',
+      datePattern: 'YYYY-MM-DD',
+      maxSize: '20m',
+      maxFiles: '7d',
+    });
+
+    logger = winston.createLogger({
+      level: logLevel,
+      format: winston.format.combine(
+        winston.format.timestamp(),
+        winston.format.printf(({ timestamp, level, message }: any) => {
+          return `[${timestamp}] ${level}: ${message}`;
+        })
+      ),
+      transports: [fileTransport],
+    });
+  } else {
+    function mkLogger(tag: string): Function {
+      return (msg: string) => {
+        console.log(`${tag}: ${msg}`);
+      };
+    }
+    logger = {
+      debug: mkLogger('DEBUG'),
+      info: mkLogger('INFO'),
+      warn: mkLogger('WARN'),
+      error: mkLogger('ERROR'),
     };
   }
-  logger = {
-    debug: mkLogger('DEBUG'),
-    info: mkLogger('INFO'),
-    warn: mkLogger('WARN'),
-    error: mkLogger('ERROR'),
-  };
+}
+
+initializeLogger();
+
+export function updateLoggerFromConfig() {
+  if (isNodeEnv && logger) {
+    const logLevel = getLogLevel();
+    logger.level = logLevel;
+    if (logger.transports && logger.transports.length > 0) {
+      logger.transports[0].level = logLevel;
+    }
+  }
 }

package/src/runtime/module.ts

@@ -1248,7 +1248,7 @@ export class Relationship extends Record {
 
   constructor(
     name: string,
-    typ: string,
+    type: string,
     node1: RelationshipNode,
     node2: RelationshipNode,
     moduleName: string,
@@ -1256,7 +1256,7 @@ export class Relationship extends Record {
     props?: Map<string, any>
   ) {
     super(name, moduleName, scm);
-    if (typ == 'between') {
+    if (type == 'between') {
       this.relType = RelType.BETWEEN;
       this.addMetaAttributes();
     }
@@ -3999,7 +3999,7 @@ export function getAttributeNames(entityFqName: string): Array<string> {
   return [...scm.keys()];
 }
 
-export function maybeSetMetaAttributes(
+export function setMetaAttributes(
   attrs: InstanceAttributes,
   env: Environment,
   inUpdateMode: boolean = false
@@ -4010,3 +4010,12 @@ export function maybeSetMetaAttributes(
     attrs.set(SysAttr_CreatedBy, user);
   }
 }
+
+export function setAllMetaAttributes(
+  attrs: InstanceAttributes,
+  env: Environment,
+  inUpdateMode: boolean = false
+) {
+  attrs.set(SysAttr_Created, now());
+  setMetaAttributes(attrs, env, inUpdateMode);
+}

package/src/runtime/modules/core.ts

@@ -28,14 +28,16 @@ import {
 import { logger } from '../logger.js';
 import { Statement } from '../../language/generated/ast.js';
 import { parseModule, parseStatements } from '../../language/parser.js';
-import { Resolver } from '../resolvers/interface.js';
+import { GenericResolver, Resolver } from '../resolvers/interface.js';
 import {
   FlowSuspensionTag,
   ForceReadPermFlag,
   InternDynamicModule,
+  isRuntimeMode_dev,
   PathAttributeName,
 } from '../defs.js';
 import { getMonitor, getMonitorsForEvent, Monitor } from '../monitor.js';
+import { registerResolver, setResolver } from '../resolvers/registry.js';
 
 const CoreModuleDefinition = `module ${DefaultModuleName}
 
@@ -146,11 +148,12 @@ entity Module {
   definition String
 }
 
-resolver moduleResolver [agentlang/Module] {
-    create Core.createModule,
-    update Core.updateModule,
-    delete Core.deleteModule,
-    query Core.getModule
+entity PersistentModule extends Module {
+}
+
+workflow savePersistentModule {
+  purge {PersistentModule {name? savePersistentModule.name}}
+  {PersistentModule {name savePersistentModule.name, definition savePersistentModule.definition}}
 }
 
 entity Migration {
@@ -464,7 +467,21 @@ export async function internModuleHelper(
 }
 
 export async function createModule(_: Resolver, inst: Instance) {
-  await internModuleHelper(inst.lookup('name'), inst.lookup('definition'));
+  const n = inst.lookup('name');
+  const d = inst.lookup('definition');
+  const env = new Environment('module-env');
+  try {
+    await parseAndEvaluateStatement(
+      `{agentlang/savePersistentModule {name "${n}", definition "${d}"}}`,
+      undefined,
+      env
+    );
+    await env.commitAllTransactions();
+  } catch (reason: any) {
+    await env.rollbackAllTransactions();
+    logger.error(`Failed to persist module ${n} - ${reason}`);
+  }
+  await internModuleHelper(n, d);
   return inst;
 }
 
@@ -473,7 +490,20 @@ export async function updateModule(r: Resolver, inst: Instance) {
 }
 
 export async function deleteModule(_: Resolver, inst: Instance) {
-  removeModule(inst.lookup('name'));
+  const n = inst.lookup('name');
+  const env = new Environment('module-env');
+  try {
+    await parseAndEvaluateStatement(
+      `purge {agentlang/PersistentModule {name? "${n}"}}`,
+      undefined,
+      env
+    );
+    await env.commitAllTransactions();
+  } catch (reason: any) {
+    await env.rollbackAllTransactions();
+    logger.error(`Failed to purge persistent module ${n} - ${reason}`);
+  }
+  removeModule(n);
   return inst;
 }
 
@@ -489,7 +519,46 @@ export async function getModule(_: Resolver, inst: Instance) {
       return [makeInstance('agentlang', 'Module', attrs)];
     }
   }
-  return null;
+  return [];
+}
+
+async function internPersistentModules() {
+  try {
+    const insts: Instance[] = await parseAndEvaluateStatement(`{agentlang/PersistentModule? {}}`);
+    for (let i = 0; i < insts.length; ++i) {
+      const inst = insts[i];
+      const n = inst.lookup('name');
+      if (!isModule(n)) await internModuleHelper(n, inst.lookup('definition'));
+    }
+  } catch (reason: any) {
+    logger.warn(`Failed to intern persistent modules: ${reason}`);
+  }
+}
+
+export function initCoreModuleManager() {
+  const ModuleResolverName = 'agentlang/moduleResolver';
+  const ModuleResolver = new GenericResolver(ModuleResolverName, {
+    create: createModule,
+    upsert: createModule,
+    update: updateModule,
+    query: getModule,
+    delete: deleteModule,
+    startTransaction: undefined,
+    commitTransaction: undefined,
+    rollbackTransaction: undefined,
+  });
+
+  registerResolver(ModuleResolverName, () => {
+    return ModuleResolver;
+  });
+
+  setResolver('agentlang/Module', ModuleResolverName);
+
+  if (isRuntimeMode_dev()) {
+    setInterval(() => {
+      internPersistentModules();
+    }, 10000);
+  }
 }
 
 const SqlSep = ';\n\n';

package/src/runtime/resolvers/interface.ts

@@ -14,7 +14,7 @@ import {
   newInstanceAttributes,
   Relationship,
 } from '../module.js';
-import { CrudType, nameToPath } from '../util.js';
+import { CrudType, nameToPath, generateLoggerCallId } from '../util.js';
 import { DefaultAuthInfo, ResolverAuthInfo } from './authinfo.js';
 
 export type JoinInfo = {
@@ -272,11 +272,22 @@ export class GenericResolver extends Resolver {
   }
 
   public override async createInstance(inst: Instance): Promise<any> {
+    const callId = generateLoggerCallId();
+    let attrVals;
+    if (inst.attributes) {
+      attrVals = JSON.stringify(Object.fromEntries(inst.attributes));
+    }
+    logger.debug(
+      `${callId}: Resolver createInstance called for ${inst.moduleName + '/' + inst.name} with values ${attrVals}`
+    );
+    let result;
     if (this.implementation?.create) {
-      return await this.implementation.create(this, inst);
+      result = await this.implementation.create(this, inst);
     } else {
-      return await super.createInstance(inst);
+      result = await super.createInstance(inst);
     }
+    logger.debug(`${callId}: Resolver createInstance response: ${JSON.stringify(result)}`);
+    return result;
   }
 
   public override async upsertInstance(inst: Instance): Promise<any> {
@@ -287,20 +298,57 @@ export class GenericResolver extends Resolver {
   }
 
   public override async updateInstance(inst: Instance, newAttrs: InstanceAttributes): Promise<any> {
+    const callId = generateLoggerCallId();
+
+    const newAttrsVals = JSON.stringify(Object.fromEntries(newAttrs));
+    logger.debug(
+      `${callId} Resolver updateInstance called for ${inst.moduleName + '/' + inst.name} with values ${newAttrsVals}`
+    );
+    if (inst.queryAttributes && inst.queryAttributeValues) {
+      const qattr = JSON.stringify(Object.fromEntries(inst.queryAttributes));
+      const qattrValues = JSON.stringify(Object.fromEntries(inst.queryAttributeValues));
+      logger.debug(`${callId}: Query attributes: ${qattr}, values ${qattrValues}`);
+    }
+
+    let result;
     if (this.implementation?.update) {
-      return await this.implementation.update(this, inst, newAttrs);
+      result = await this.implementation.update(this, inst, newAttrs);
+    } else {
+      result = await super.updateInstance(inst, newAttrs);
     }
-    return await super.updateInstance(inst, newAttrs);
+    logger.debug(`${callId}: Resolver updateInstance response: ${JSON.stringify(result)}`);
+    return result;
   }
 
   public override async queryInstances(inst: Instance, queryAll: boolean): Promise<any> {
+    const callId = generateLoggerCallId();
+    logger.debug(
+      `${callId}: Resolver queryInstances called for ${inst.moduleName + '/' + inst.name}`
+    );
+    if (inst.queryAttributes && inst.queryAttributeValues) {
+      const qattr = JSON.stringify(Object.fromEntries(inst.queryAttributes));
+      const qattrValues = JSON.stringify(Object.fromEntries(inst.queryAttributeValues));
+      logger.debug(`${callId}: Query attributes: ${qattr}, values ${qattrValues}`);
+    }
+    let result;
     if (this.implementation?.query) {
-      return await this.implementation.query(this, inst, queryAll);
+      result = await this.implementation.query(this, inst, queryAll);
+    } else {
+      result = await super.queryInstances(inst, queryAll);
     }
-    return await super.queryInstances(inst, queryAll);
+    logger.debug(`${callId}: Resolver queryInstances response: ${JSON.stringify(result)}`);
+    return result;
   }
 
   public override async deleteInstance(inst: Instance | Instance[], purge: boolean): Promise<any> {
+    if (inst instanceof Instance) {
+      if (inst.queryAttributes)
+        logger.debug(`Resolver deleteInstance called for ${inst.moduleName + '/' + inst.name}`);
+    } else {
+      logger.debug(
+        `Resolver deleteInstance called for ${inst.map(i => i.moduleName + '/' + i.name).join(', ')}`
+      );
+    }
     if (this.implementation?.delete) {
       return await this.implementation.delete(this, inst, purge);
     }

package/src/runtime/resolvers/sqldb/database.ts

@@ -24,6 +24,7 @@ import {
   RbacPermissionFlag,
   RbacSpecification,
   Relationship,
+  setAllMetaAttributes,
 } from '../../module.js';
 import { isString } from '../../util.js';
 import {
@@ -33,6 +34,7 @@ import {
   isRuntimeMode_generate_migration,
   isRuntimeMode_init_schema,
   isRuntimeMode_migration,
+  isRuntimeMode_test,
   isRuntimeMode_undo_migration,
   PathAttributeName,
   UnauthorisedError,
@@ -170,11 +172,15 @@ function mkDbName(): string {
   return process.env.AGENTLANG_DB_NAME || `db-${Date.now()}`;
 }
 
+function needSync(): boolean {
+  return isRuntimeMode_dev() || isRuntimeMode_test() || isRuntimeMode_init_schema();
+}
+
 function makePostgresDataSource(
   entities: EntitySchema[],
   config: DatabaseConfig | undefined
 ): DataSource {
-  const synchronize = isRuntimeMode_dev() || isRuntimeMode_init_schema();
+  const synchronize = needSync();
   //const runMigrations = isRuntimeMode_migration() || isRuntimeMode_undo_migration() || !synchronize;
   return new DataSource({
     type: 'postgres',
@@ -207,7 +213,7 @@ function makeSqliteDataSource(
   entities: EntitySchema[],
   config: DatabaseConfig | undefined
 ): DataSource {
-  const synchronize = isRuntimeMode_dev() || isRuntimeMode_init_schema();
+  const synchronize = needSync();
   //const runMigrations = isRuntimeMode_migration() || isRuntimeMode_undo_migration() || !synchronize;
   return new DataSource({
     type: 'sqlite',
@@ -646,6 +652,7 @@ export async function insertBetweenRow(
     if (relEntry.isOneToMany()) {
       attrs.set(relEntry.joinNodesAttributeName(), `${p1}_${p2}`);
     }
+    setAllMetaAttributes(attrs, ctx.activeEnv);
     const row = Object.fromEntries(attrs);
     await insertRow(n, row, ctx.clone().setNeedAuthCheck(false), false);
   } else {

package/src/runtime/resolvers/sqldb/impl.ts

@@ -435,6 +435,7 @@ export class SqlDbResolver extends Resolver {
     const a1: string = relEntry.node1.alias;
     const a2: string = relEntry.node2.alias;
     const n1path: any = orUpdate ? firstNode.lookup(PathAttributeName) : undefined;
+    const ctx = this.getDbContext(relEntry.getFqName());
     if (relEntry.isOneToOne()) {
       await this.updateInstance(
         node1,
@@ -452,18 +453,10 @@ export class SqlDbResolver extends Resolver {
             [a1, n1path],
             [a2, secondNode.lookup(PathAttributeName)],
           ],
-          this.getDbContext(relEntry.getFqName())
+          ctx
         );
       }
-      await insertBetweenRow(
-        n,
-        a1,
-        a2,
-        firstNode,
-        secondNode,
-        relEntry,
-        this.getDbContext(relEntry.getFqName())
-      );
+      await insertBetweenRow(n, a1, a2, firstNode, secondNode, relEntry, ctx);
     }
   }
 

package/src/runtime/state.ts

@@ -111,6 +111,11 @@ export const ConfigSchema = z.object({
       })
     )
     .optional(),
+  logging: z
+    .object({
+      level: z.enum(['error', 'warn', 'info', 'debug']).default('info'),
+    })
+    .optional(),
 });
 
 export type Config = z.infer<typeof ConfigSchema>;

package/src/runtime/util.ts

@@ -612,6 +612,10 @@ export function generateUrlSafePassword(length: number = 8): string {
   return chars.join('');
 }
 
+export function generateLoggerCallId() {
+  return Math.floor(10000 + Math.random() * 90000);
+}
+
 const JS_PREFIX = '#js';
 
 export function preprocessRawConfig(rawConfig: any): any {
@@ -637,3 +641,12 @@ declare global {
 export function setScecretReader(f: ReadSecret) {
   globalThis.readSecret = f;
 }
+
+export function objectAsString(obj: any) {
+  const entries = new Array<string>();
+  Object.entries(obj).forEach(([k, v]) => {
+    const vv = typeof v === 'string' ? `"${v}"` : v;
+    entries.push(`${k}: ${vv}`);
+  });
+  return `{${entries.join(', ')}}`;
+}