agentlang 0.7.9 → 0.7.11

package/src/runtime/resolvers/sqldb/database.ts

@@ -41,6 +41,7 @@ import {
 } from '../../defs.js';
 import { saveMigration } from '../../modules/core.js';
 import { getAppSpec } from '../../loader.js';
+import { WhereClause } from '../interface.js';
 
 export let defaultDataSource: DataSource | undefined;
 
@@ -880,17 +881,30 @@ function objectToRawWhereClause(queryObj: object, queryVals: any, tableName?: st
   }
 }
 
+export type QuerySpec = {
+  queryObj: object | undefined;
+  queryVals: object | undefined;
+  aggregates: Map<string, string> | undefined;
+  groupBy: string[] | undefined;
+  orderBy: string[] | undefined;
+  orderByDesc: 'DESC' | 'ASC';
+  joinClauses: JoinClause[] | undefined;
+  intoSpec: Map<string, string> | undefined;
+  whereClauses: WhereClause[] | undefined;
+  distinct: boolean;
+};
+
 export async function getMany(
   tableName: string,
-  queryObj: object | undefined,
-  queryVals: object | undefined,
-  distinct: boolean,
+  querySpec: QuerySpec,
   ctx: DbContext
 ): Promise<any> {
   const alias: string = tableName.toLowerCase();
   const queryStr: string = withNotDeletedClause(
     alias,
-    queryObj !== undefined ? objectToWhereClause(queryObj, queryVals, alias) : ''
+    querySpec.queryObj !== undefined
+      ? objectToWhereClause(querySpec.queryObj, querySpec.queryVals, alias)
+      : ''
   );
   let ownersJoinCond: string[] | undefined;
   let ot: string = '';
@@ -927,30 +941,54 @@ export async function getMany(
   const qb: SelectQueryBuilder<any> = getDatasourceForTransaction(ctx.txnId)
     .getRepository(tableName)
     .createQueryBuilder();
+  const hasAggregates = querySpec.aggregates !== undefined;
+  if (hasAggregates) {
+    qb.select([]);
+    querySpec.aggregates?.forEach((f: string, n: string) => {
+      qb.addSelect(f, n);
+    });
+  }
+  if (querySpec.groupBy !== undefined) {
+    querySpec.groupBy.forEach((gb: string) => {
+      qb.addGroupBy(gb);
+    });
+  }
+  if (querySpec.orderBy !== undefined) {
+    querySpec.orderBy.forEach((ob: string) => {
+      qb.addOrderBy(ob, querySpec.orderByDesc);
+    });
+  }
   if (ownersJoinCond) {
     qb.innerJoin(ot, otAlias, ownersJoinCond.join(' AND '));
   }
-  if (distinct) {
+  if (querySpec.distinct) {
     qb.distinct(true);
   }
-  qb.where(queryStr, queryVals);
-  return await qb.getMany();
+  qb.where(queryStr, querySpec.queryVals);
+  if (hasAggregates) return await qb.getRawMany();
+  else return await qb.getMany();
 }
 
 export async function getManyByJoin(
   tableName: string,
-  queryObj: object | undefined,
-  queryVals: object | undefined,
-  joinClauses: JoinClause[],
-  intoSpec: Map<string, string>,
-  distinct: boolean,
+  querySpec: QuerySpec,
   ctx: DbContext
 ): Promise<any> {
   const alias: string = tableName.toLowerCase();
-  const queryStr: string = withNotDeletedClause(
+  let queryStr: string = withNotDeletedClause(
     alias,
-    queryObj !== undefined ? objectToRawWhereClause(queryObj, queryVals, alias) : ''
+    querySpec.queryObj !== undefined
+      ? objectToRawWhereClause(querySpec.queryObj, querySpec.queryVals, alias)
+      : ''
   );
+  if (querySpec.whereClauses) {
+    const qs = new Array<string>();
+    querySpec.whereClauses.forEach((wc: WhereClause) => {
+      const v = isString(wc.qval) ? `'${wc.qval}'` : wc.qval;
+      qs.push(`${wc.attrName} ${wc.op} ${v}`);
+    });
+    queryStr = `${queryStr} AND ${qs.join(' AND ')}`;
+  }
   let ot: string = '';
   let otAlias: string = '';
   if (!ctx.isPermitted()) {
@@ -961,7 +999,7 @@ export async function getManyByJoin(
     if (!hasGlobalPerms) {
       ot = ownersTable(tableName);
       otAlias = ot.toLowerCase();
-      joinClauses.push({
+      querySpec.joinClauses?.push({
         tableName: otAlias,
         joinOn: [
           makeJoinOn(`${otAlias}.path`, `${alias}.${PathAttributeName}`),
@@ -972,7 +1010,7 @@ export async function getManyByJoin(
     }
   }
   const joinSql = new Array<string>();
-  joinClauses.forEach((jc: JoinClause) => {
+  querySpec.joinClauses?.forEach((jc: JoinClause) => {
     const joinType = jc.joinType ? jc.joinType : 'inner join';
     joinSql.push(
       `${joinType} ${jc.tableName} as ${jc.tableName} on ${joinOnAsSql(jc.joinOn)} AND ${jc.tableName}.${DeletedFlagAttributeName} = false`
@@ -984,7 +1022,20 @@ export async function getManyByJoin(
       }
     }
   });
-  const sql = `SELECT ${distinct ? 'DISTINCT' : ''} ${intoSpecToSql(intoSpec)} FROM ${tableName} ${joinSql.join('\n')} WHERE ${queryStr}`;
+  if (querySpec.intoSpec === undefined) {
+    throw new Error('SELECT-INTO pattern is missing');
+  }
+  const intos = intoSpecToSql(querySpec.intoSpec);
+  const aggrs =
+    querySpec.aggregates !== undefined ? intoSpecToSql(querySpec.aggregates) : undefined;
+  const cols = aggrs ? `${intos}, ${aggrs}` : intos;
+  let sql = `SELECT ${querySpec.distinct ? 'DISTINCT' : ''} ${cols} FROM ${tableName} ${joinSql.join('\n')} WHERE ${queryStr}`;
+  if (querySpec.groupBy !== undefined) {
+    sql = `${sql} GROUP BY ${querySpec.groupBy.join(', ')}`;
+  }
+  if (querySpec.orderBy !== undefined) {
+    sql = `${sql} ORDER BY ${querySpec.orderBy.join(', ')} ${querySpec.orderByDesc}`;
+  }
   logger.debug(`Join Query: ${sql}`);
   const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
   return await qb.query(sql);

package/src/runtime/resolvers/sqldb/dbutil.ts

@@ -28,8 +28,13 @@ import {
   Module,
 } from '../../module.js';
 import { buildGraph } from '../../relgraph.js';
-import { makeFqName } from '../../util.js';
-import { DeletedFlagAttributeName, FkSpec, ParentAttributeName, PathAttributeName } from '../../defs.js';
+import { makeFqName, splitFqName, splitRefs } from '../../util.js';
+import {
+  DeletedFlagAttributeName,
+  FkSpec,
+  ParentAttributeName,
+  PathAttributeName,
+} from '../../defs.js';
 
 export const DefaultVectorDimension = 1536;
 
@@ -50,6 +55,22 @@ export function asTableReference(moduleName: string, ref: string): string {
   }
 }
 
+export function asColumnReference(n: string, tableName: string, entityName: string, entityFqName: string, moduleName: string, quoted: boolean = false): string {
+  const refs = splitRefs(n);
+  const rlen = refs.length
+  if (rlen == 1 || refs[0] == entityName || refs[0] == entityFqName) {
+    const r = rlen == 1 ? refs[0] : refs[1]
+    return quoted ? `"${tableName}"."${r}"` : `${tableName}.${r}`;
+  } else {
+    const p = splitFqName(refs[0]);
+    if (p.length == 2) {
+      return quoted ? `"${asTableReference(p[0], p[1])}"."${refs[1]}"` : `${asTableReference(p[0], p[1])}.${refs[1]}`;
+    } else {
+      return quoted ? `"${asTableReference(moduleName, p[0])}"."${refs[1]}"` : `${asTableReference(moduleName, p[0])}.${refs[1]}`;
+    }
+  }
+}
+
 export function modulesAsDbSchema(): TableSchema[] {
   const result: TableSchema[] = new Array<TableSchema>();
   getModuleNames().forEach((n: string) => {
@@ -96,11 +117,11 @@ export function modulesAsOrmSchema(): OrmSchema {
       }
     });
     entities.forEach((r: Record) => {
-      const fks = r.getFkAttributeSpecs()
+      const fks = r.getFkAttributeSpecs();
       if (fks.length > 0) {
-        fkSpecs.push(...fks)
+        fkSpecs.push(...fks);
       }
-    })
+    });
   });
   return { entities: ents, vectorEntities: vects, fkSpecs };
 }
@@ -291,7 +312,8 @@ export function asSqlType(type: string): ColumnType {
     t == 'url' ||
     t == 'map' ||
     t == 'any' ||
-    t == 'path'
+    t == 'path' ||
+    t == 'password'
   )
     return 'varchar';
   else if (t == 'int') return 'integer';

package/src/runtime/resolvers/sqldb/impl.ts

@@ -22,8 +22,8 @@ import {
   splitFqName,
   splitRefs,
 } from '../../util.js';
-import { JoinInfo, Resolver } from '../interface.js';
-import { asTableReference } from './dbutil.js';
+import { JoinInfo, Resolver, WhereClause } from '../interface.js';
+import { asColumnReference, asTableReference } from './dbutil.js';
 import {
   getMany,
   insertRow,
@@ -43,8 +43,9 @@ import {
   JoinOn,
   makeJoinOn,
   getManyByJoin,
+  QuerySpec,
 } from './database.js';
-import { Environment } from '../../interpreter.js';
+import { AggregateFunctionCall, Environment } from '../../interpreter.js';
 import { OpenAIEmbeddings } from '@langchain/openai';
 import { Embeddings } from '@langchain/core/embeddings';
 import { DeletedFlagAttributeName, ParentAttributeName, PathAttributeName } from '../../defs.js';
@@ -164,6 +165,26 @@ export class SqlDbResolver extends Resolver {
     return Instance.clone(inst).mergeAttributes(newAttrs);
   }
 
+  private static normalizedAggregates(
+    inst: Instance,
+    tableName: string
+  ): Map<string, string> | undefined {
+    if (inst.aggregates !== undefined) {
+      const entn = inst.name;
+      const entfqn = inst.getFqName();
+      const mn = inst.moduleName;
+      const result = new Map<string, string>();
+      inst.aggregates.forEach((f: AggregateFunctionCall, n: string) => {
+        const args = f.args.map((v: string) => {
+          return asColumnReference(v, tableName, entn, entfqn, mn, true);
+        });
+        result.set(n, `${f.name}(${args.join(', ')})`);
+      });
+      return result;
+    }
+    return undefined;
+  }
+
   static EmptyResultSet: Array<Instance> = new Array<Instance>();
 
   public override async queryInstances(
@@ -178,7 +199,32 @@ export class SqlDbResolver extends Resolver {
     const ctx = this.getDbContext(fqName);
     const qattrs: any = queryAll ? undefined : inst.queryAttributesAsObject();
     const qvals: any = queryAll ? undefined : inst.queryAttributeValuesAsObject();
-    const rslt: any = await getMany(tableName, qattrs, qvals, distinct, ctx);
+    const groupBy = inst.groupBy
+      ? inst.groupBy.map((gb: string) => {
+          return asColumnReference(gb, tableName, inst.name, fqName, inst.moduleName, true);
+        })
+      : undefined;
+    const orderBy = inst.orderBy
+      ? inst.orderBy.map((ob: string) => {
+          return asColumnReference(ob, tableName, inst.name, fqName, inst.moduleName, true);
+        })
+      : undefined;
+    const orderByDesc = inst.orderByDesc ? 'DESC' : 'ASC';
+    const aggregates = SqlDbResolver.normalizedAggregates(inst, tableName);
+    const qspec: QuerySpec = {
+      queryObj: qattrs,
+      queryVals: qvals,
+      distinct,
+      groupBy,
+      orderBy,
+      orderByDesc,
+      aggregates,
+      joinClauses: undefined,
+      intoSpec: undefined,
+      whereClauses: undefined,
+    };
+    const readOnlyAttrs = inst.record.getWriteOnlyAttributes();
+    const rslt: any = await getMany(tableName, qspec, ctx);
     if (rslt instanceof Array) {
       result = new Array<Instance>();
       rslt.forEach((r: object) => {
@@ -187,6 +233,9 @@ export class SqlDbResolver extends Resolver {
           new Map(Object.entries(r))
         );
         attrs.delete(DeletedFlagAttributeName);
+        readOnlyAttrs?.forEach((n: string) => {
+          attrs.delete(n);
+        });
         result.push(Instance.newWithAttributes(inst, attrs));
       });
     }
@@ -266,12 +315,15 @@ export class SqlDbResolver extends Resolver {
     joinInfo: JoinInfo[],
     intoSpec: Map<string, string>,
     distinct: boolean = false,
-    rawJoinSpec?: JoinSpec
+    rawJoinSpec?: JoinSpec[],
+    whereClauses?: WhereClause[]
   ): Promise<any> {
     const tableName = asTableReference(inst.moduleName, inst.name);
     const joinClauses: JoinClause[] = [];
     if (rawJoinSpec) {
-      this.processRawJoinSpec(tableName, inst, rawJoinSpec, joinClauses);
+      rawJoinSpec.forEach((rjs: JoinSpec) => {
+        this.processRawJoinSpec(tableName, inst, rjs, joinClauses);
+      });
     } else {
       this.processJoinInfo(tableName, inst, joinInfo, joinClauses);
     }
@@ -280,15 +332,35 @@ export class SqlDbResolver extends Resolver {
       const mn = p.hasModule() ? p.getModuleName() : inst.moduleName;
       intoSpec.set(k, asTableReference(mn, p.getEntryName()));
     });
-    const rslt: any = await getManyByJoin(
-      tableName,
-      inst.queryAttributesAsObject(),
-      inst.queryAttributeValuesAsObject(),
+    const fqName = inst.getFqName();
+    const groupBy = inst.groupBy
+      ? inst.groupBy.map((gb: string) => {
+          return asColumnReference(gb, tableName, inst.name, fqName, inst.moduleName, true);
+        })
+      : undefined;
+    const orderBy = inst.orderBy
+      ? inst.orderBy.map((ob: string) => {
+          return asColumnReference(ob, tableName, inst.name, fqName, inst.moduleName, true);
+        })
+      : undefined;
+    const orderByDesc = inst.orderByDesc ? 'DESC' : 'ASC';
+    const aggregates = SqlDbResolver.normalizedAggregates(inst, tableName);
+    whereClauses?.forEach((wc: WhereClause) => {
+      wc.attrName = asColumnReference(wc.attrName, tableName, inst.name, fqName, inst.moduleName);
+    });
+    const qspec: QuerySpec = {
+      queryObj: inst.queryAttributesAsObject(),
+      queryVals: inst.queryAttributeValuesAsObject(),
+      distinct,
+      groupBy,
+      orderBy,
+      orderByDesc,
+      aggregates,
       joinClauses,
+      whereClauses,
       intoSpec,
-      distinct,
-      this.getDbContext(inst.getFqName())
-    );
+    };
+    const rslt: any = await getManyByJoin(tableName, qspec, this.getDbContext(inst.getFqName()));
     return rslt;
   }
 
@@ -536,7 +608,7 @@ function maybeNormalizeAttributeNames(
     ks.forEach((k: string) => {
       const v = attrs.get(k);
       attrs.delete(k);
-      attrs.set(k.substring(n + 1), v);
+      attrs.set(k.substring(n + 1) || k, v);
     });
   }
   return attrs;

package/src/runtime/state.ts

@@ -116,6 +116,20 @@ export const ConfigSchema = z.object({
       level: z.enum(['error', 'warn', 'info', 'debug']).default('info'),
     })
     .optional(),
+  retry: z
+    .array(
+      z.object({
+        name: z.string(),
+        attempts: z.number(),
+        backoff: z.object({
+          strategy: z.enum(['linear', 'exponential', 'constant']),
+          delay: z.number(),
+          magnitude: z.enum(['seconds', 'milliSeconds', 'minutes']),
+          factor: z.number(),
+        }),
+      })
+    )
+    .optional(),
 });
 
 export type Config = z.infer<typeof ConfigSchema>;

package/src/runtime/util.ts

@@ -621,16 +621,18 @@ export function generateLoggerCallId() {
 const JS_PREFIX = '#js';
 
 export function preprocessRawConfig(rawConfig: any): any {
-  const keys = Object.keys(rawConfig);
-  keys.forEach((k: any) => {
-    const v = rawConfig[k];
-    if (isString(v) && v.startsWith(JS_PREFIX)) {
-      const s = v.substring(3).trim();
-      rawConfig[k] = eval(s);
-    } else if (typeof v == 'object') {
-      preprocessRawConfig(v);
-    }
-  });
+  if (rawConfig) {
+    const keys = Object.keys(rawConfig);
+    keys.forEach((k: any) => {
+      const v = rawConfig[k];
+      if (isString(v) && v.startsWith(JS_PREFIX)) {
+        const s = v.substring(3).trim();
+        rawConfig[k] = eval(s);
+      } else if (typeof v == 'object') {
+        preprocessRawConfig(v);
+      }
+    });
+  }
   return rawConfig;
 }
 
@@ -644,11 +646,22 @@ export function setScecretReader(f: ReadSecret) {
   globalThis.readSecret = f;
 }
 
-export function objectAsString(obj: any) {
+export function objectAsString(obj: any, keyAsString: boolean = false) {
   const entries = new Array<string>();
   Object.entries(obj).forEach(([k, v]) => {
     const vv = typeof v === 'string' ? `"${v}"` : v;
-    entries.push(`${k}: ${vv}`);
+    let fv = vv;
+    if (fv instanceof Array) {
+      fv = fv.map((v: any) => {
+        return objectAsString(v, true);
+      });
+    } else if (fv instanceof Object) {
+      fv = objectAsString(fv, true);
+    }
+    if (keyAsString) {
+      k = `"${k}"`;
+    }
+    entries.push(`${k}: ${fv}`);
   });
   return `{${entries.join(', ')}}`;
 }

package/src/syntaxes/agentlang.monarch.ts

@@ -1,7 +1,7 @@
 // Monarch syntax highlighting for the agentlang language.
 export default {
     keywords: [
-        '@actions','@after','@as','@async','@before','@catch','@distinct','@enum','@expr','@from','@full_join','@inner_join','@into','@join','@left_join','@meta','@oneof','@public','@rbac','@ref','@right_join','@then','@upsert','@with_unique','agent','agentlang/retry','allow','and','attempts','await','backoff','between','case','commitTransaction','contains','create','decision','delete','directive','else','entity','error','event','extends','false','flow','for','glossaryEntry','if','import','in','like','module','not','not_found','onSubscription','or','purge','query','read','record','relationship','resolver','return','roles','rollbackTransaction','scenario','startTransaction','subscribe','throw','true','update','upsert','where','workflow'
+        '@actions','@after','@as','@asc','@async','@before','@catch','@desc','@distinct','@enum','@expr','@from','@full_join','@groupBy','@inner_join','@into','@join','@left_join','@meta','@oneof','@orderBy','@public','@rbac','@ref','@right_join','@then','@upsert','@where','@with_unique','agent','agentlang/retry','allow','and','attempts','await','backoff','between','case','commitTransaction','contains','create','decision','delete','directive','else','entity','error','event','extends','false','flow','for','glossaryEntry','if','import','in','like','module','not','not_found','onSubscription','or','purge','query','read','record','relationship','resolver','return','roles','rollbackTransaction','scenario','startTransaction','subscribe','throw','true','update','upsert','where','workflow'
     ],
     operators: [
         '!=','*','+',',','-','-->','.','/',':',';','<','<=','<>','=','==','>','>=','?','@'