agentlang 0.10.2 → 0.10.3

package/src/runtime/embeddings/openai.ts

@@ -1,6 +1,7 @@
 import { OpenAIEmbeddings } from '@langchain/openai';
 import { EmbeddingProvider, EmbeddingProviderConfig } from './provider.js';
 import { getLocalEnv } from '../auth/defs.js';
+import { logger } from '../logger.js';
 
 export interface OpenAIEmbeddingConfig extends EmbeddingProviderConfig {
   model?: string;
@@ -14,6 +15,9 @@ export class OpenAIEmbeddingProvider extends EmbeddingProvider {
   constructor(config?: EmbeddingProviderConfig) {
     super(config || {});
     this.openaiConfig = (this.config as OpenAIEmbeddingConfig) || {};
+    logger.debug(
+      `[OPENAI-EMBEDDING] Provider created with model: ${this.openaiConfig.model || 'default'}`
+    );
   }
 
   protected createEmbeddings(): OpenAIEmbeddings {
@@ -21,26 +25,40 @@ export class OpenAIEmbeddingProvider extends EmbeddingProvider {
       apiKey: this.resolveApiKey(),
     };
 
-    if (this.openaiConfig.model) {
-      config.model = this.openaiConfig.model;
+    // Use this.config directly since this.openaiConfig is not initialized yet
+    // during parent constructor (super() calls createEmbeddings before child init)
+    const openaiConfig = (this.config as OpenAIEmbeddingConfig) || {};
+
+    if (openaiConfig?.model) {
+      config.model = openaiConfig.model;
     }
 
-    if (this.openaiConfig.dimensions) {
-      config.dimensions = this.openaiConfig.dimensions;
+    if (openaiConfig?.dimensions) {
+      config.dimensions = openaiConfig.dimensions;
     }
 
-    if (this.openaiConfig.maxRetries !== undefined) {
-      config.maxRetries = this.openaiConfig.maxRetries;
+    if (openaiConfig?.maxRetries !== undefined) {
+      config.maxRetries = openaiConfig.maxRetries;
     }
 
     return new OpenAIEmbeddings(config);
   }
 
   protected resolveApiKey(): string {
-    if (this.openaiConfig.apiKey) {
-      return this.openaiConfig.apiKey;
+    // Use this.config directly since this.openaiConfig may not be initialized yet
+    // during constructor (createEmbeddings is called during super())
+    const config = this.openaiConfig || (this.config as OpenAIEmbeddingConfig) || {};
+    if (config.apiKey) {
+      return config.apiKey;
+    }
+    const envKey = process.env.AGENTLANG_OPENAI_KEY || getLocalEnv('AGENTLANG_OPENAI_KEY');
+    if (envKey) {
+      return envKey;
     }
-    return process.env.AGENTLANG_OPENAI_KEY || getLocalEnv('AGENTLANG_OPENAI_KEY') || '';
+    logger.warn(
+      `[OPENAI-EMBEDDING] No API key found! Set AGENTLANG_OPENAI_KEY environment variable.`
+    );
+    return '';
   }
 
   getProviderName(): string {

package/src/runtime/embeddings/provider.ts

@@ -1,4 +1,5 @@
 import { Embeddings } from '@langchain/core/embeddings';
+import { logger } from '../logger.js';
 
 export interface EmbeddingProviderConfig {
   chunkSize?: number;
@@ -23,7 +24,27 @@ export abstract class EmbeddingProvider {
   abstract getProviderName(): string;
 
   async embedText(text: string): Promise<number[]> {
-    return await this.embeddings.embedQuery(text);
+    logger.debug(`[EMBEDDING-PROVIDER] embedText called (${text.length} chars)`);
+    const startTime = Date.now();
+    const result = await this.embeddings.embedQuery(text);
+    const duration = Date.now() - startTime;
+    logger.debug(
+      `[EMBEDDING-PROVIDER] embedText completed in ${duration}ms (${result.length} dimensions)`
+    );
+    return result;
+  }
+
+  async embedTexts(texts: string[]): Promise<number[][]> {
+    if (texts.length === 0) return [];
+    if (texts.length === 1) return [await this.embedText(texts[0])];
+    logger.debug(`[EMBEDDING-PROVIDER] embedTexts called (${texts.length} texts)`);
+    const startTime = Date.now();
+    const results = await this.embeddings.embedDocuments(texts);
+    const duration = Date.now() - startTime;
+    logger.debug(
+      `[EMBEDDING-PROVIDER] embedTexts completed in ${duration}ms (${texts.length} texts, ${results[0]?.length || 0} dimensions)`
+    );
+    return results;
   }
 
   getConfig(): EmbeddingProviderConfig {

package/src/runtime/integration-client.ts

@@ -0,0 +1,158 @@
+let host: string | undefined;
+let headers: Record<string, string> | undefined;
+
+export function configureIntegrationClient(h: string, hdrs?: Record<string, string>): void {
+  host = h;
+  headers = hdrs;
+}
+
+export function isIntegrationClientConfigured(): boolean {
+  return host !== undefined;
+}
+
+export async function getIntegrationAuthHeaders(
+  integrationName: string
+): Promise<Record<string, string>> {
+  if (!host) {
+    throw new Error('Integration client not configured — call configureIntegrationClient() first');
+  }
+
+  const url = `${host}/integmanager.auth/authHeaders/${encodeURIComponent(integrationName)}`;
+  const response = await fetch(url, {
+    method: 'GET',
+    headers: { 'Content-Type': 'application/json', ...headers },
+  });
+
+  if (!response.ok) {
+    throw new Error(
+      `Failed to get auth headers for integration "${integrationName}": ${response.status} ${response.statusText}`
+    );
+  }
+
+  const data = await response.json();
+  // The response is either the entity directly or wrapped in an array
+  const result = Array.isArray(data) ? data[0] : data;
+  return result?.headers ?? {};
+}
+
+export async function refreshIntegrationAuth(integrationName: string): Promise<void> {
+  if (!host) {
+    throw new Error('Integration client not configured — call configureIntegrationClient() first');
+  }
+
+  const url = `${host}/integmanager.auth/authRefresh`;
+  const response = await fetch(url, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json', ...headers },
+    body: JSON.stringify({ integrationName }),
+  });
+
+  if (!response.ok) {
+    throw new Error(
+      `Failed to refresh auth for integration "${integrationName}": ${response.status} ${response.statusText}`
+    );
+  }
+}
+
+export async function integrationAuthFetch(
+  integrationName: string,
+  url: string | URL,
+  options: RequestInit = {}
+): Promise<Response> {
+  const authHeaders = await getIntegrationAuthHeaders(integrationName);
+  const mergedHeaders = { ...authHeaders, ...((options.headers as Record<string, string>) || {}) };
+  return fetch(url, { ...options, headers: mergedHeaders });
+}
+
+// --- OAuth consent flow helpers ---
+
+export async function getOAuthAuthorizeUrl(
+  integrationName: string,
+  redirectUri: string
+): Promise<{ authorizationUrl: string; state: string }> {
+  if (!host) {
+    throw new Error('Integration client not configured — call configureIntegrationClient() first');
+  }
+
+  const params = new URLSearchParams({
+    action: 'authorize',
+    integrationName,
+    redirectUri,
+  });
+  const url = `${host}/integmanager.auth/oauthFlow?${params.toString()}`;
+  const response = await fetch(url, {
+    method: 'GET',
+    headers: { 'Content-Type': 'application/json', ...headers },
+  });
+
+  if (!response.ok) {
+    throw new Error(
+      `Failed to get OAuth authorize URL for "${integrationName}": ${response.status} ${response.statusText}`
+    );
+  }
+
+  const data = await response.json();
+  const result = Array.isArray(data) ? data[0] : data;
+  return { authorizationUrl: result.authorizationUrl, state: result.state };
+}
+
+export async function exchangeOAuthCode(
+  integrationName: string,
+  code: string,
+  state: string
+): Promise<{ accessToken: string; refreshToken: string; expiresIn: number; tokenType: string }> {
+  if (!host) {
+    throw new Error('Integration client not configured — call configureIntegrationClient() first');
+  }
+
+  const url = `${host}/integmanager.auth/oauthFlow`;
+  const response = await fetch(url, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json', ...headers },
+    body: JSON.stringify({ action: 'exchange', integrationName, code, state }),
+  });
+
+  if (!response.ok) {
+    throw new Error(
+      `Failed to exchange OAuth code for "${integrationName}": ${response.status} ${response.statusText}`
+    );
+  }
+
+  const data = await response.json();
+  const result = Array.isArray(data) ? data[0] : data;
+  return {
+    accessToken: result.accessToken,
+    refreshToken: result.refreshToken,
+    expiresIn: result.expiresIn,
+    tokenType: result.tokenType,
+  };
+}
+
+export async function getIntegrationAccessToken(
+  integrationName: string
+): Promise<{ accessToken: string; expiresIn: number; tokenType: string }> {
+  if (!host) {
+    throw new Error('Integration client not configured — call configureIntegrationClient() first');
+  }
+
+  const params = new URLSearchParams({ integrationName });
+  const url = `${host}/integmanager.auth/oauthToken?${params.toString()}`;
+  const response = await fetch(url, {
+    method: 'GET',
+    headers: { 'Content-Type': 'application/json', ...headers },
+  });
+
+  if (!response.ok) {
+    throw new Error(
+      `Failed to get access token for "${integrationName}": ${response.status} ${response.statusText}`
+    );
+  }
+
+  const data = await response.json();
+  const result = Array.isArray(data) ? data[0] : data;
+  return {
+    accessToken: result.accessToken,
+    expiresIn: result.expiresIn,
+    tokenType: result.tokenType,
+  };
+}

package/src/runtime/integrations.ts

@@ -1,7 +1,6 @@
-import { Instance } from './module.js';
 import { isString } from './util.js';
 
-const Integrations = new Map<string, Instance>();
+const Integrations = new Map<string, any>();
 
 const IntegManagerModel = 'integmanager.core';
 
@@ -16,7 +15,8 @@ export async function prepareIntegrations(
   const keys = [...integConfig.keys()];
   for (let i = 0; i < keys.length; ++i) {
     const configName = keys[i];
-    const configPath = integConfig.get(configName);
+    const entry = integConfig.get(configName);
+    const configPath = typeof entry === 'string' ? entry : entry?.config;
     if (configPath) {
       const apiUrl = mkApiUrl(integManagerHost, configPath);
       try {
@@ -35,9 +35,6 @@ export async function prepareIntegrations(
         const data = await response.json();
         if (data.length > 0) {
           const inst: any = data[0].config;
-          if (inst.type == 'custom' && isString(inst.parameter)) {
-            inst.parameter = new Map(Object.entries(JSON.parse(inst.parameter)));
-          }
           Integrations.set(configName, inst);
         } else {
           console.error(`Integration not found for ${configPath}`);
@@ -84,14 +81,26 @@ function mkApiUrl(integManagerHost: string, configPath: string): string {
   const parts = configPath.split('/');
   const integId = parts[0];
   const configId = parts[1];
-  return `${integManagerHost}/${IntegManagerModel}/integration/${integId}/integrationConfig/config/${configId}`;
+  return `${integManagerHost}/${IntegManagerModel}/integration/${integId}/integrationConfig/config/${configId}?tree=true`;
 }
 
 export function getIntegrationConfig(name: string, configName: string): any {
   const config: any = Integrations.get(name);
-  if (config) {
-    return config.parameter.get(configName);
-  } else {
-    return undefined;
+  if (!config) return undefined;
+  if (config.parameter == null) return undefined;
+
+  if (config.parameter instanceof Map) {
+    return Object.fromEntries(config.parameter).get(configName);
+  }
+  if (isString(config.parameter)) {
+    try {
+      return JSON.parse(config.parameter)[configName];
+    } catch {
+      return undefined;
+    }
+  }
+  if (typeof config.parameter === 'object') {
+    return config.parameter[configName];
   }
+  return undefined;
 }

package/src/runtime/interpreter.ts

@@ -47,6 +47,7 @@ import {
   isEntityInstance,
   isEventInstance,
   isInstanceOfType,
+  getAllBetweenRelationshipsForEntity,
   isOneToOneBetweenRelationship,
   isTimer,
   makeInstance,
@@ -65,6 +66,7 @@ import {
   DefaultModuleName,
   escapeFqName,
   escapeQueryName,
+  firstAliasSpec,
   fqNameFromPath,
   isCoreModule,
   isFqName,
@@ -77,6 +79,7 @@ import {
   preprocessRawConfig,
   QuerySuffix,
   restoreSpecialChars,
+  splitFqName,
   splitRefs,
 } from './util.js';
 import { getResolver, getResolverNameForPath } from './resolvers/registry.js';
@@ -88,9 +91,12 @@ import {
 } from '../language/parser.js';
 import { ActiveSessionInfo, AdminSession, AdminUserId } from './auth/defs.js';
 import {
+  AgentCancelledException,
   AgentEntityName,
   AgentFqName,
   AgentInstance,
+  checkCancelled,
+  clearCancellation,
   findAgentByName,
   normalizeGeneratedCode,
   saveFlowStepResult,
@@ -1133,10 +1139,11 @@ export async function evaluateStatement(stmt: Statement, env: Environment): Prom
       handlersPushed = env.pushHandlers(handlers);
     }
     await evaluatePattern(stmt.pattern, env);
+    let skipOuterAlias = false;
     if (hasHints) {
-      await maybeHandleEmpty(hints, env);
+      skipOuterAlias = await maybeHandleEmpty(hints, env);
     }
-    if (hasHints) {
+    if (hasHints && !skipOuterAlias) {
       maybeBindStatementResultToAlias(hints, env);
     }
     await maybeHandleNotFound(handlers, env);
@@ -1165,7 +1172,7 @@ async function maybeHandleNotFound(handlers: CatchHandlers | undefined, env: Env
   }
 }
 
-async function maybeHandleEmpty(hints: RuntimeHint[], env: Environment) {
+async function maybeHandleEmpty(hints: RuntimeHint[], env: Environment): Promise<boolean> {
   const lastResult: Result = env.getLastResult();
   if (
     lastResult === null ||
@@ -1177,10 +1184,17 @@ async function maybeHandleEmpty(hints: RuntimeHint[], env: Environment) {
         const newEnv = new Environment('empty-env', env).unsetEventExecutor();
         await evaluateStatement(rh.emptySpec.stmt, newEnv);
         env.setLastResult(newEnv.getLastResult());
+        // If inner statement has its own @as, propagate binding to parent env
+        const innerAlias = firstAliasSpec(rh.emptySpec.stmt);
+        if (innerAlias) {
+          maybeBindStatementResultToAlias(rh.emptySpec.stmt.hints, env);
+          return true; // signal: skip outer @as
+        }
         break;
       }
     }
   }
+  return false;
 }
 
 async function maybeHandleError(
@@ -1571,6 +1585,12 @@ function maybeSetQueryClauses(inst: Instance, qopts: ExtractedQueryOptions) {
   if (qopts.orderByClause) {
     inst.setOrderBy(qopts.orderByClause.colNames, qopts.orderByClause.order === '@desc');
   }
+  if (qopts.limitClause) {
+    inst.setLimit(qopts.limitClause.value);
+  }
+  if (qopts.offsetClause) {
+    inst.setOffset(qopts.offsetClause.value);
+  }
 }
 
 async function evaluateCrudMap(crud: CrudMap, env: Environment): Promise<void> {
@@ -1624,7 +1644,7 @@ async function evaluateCrudMap(crud: CrudMap, env: Environment): Promise<void> {
       }
       const res: Resolver = await getResolverForPath(entryName, moduleName, env);
       let r: Instance | undefined;
-      await computeExprAttributes(inst, undefined, undefined, env);
+      await computeExprAttributes(inst, crud.body?.attributes, inst.attributes, env);
       await setMetaAttributes(inst.attributes, env);
       if (env.isInUpsertMode()) {
         await runPreUpdateEvents(inst, env);
@@ -1913,24 +1933,98 @@ async function computeExprAttributes(
     updatedAttrs?.forEach((v: any, k: string) => {
       if (v !== undefined) newEnv.bind(k, v);
     });
+    // Bind related instances from between-relationships so that @expr references
+    // like DeptEmployee.Department.BudgetMultiplier can be resolved via followReference.
+    const entityFqName = inst.getFqName();
+    const fqParts = splitFqName(entityFqName);
+    const rels = getAllBetweenRelationshipsForEntity(fqParts[0], fqParts[1]);
+    for (const rel of rels) {
+      const isFirst = rel.isFirstNodeName(entityFqName);
+      // Only bind if this entity is on a valid side for single-entity resolution
+      if (rel.isManyToMany()) continue;
+      if (rel.isOneToMany() && isFirst) continue;
+      // Determine the target entity on the other side
+      const targetNode = isFirst ? rel.node2 : rel.node1;
+      const targetAlias = targetNode.alias;
+      let connectedInst: Instance | undefined;
+      // Fast path: check if the environment carries between-rel info for this relationship
+      const betRelInfo = env.getBetweenRelInfo();
+      if (betRelInfo && betRelInfo.relationship.name === rel.name) {
+        connectedInst = betRelInfo.connectedInstance;
+      } else if (inst.lookup(PathAttributeName)) {
+        // Slow path: query the connected instance through the resolver.
+        // Only possible if the current instance has been persisted (has a __path__).
+        try {
+          const targetFqName = targetNode.path.asFqName();
+          const targetParts = splitFqName(targetFqName);
+          const res: Resolver = await getResolverForPath(targetParts[1], targetParts[0], env);
+          const queryInst = Instance.EmptyInstance(targetParts[1], targetParts[0]);
+          const connected: Instance[] = await res.queryConnectedInstances(rel, inst, queryInst);
+          if (connected && connected.length > 0) {
+            connectedInst = connected[0];
+          }
+        } catch (reason: any) {
+          logger.debug(
+            `Relationship query failed (e.g. not yet linked) - skip binding - ${reason}`
+          );
+        }
+      }
+      if (connectedInst) {
+        const relMap = new Map<string, Instance>();
+        relMap.set(targetAlias, connectedInst);
+        newEnv.bind(rel.name, relMap);
+      }
+    }
+    // Build a map of user-provided values for @expr attributes so that
+    // overrides are applied inline during expression evaluation, allowing
+    // dependent expressions to see the user's value immediately.
+    let userExprOverrides: Map<string, Expr> | undefined;
+    if (exprAttrs && origAttrs) {
+      for (let i = 0; i < origAttrs.length; ++i) {
+        const a: SetAttribute = origAttrs[i];
+        const n = a.name;
+        if (exprAttrs.has(n) && !n.endsWith(QuerySuffix) && a.value !== undefined) {
+          if (userExprOverrides === undefined) {
+            userExprOverrides = new Map();
+          }
+          userExprOverrides.set(n, a.value);
+        }
+      }
+    }
     if (exprAttrs) {
       const ks = [...exprAttrs.keys()];
       for (let i = 0; i < ks.length; ++i) {
         const n = ks[i];
-        const expr: Expr | undefined = exprAttrs.get(n);
-        if (expr) {
-          await evaluateExpression(expr, newEnv);
-          const v: Result = newEnv.getLastResult();
-          newEnv.bind(n, v);
-          inst.attributes.set(n, v);
-          updatedAttrs?.set(n, v);
+        const userValue = userExprOverrides?.get(n);
+        if (userValue !== undefined) {
+          // User explicitly provided a value for this @expr attribute - use it
+          await evaluateExpression(userValue, newEnv);
+        } else {
+          // No user override - evaluate the @expr expression
+          const expr: Expr | undefined = exprAttrs.get(n);
+          if (expr) {
+            await evaluateExpression(expr, newEnv);
+          } else {
+            continue;
+          }
         }
+        const v: Result = newEnv.getLastResult();
+        newEnv.bind(n, v);
+        inst.attributes.set(n, v);
+        updatedAttrs?.set(n, v);
       }
     }
-    if (origAttrs && updatedAttrs) {
+    // Re-evaluate non-@expr attribute expressions from origAttrs in the context
+    // of the queried instance. This handles workflow update expressions like
+    // `balance balance + (balance * interestRate) + makeDeposit.amount` where
+    // the local attribute references must resolve from the existing instance.
+    // Only runs on the update path (where updatedAttrs is a separate map from
+    // inst.attributes) to avoid double-evaluating expressions on create.
+    if (origAttrs && updatedAttrs && updatedAttrs !== inst.attributes) {
       for (let i = 0; i < origAttrs.length; ++i) {
         const a: SetAttribute = origAttrs[i];
         const n = a.name;
+        if (exprAttrs?.has(n)) continue;
         if (!n.endsWith(QuerySuffix) && updatedAttrs.has(n) && a.value !== undefined) {
           await evaluateExpression(a.value, newEnv);
           const v: Result = newEnv.getLastResult();
@@ -2085,6 +2179,8 @@ async function agentInvoke(agent: AgentInstance, msg: string, env: Environment):
   console.debug(invokeDebugMsg);
   //
 
+  const agentChatId = env.getAgentChatId() || env.getActiveChatId() || '';
+  await clearCancellation(agentChatId);
   const monitoringEnabled = isMonitoringEnabled();
 
   await agent.invoke(msg, env);
@@ -2101,6 +2197,7 @@ async function agentInvoke(agent: AgentInstance, msg: string, env: Environment):
       }
       let retries = 0;
       while (true) {
+        await checkCancelled(agentChatId);
         try {
           let rs: string = result ? normalizeGeneratedCode(result) : '';
           if (agent.tools) {
@@ -2162,6 +2259,7 @@ async function agentInvoke(agent: AgentInstance, msg: string, env: Environment):
     } else {
       let retries = 0;
       while (true) {
+        await checkCancelled(agentChatId);
         try {
           result = normalizeGeneratedCode(result);
           const obj = agent.maybeValidateJsonResponse(result);
@@ -2287,9 +2385,11 @@ async function iterateOnFlow(
   rootAgent.disableSession();
   const chatId = env.getActiveEventInstance()?.lookup('chatId');
   const iterId = chatId || crypto.randomUUID();
+  await clearCancellation(iterId);
   let step = '';
   let fullFlowRetries = 0;
   while (true) {
+    await checkCancelled(iterId);
     try {
       const initContext = msg;
       const s = `Now consider the following flowchart and return the next step:\n${flow}\n
@@ -2312,6 +2412,7 @@ async function iterateOnFlow(
         env.flagMonitorEntryAsFlow().incrementMonitor();
       }
       while (step != 'DONE' && !executedSteps.has(step)) {
+        await checkCancelled(iterId);
         if (stepc > MaxFlowSteps) {
           throw new Error(`Flow execution exceeded maximum steps limit`);
         }
@@ -2366,6 +2467,9 @@ async function iterateOnFlow(
         needAgentProcessing = preprocResult.needAgentProcessing;
       }
     } catch (reason: any) {
+      if (reason instanceof AgentCancelledException) {
+        throw reason;
+      }
       if (fullFlowRetries < MaxFlowRetries) {
         msg = `The previous attempt failed at step ${step} with the error ${reason}. Restart the flow the appropriate step
 (maybe even from the first step) and try to fix the issue.`;
@@ -2598,6 +2702,32 @@ export async function evaluateExpression(expr: Expr, env: Environment): Promise<
   env.setLastResult(result);
 }
 
+export function extractRefsFromExpr(expr: Expr): string[] {
+  const refs: string[] = [];
+  function walk(e: Expr) {
+    if (isBinExpr(e)) {
+      walk(e.e1);
+      walk(e.e2);
+    } else if (isLiteral(e)) {
+      if (e.ref) refs.push(e.ref);
+      if (e.fnCall) {
+        for (const arg of e.fnCall.args) walk(arg);
+      }
+      if (e.asyncFnCall) {
+        for (const arg of e.asyncFnCall.fnCall.args) walk(arg);
+      }
+    } else if (isGroup(e)) {
+      walk(e.ge);
+    } else if (isNegExpr(e)) {
+      walk(e.ne);
+    } else if (isNotExpr(e)) {
+      walk(e.ne);
+    }
+  }
+  walk(expr);
+  return refs;
+}
+
 async function getRef(r: string, src: any, env: Environment): Promise<Result> {
   if (Instance.IsInstance(src)) return src.lookup(r);
   else if (src instanceof Map) return src.get(r);