@turingpulse/sdk 1.0.1

package/src/kpi.ts

@@ -0,0 +1,112 @@
+import { ExecutionContext } from './context';
+
+export type KPIComparator = 'gt' | 'gte' | 'lt' | 'lte' | 'eq';
+
+export type KPIValueFn = (context: ExecutionContext) => number | undefined | null;
+
+export interface KPIConfig {
+  kpiId: string;
+  value?: number | KPIValueFn;
+  useDuration?: boolean;
+  fromResultPath?: string;
+  labels?: Record<string, string>;
+  alertThreshold?: number;
+  comparator?: KPIComparator;
+  description?: string;
+}
+
+export interface KPIResult {
+  config: KPIConfig;
+  value: number;
+  alert: boolean;
+}
+
+export function evaluateKpis(configs: ReadonlyArray<KPIConfig>, context: ExecutionContext): KPIResult[] {
+  const results: KPIResult[] = [];
+  for (const cfg of configs) {
+    const value = evaluateKpi(cfg, context);
+    if (value === undefined || Number.isNaN(value)) {
+      continue;
+    }
+    results.push({
+      config: cfg,
+      value,
+      alert: isAlert(cfg, value),
+    });
+  }
+  return results;
+}
+
+function evaluateKpi(config: KPIConfig, context: ExecutionContext): number | undefined {
+  if (config.useDuration && typeof context.durationMs === 'number') {
+    return context.durationMs;
+  }
+  if (typeof config.value === 'number') {
+    return config.value;
+  }
+  if (typeof config.value === 'function') {
+    const output = config.value(context);
+    return output === undefined || output === null ? undefined : Number(output);
+  }
+  if (config.fromResultPath) {
+    const extracted = extractFromResult(context.result, config.fromResultPath);
+    return extracted === undefined || extracted === null ? undefined : Number(extracted);
+  }
+  return undefined;
+}
+
+function isAlert(config: KPIConfig, value: number): boolean {
+  if (config.alertThreshold === undefined) {
+    return false;
+  }
+  const threshold = config.alertThreshold;
+  const comparator = config.comparator ?? 'gt';
+  switch (comparator) {
+    case 'gt':
+      return value > threshold;
+    case 'gte':
+      return value >= threshold;
+    case 'lt':
+      return value < threshold;
+    case 'lte':
+      return value <= threshold;
+    case 'eq':
+      return value === threshold;
+    default:
+      return false;
+  }
+}
+
+import { FORBIDDEN_KEYS } from './utils';
+
+function extractFromResult(result: unknown, path: string): unknown {
+  if (!result) {
+    return undefined;
+  }
+  return path.split('.').reduce<unknown>((current, key) => {
+    if (current == null || FORBIDDEN_KEYS.has(key)) {
+      return undefined;
+    }
+    if (typeof current === 'object' && key in (current as Record<string, unknown>)) {
+      return (current as Record<string, unknown>)[key];
+    }
+    return undefined;
+  }, result);
+}
+
+export function kpiMetadata(result: KPIResult): Record<string, string> {
+  const metadata: Record<string, string> = {
+    [`metric.${result.config.kpiId}`]: String(result.value),
+  };
+  if (result.alert) {
+    metadata[`metric.alert.${result.config.kpiId}`] = 'true';
+  }
+  if (result.config.labels) {
+    for (const [key, value] of Object.entries(result.config.labels)) {
+      metadata[`metric.${result.config.kpiId}.${key}`] = String(value);
+    }
+  }
+  return metadata;
+}
+
+

package/src/llmDetector.ts

@@ -0,0 +1,330 @@
+/**
+ * Automatic detection of LLM calls and extraction of metadata.
+ *
+ * Uses a data-driven provider registry instead of per-provider methods,
+ * making it trivial to add new providers without modifying detection logic.
+ */
+
+export interface LLMCallInfo {
+  provider: string;
+  model?: string;
+  prompt?: string;
+  config: Record<string, unknown>;
+  nodeType: string;
+  streaming: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Provider registry — add new providers by appending to PROVIDERS
+// ---------------------------------------------------------------------------
+
+interface ProviderSpec {
+  name: string;
+  qualnamePatterns: string[];
+  configKeys: string[];
+  methodPatterns?: string[];
+  methodContext?: string;
+  systemParam?: string;
+  /** messages | firstArg | inputKwarg | stateMessages | inputsKwarg | messageKwarg | queryKwarg | bedrock */
+  promptStrategy?: string;
+  modelKey?: string;
+  altModelKey?: string;
+}
+
+const PROVIDERS: ProviderSpec[] = [
+  {
+    name: 'openai',
+    qualnamePatterns: ['chat.completions.create', 'completions.create',
+      'chatcompletion.create', 'completion.create', 'openai.chat'],
+    configKeys: ['model', 'temperature', 'max_tokens', 'top_p',
+      'frequency_penalty', 'presence_penalty', 'stop'],
+  },
+  {
+    name: 'anthropic',
+    qualnamePatterns: ['messages.create', 'anthropic.messages', 'claude'],
+    configKeys: ['model', 'temperature', 'max_tokens', 'top_p', 'stop_sequences'],
+    systemParam: 'system',
+  },
+  {
+    name: 'google',
+    qualnamePatterns: ['generatecontent', 'generativemodel', 'google.generativeai', 'gemini'],
+    configKeys: ['model', 'temperature', 'maxOutputTokens', 'topP', 'topK'],
+    promptStrategy: 'firstArg',
+    altModelKey: 'modelName',
+  },
+  {
+    name: 'vertexai',
+    qualnamePatterns: ['vertexai', 'palm'],
+    configKeys: ['model', 'temperature', 'maxOutputTokens', 'topP', 'topK', 'candidateCount'],
+    promptStrategy: 'firstArg',
+    altModelKey: 'modelName',
+  },
+  {
+    name: 'langchain',
+    qualnamePatterns: ['langchain', 'chatmodel', 'llmchain', 'runnablesequence'],
+    configKeys: ['modelName', 'temperature', 'maxTokens', 'modelKwargs'],
+    methodPatterns: ['invoke', 'ainvoke', 'batch', 'stream'],
+    methodContext: 'chain',
+    promptStrategy: 'inputKwarg',
+    modelKey: 'modelName',
+    altModelKey: 'model',
+  },
+  {
+    name: 'langgraph',
+    qualnamePatterns: ['langgraph', 'stategraph', 'compiledgraph', 'messagesgraph'],
+    configKeys: ['modelName', 'temperature', 'maxTokens', 'recursionLimit', 'configurable'],
+    methodPatterns: ['invoke', 'ainvoke', 'stream', 'astream', 'batch'],
+    methodContext: 'graph',
+    promptStrategy: 'stateMessages',
+    modelKey: 'modelName',
+    altModelKey: 'model',
+  },
+  {
+    name: 'crewai',
+    qualnamePatterns: ['crewai', 'crew.kickoff', 'agent.execute', 'task.execute'],
+    configKeys: ['model', 'temperature', 'maxTokens', 'verbose', 'memory'],
+    methodPatterns: ['kickoff', 'kickoffAsync', 'executeTask'],
+    promptStrategy: 'inputsKwarg',
+  },
+  {
+    name: 'autogen',
+    qualnamePatterns: ['autogen', 'conversableagent', 'assistantagent', 'useragent', 'groupchat'],
+    configKeys: ['model', 'temperature', 'maxTokens', 'timeout', 'cacheSeed'],
+    methodPatterns: ['initiateChat', 'generateReply', 'run'],
+    promptStrategy: 'messageKwarg',
+  },
+  {
+    name: 'llamaindex',
+    qualnamePatterns: ['llamaindex', 'llama_index', 'queryengine', 'chatengine', 'vectorstoreindex'],
+    configKeys: ['model', 'temperature', 'maxTokens', 'contextWindow'],
+    methodPatterns: ['query', 'chat', 'retrieve', 'asQueryEngine'],
+    methodContext: 'index|engine',
+    promptStrategy: 'queryKwarg',
+    altModelKey: 'llm',
+  },
+  {
+    name: 'bedrock',
+    qualnamePatterns: ['bedrock', 'invokemodel', 'invokemodelwithresponsestream', 'converse', 'conversestream'],
+    configKeys: ['modelId', 'temperature', 'maxTokens', 'topP', 'stopSequences'],
+    promptStrategy: 'bedrock',
+    modelKey: 'modelId',
+  },
+  {
+    name: 'cohere',
+    qualnamePatterns: ['cohere.chat', 'cohere.generate', 'cohereclient'],
+    configKeys: ['model', 'temperature', 'max_tokens', 'p', 'k'],
+    systemParam: 'preamble',
+    promptStrategy: 'messageKwarg',
+  },
+  {
+    name: 'mistral',
+    qualnamePatterns: ['mistral.chat', 'mistral.complete', 'mistralclient'],
+    configKeys: ['model', 'temperature', 'max_tokens', 'top_p'],
+  },
+];
+
+// ---------------------------------------------------------------------------
+// Detection engine
+// ---------------------------------------------------------------------------
+
+function matchesProvider(spec: ProviderSpec, qualname: string, funcName: string): boolean {
+  if (spec.qualnamePatterns.some((p) => qualname.includes(p))) {
+    return true;
+  }
+  if (spec.methodPatterns?.includes(funcName)) {
+    if (spec.methodContext) {
+      return spec.methodContext.split('|').some((ctx) => qualname.includes(ctx));
+    }
+    return true;
+  }
+  return false;
+}
+
+import {
+  extractConfigSubset as extractConfig,
+  extractPromptFromMessages as extractSystemPrompt,
+} from './fingerprint';
+
+function extractPrompt(
+  spec: ProviderSpec,
+  args: unknown[],
+  kwargs: Record<string, unknown>,
+): string | undefined {
+  const strategy = spec.promptStrategy ?? 'messages';
+
+  if (strategy === 'messages') {
+    if (spec.systemParam) {
+      const sys = kwargs[spec.systemParam];
+      if (typeof sys === 'string') return sys;
+    }
+    return extractSystemPrompt(
+      (kwargs.messages as Array<{ role?: string; content?: string | unknown[] }>) ?? [],
+    );
+  }
+
+  if (strategy === 'firstArg') {
+    if (args.length > 0) {
+      if (typeof args[0] === 'string') return args[0];
+      if (Array.isArray(args[0])) return String(args[0]);
+    }
+    return undefined;
+  }
+
+  if (strategy === 'inputKwarg') {
+    if ('input' in kwargs) {
+      const val = kwargs.input;
+      if (typeof val === 'object' && val !== null && 'question' in val) {
+        return (val as { question: string }).question;
+      }
+      if (typeof val === 'string') return val;
+    }
+    if (args.length > 0 && typeof args[0] === 'string') return args[0];
+    return undefined;
+  }
+
+  if (strategy === 'stateMessages') {
+    if ('messages' in kwargs) {
+      return extractSystemPrompt(kwargs.messages as Array<{ role?: string; content?: string | unknown[] }>);
+    }
+    if (args.length > 0 && typeof args[0] === 'object' && args[0] !== null) {
+      const state = args[0] as Record<string, unknown>;
+      if ('messages' in state) {
+        return extractSystemPrompt(state.messages as Array<{ role?: string; content?: string | unknown[] }>);
+      }
+      if ('input' in state) return state.input as string;
+    }
+    return undefined;
+  }
+
+  if (strategy === 'inputsKwarg') {
+    if ('inputs' in kwargs) {
+      const inputs = kwargs.inputs;
+      if (typeof inputs === 'object' && inputs !== null) {
+        const obj = inputs as Record<string, unknown>;
+        return (obj.topic as string) || (obj.question as string) || String(inputs);
+      }
+    }
+    if (args.length > 0 && typeof args[0] === 'object') return String(args[0]);
+    return undefined;
+  }
+
+  if (strategy === 'messageKwarg') {
+    if ('message' in kwargs && typeof kwargs.message === 'string') return kwargs.message;
+    if (args.length > 0 && typeof args[0] === 'string') return args[0];
+    return undefined;
+  }
+
+  if (strategy === 'queryKwarg') {
+    if (args.length > 0 && typeof args[0] === 'string') return args[0];
+    return (kwargs.query as string) ?? (kwargs.message as string) ?? undefined;
+  }
+
+  if (strategy === 'bedrock') {
+    return extractBedrockPrompt(kwargs);
+  }
+
+  return undefined;
+}
+
+function extractBedrockPrompt(kwargs: Record<string, unknown>): string | undefined {
+  const modelId = kwargs.modelId as string | undefined;
+  const body = kwargs.body;
+  if (body) {
+    try {
+      const bodyData: Record<string, unknown> =
+        typeof body === 'string' ? JSON.parse(body) : (body as Record<string, unknown>);
+      if (modelId?.toLowerCase().includes('anthropic')) {
+        return extractSystemPrompt(
+          bodyData.messages as Array<{ role?: string; content?: string | unknown[] }>,
+        );
+      }
+      return (bodyData.prompt as string) || (bodyData.inputText as string);
+    } catch {
+      // ignore parse errors
+    }
+  }
+  if ('messages' in kwargs) {
+    const messages = kwargs.messages as Array<{ role?: string; content?: Array<{ text?: string }> }>;
+    if (messages?.length > 0) {
+      for (const msg of messages) {
+        if (msg.role === 'user' && msg.content?.length) {
+          return msg.content[0].text;
+        }
+      }
+    }
+  }
+  return undefined;
+}
+
+function extractModel(spec: ProviderSpec, kwargs: Record<string, unknown>): string | undefined {
+  const key = spec.modelKey ?? 'model';
+  return (kwargs[key] as string) ?? (spec.altModelKey ? (kwargs[spec.altModelKey] as string) : undefined);
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+export class LLMDetector {
+  static detectLLMCall(
+    funcName: string,
+    funcQualname: string,
+    args: unknown[],
+    kwargs: Record<string, unknown>,
+  ): LLMCallInfo | undefined {
+    const qualnameLower = funcQualname.toLowerCase();
+    const funcNameLower = funcName.toLowerCase();
+
+    for (const spec of PROVIDERS) {
+      if (matchesProvider(spec, qualnameLower, funcNameLower)) {
+        const isStreaming = Boolean(
+          kwargs.stream || kwargs.stream_mode || kwargs.response_mode === 'streaming',
+        );
+        return {
+          provider: spec.name,
+          model: extractModel(spec, kwargs),
+          prompt: extractPrompt(spec, args, kwargs),
+          config: extractConfig(kwargs, spec.configKeys),
+          nodeType: 'llm',
+          streaming: isStreaming,
+        };
+      }
+    }
+    return undefined;
+  }
+}
+
+const TOOL_PATTERNS = ['search', 'query', 'retrieve', 'fetch', 'callapi',
+  'execute', 'runtool', 'invoketool'];
+
+export class ToolDetector {
+  static detectToolCall(
+    funcName: string,
+    funcQualname: string,
+    _args: unknown[],
+    _kwargs: Record<string, unknown>,
+  ): string | undefined {
+    const fn = funcName.toLowerCase();
+    const qn = funcQualname.toLowerCase();
+
+    for (const pattern of TOOL_PATTERNS) {
+      if (fn.includes(pattern) || qn.includes(pattern)) return 'tool';
+    }
+    if (qn.includes('retriever') || qn.includes('vectorstore')) return 'retriever';
+    if (qn.includes('database') || qn.includes('sql')) return 'database';
+    return undefined;
+  }
+}
+
+export function detectNodeType(
+  funcName: string,
+  funcQualname: string,
+  args: unknown[],
+  kwargs: Record<string, unknown>,
+): [string, LLMCallInfo | undefined] {
+  const llmInfo = LLMDetector.detectLLMCall(funcName, funcQualname, args, kwargs);
+  if (llmInfo) return ['llm', llmInfo];
+  const toolType = ToolDetector.detectToolCall(funcName, funcQualname, args, kwargs);
+  if (toolType) return [toolType, undefined];
+  return ['function', undefined];
+}