@garrix82/reactgenie-dsl 1.0.0

package/lib/nl/nl-parser.ts

@@ -0,0 +1,616 @@
+import { AgentResponsePayload, PromptGen } from "./prompt-gen";
+import { getLLMMonitor, type LLMMonitor } from "./llm-monitoring";
+import {
+  buildCommandRepairInputText,
+  buildCommandRepairMessages,
+  buildFallbackAgentResponse,
+  CommandValidator,
+  createJsonChatCompletion,
+  DEFAULT_SEMANTIC_PARSER_SAMPLING,
+  extractResponseOutputText,
+  extractStructuredAgentResponse,
+  extractStructuredCommand,
+  getStructuredAgentResponseFormat,
+  getStructuredResponseFormat,
+  getStructuredResponseTextFormat,
+  normalizeAgentResponse,
+  normalizeBaseUrl,
+  SamplingParams,
+} from "./nl-parser-utils";
+export type { SamplingParams } from "./nl-parser-utils";
+
+export interface OpenAIParserOptions {
+  statefulValidationRetry?: boolean;
+}
+
+function buildOpenAIEndpoint(baseUrl: string, suffix: string): string {
+  const normalized = normalizeBaseUrl(baseUrl);
+  if (normalized.endsWith(`/v1/${suffix}`)) {
+    return normalized;
+  }
+  if (normalized.endsWith("/v1")) {
+    return `${normalized}/${suffix}`;
+  }
+  return `${normalized}/v1/${suffix}`;
+}
+
+function requireConfiguredModel(model: string): string {
+  const trimmed = model.trim();
+  if (!trimmed) {
+    throw new Error(
+      "OpenAI parser requires semanticModel / SEMANTIC_MODEL configuration."
+    );
+  }
+  return trimmed;
+}
+
+function asFiniteNumber(value: unknown): number | undefined {
+  return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+
+function extractCostMetrics(response: any): {
+  inputCost?: number;
+  outputCost?: number;
+  totalCost?: number;
+} {
+  const usage = response?.usage ?? response?.usage_metadata ?? {};
+  const inputCost =
+    asFiniteNumber(usage?.input_cost) ??
+    asFiniteNumber(usage?.prompt_cost) ??
+    asFiniteNumber(usage?.prompt_cost_usd);
+  const outputCost =
+    asFiniteNumber(usage?.output_cost) ??
+    asFiniteNumber(usage?.completion_cost) ??
+    asFiniteNumber(usage?.completion_cost_usd);
+  const totalCost =
+    asFiniteNumber(usage?.total_cost) ??
+    asFiniteNumber(usage?.cost) ??
+    asFiniteNumber(usage?.total_cost_usd) ??
+    (typeof inputCost === "number" && typeof outputCost === "number"
+      ? inputCost + outputCost
+      : undefined);
+
+  return {
+    ...(typeof inputCost === "number" ? { inputCost } : {}),
+    ...(typeof outputCost === "number" ? { outputCost } : {}),
+    ...(typeof totalCost === "number" ? { totalCost } : {}),
+  };
+}
+
+export class NlParser {
+  private chatCompletionsEndpoint: string;
+  private responsesEndpoint: string;
+  private samplingParams: Required<SamplingParams>;
+  private llmMonitor!: LLMMonitor;
+  private statefulValidationRetry: boolean;
+
+  constructor(
+    public prompt: PromptGen,
+    private apiKey: string,
+    private basePath: string,
+    private model: string,
+    samplingParams?: SamplingParams,
+    langsmithApiKey?: string,
+    langsmithProject?: string,
+    langsmithEndpoint?: string,
+    private validateCommand?: CommandValidator,
+    parserOptions?: OpenAIParserOptions
+  ) {
+    this.model = requireConfiguredModel(model);
+    this.chatCompletionsEndpoint = buildOpenAIEndpoint(this.basePath, "chat/completions");
+    this.responsesEndpoint = buildOpenAIEndpoint(this.basePath, "responses");
+    this.samplingParams = {
+      ...DEFAULT_SEMANTIC_PARSER_SAMPLING,
+      ...samplingParams,
+    };
+    this.statefulValidationRetry = !!parserOptions?.statefulValidationRetry;
+
+    this.llmMonitor = getLLMMonitor({
+      apiKey: langsmithApiKey,
+      project: langsmithProject,
+      endpoint: langsmithEndpoint,
+    });
+  }
+
+  private async createChatCompletion(body: Record<string, unknown>): Promise<any> {
+    return createJsonChatCompletion({
+      endpoint: this.chatCompletionsEndpoint,
+      apiKey: this.apiKey,
+      body,
+      provider: "openai",
+      requestLabel: "OpenAI-compatible request failed",
+      invalidPayloadLabel: "OpenAI-compatible endpoint returned an invalid JSON payload",
+    });
+  }
+
+  private async createResponse(body: Record<string, unknown>): Promise<any> {
+    return createJsonChatCompletion({
+      endpoint: this.responsesEndpoint,
+      apiKey: this.apiKey,
+      body,
+      provider: "openai",
+      requestLabel: "OpenAI Responses request failed",
+      invalidPayloadLabel: "OpenAI Responses endpoint returned an invalid JSON payload",
+    });
+  }
+
+  private async runStructuredCommand(
+    traceName: string,
+    promptVariants: Array<{ label: string; text: string }>,
+    userUtterance: string
+  ): Promise<{
+    response: any;
+    completion: string;
+    usedPromptText: string;
+    responseId?: string;
+  }> {
+    let usedPromptText = promptVariants[0]?.text || "";
+    let lastError: unknown;
+
+    const runParse = async (
+      strict: boolean,
+      promptText: string
+    ): Promise<{ response: any; command: string; responseId?: string }> => {
+      if (this.statefulValidationRetry) {
+        return this.runStructuredCommandResponse(promptText, strict);
+      }
+      return this.runStructuredCommandMessages([
+        {
+          role: "user",
+          content: promptText,
+        },
+      ], strict);
+    };
+
+    for (const promptVariant of promptVariants) {
+      usedPromptText = promptVariant.text;
+      try {
+        const result = await this.llmMonitor.traceCall(
+          `${traceName}_${promptVariant.label}`,
+          { model: this.model, provider: "openai" },
+          async () => runParse(false, promptVariant.text)
+        );
+        const validated = await this.validateOrRepairCommand(
+          traceName,
+          promptVariant.label,
+          userUtterance,
+          result.command,
+          usedPromptText,
+          result.responseId
+        );
+        return {
+          response: result.response,
+          completion: validated,
+          usedPromptText,
+          ...(result.responseId ? { responseId: result.responseId } : {}),
+        };
+      } catch {
+        try {
+          const result = await this.llmMonitor.traceCall(
+            `${traceName}_${promptVariant.label}`,
+            { model: this.model, provider: "openai" },
+            async () => runParse(true, promptVariant.text)
+          );
+          const validated = await this.validateOrRepairCommand(
+            traceName,
+            promptVariant.label,
+            userUtterance,
+            result.command,
+            usedPromptText,
+            result.responseId
+          );
+          return {
+            response: result.response,
+            completion: validated,
+            usedPromptText,
+            ...(result.responseId ? { responseId: result.responseId } : {}),
+          };
+        } catch (strictError) {
+          lastError = strictError;
+        }
+      }
+    }
+
+    throw (
+      lastError instanceof Error
+        ? lastError
+        : new Error("OpenAI parse failed for filtered prompt")
+    );
+  }
+
+  private async runStructuredCommandResponse(
+    promptText: string,
+    strict: boolean
+  ): Promise<{ response: any; command: string; responseId?: string }> {
+    const response = await this.createResponse({
+      model: this.model,
+      store: true,
+      input: [
+        {
+          role: "user",
+          content: [{ type: "input_text", text: promptText }],
+        },
+      ],
+      text: {
+        format: getStructuredResponseTextFormat(strict),
+      },
+    });
+
+    const raw = extractResponseOutputText(response);
+    const command = extractStructuredCommand(raw);
+    if (!command) {
+      throw new Error("OpenAI Responses structured output missing command");
+    }
+
+    return {
+      response,
+      command,
+      ...(typeof response?.id === "string" ? { responseId: response.id } : {}),
+    };
+  }
+
+  private async runStructuredCommandMessages(
+    messages: ReadonlyArray<{
+      role: "system" | "user" | "assistant";
+      content: string;
+    }>,
+    strict: boolean
+  ): Promise<{ response: any; command: string }> {
+      const response = await this.createChatCompletion({
+        model: this.model,
+        messages: [...messages],
+        temperature: this.samplingParams.temperature,
+        max_tokens: 256,
+        top_p: this.samplingParams.top_p,
+        frequency_penalty: this.samplingParams.frequency_penalty,
+        presence_penalty: this.samplingParams.presence_penalty,
+        response_format: getStructuredResponseFormat(strict),
+      });
+
+      const raw = response.choices[0]?.message?.content ?? null;
+      const command = extractStructuredCommand(raw);
+      if (!command) {
+        throw new Error("OpenAI structured output missing command");
+      }
+      return { response, command };
+  }
+
+  private async validateOrRepairCommand(
+    traceName: string,
+    promptLabel: string,
+    userUtterance: string,
+    command: string,
+    lastUsedPrompt?: string,
+    responseId?: string
+  ): Promise<string> {
+    if (!this.validateCommand) {
+      return command;
+    }
+
+    try {
+      await this.validateCommand(command);
+      return command;
+    } catch (validationError) {
+      const validationMessage =
+        validationError instanceof Error ? validationError.message : String(validationError);
+
+      if (this.statefulValidationRetry && responseId) {
+        let lastError: unknown = validationError;
+        for (const strict of [false, true]) {
+          try {
+            const repaired = await this.llmMonitor.traceCall(
+              `${traceName}_${promptLabel}_repair_${strict ? "strict" : "lenient"}`,
+              { model: this.model, provider: "openai" },
+              async () =>
+                this.runStructuredCommandRepairResponse(
+                  responseId,
+                  buildCommandRepairInputText(userUtterance, command, validationMessage),
+                  strict
+                )
+            );
+            await this.validateCommand(repaired.command);
+            return repaired.command;
+          } catch (repairError) {
+            lastError = repairError;
+          }
+        }
+
+        throw (
+          lastError instanceof Error
+            ? lastError
+            : new Error("OpenAI repair failed after semantic validation")
+        );
+      }
+
+      const messages = buildCommandRepairMessages(
+        userUtterance,
+        command,
+        validationMessage,
+        lastUsedPrompt
+      );
+
+      let lastError: unknown = validationError;
+      for (const strict of [false, true]) {
+        try {
+          const repaired = await this.llmMonitor.traceCall(
+            `${traceName}_${promptLabel}_repair_${strict ? "strict" : "lenient"}`,
+            { model: this.model, provider: "openai" },
+            async () => this.runStructuredCommandMessages(messages, strict)
+          );
+          await this.validateCommand(repaired.command);
+          return repaired.command;
+        } catch (repairError) {
+          lastError = repairError;
+        }
+      }
+
+      throw (
+        lastError instanceof Error
+          ? lastError
+          : new Error("OpenAI repair failed after semantic validation")
+      );
+    }
+  }
+
+  private async runStructuredCommandRepairResponse(
+    previousResponseId: string,
+    repairText: string,
+    strict: boolean
+  ): Promise<{ response: any; command: string; responseId?: string }> {
+    const response = await this.createResponse({
+      model: this.model,
+      store: true,
+      previous_response_id: previousResponseId,
+      input: [
+        {
+          role: "user",
+          content: [{ type: "input_text", text: repairText }],
+        },
+      ],
+      text: {
+        format: getStructuredResponseTextFormat(strict),
+      },
+    });
+
+    const raw = extractResponseOutputText(response);
+    const command = extractStructuredCommand(raw);
+    if (!command) {
+      throw new Error("OpenAI Responses repair output missing command");
+    }
+
+    return {
+      response,
+      command,
+      ...(typeof response?.id === "string" ? { responseId: response.id } : {}),
+    };
+  }
+
+  /** @deprecated Prefer parse() for all new code paths. */
+  async oldParse(nl: string): Promise<string | null> {
+    const promptText = this.prompt.prompt(nl);
+    const startTime = Date.now();
+
+    try {
+      const result = await this.runStructuredCommand("oldParse", [
+        { label: "legacy_prompt", text: promptText },
+      ], nl);
+
+      await this.llmMonitor.logCall({
+        model: this.model,
+        provider: "openai",
+        promptTokens: result.response.usage?.prompt_tokens,
+        completionTokens: result.response.usage?.completion_tokens,
+        totalTokens: result.response.usage?.total_tokens,
+        ...extractCostMetrics(result.response),
+        prompt: result.usedPromptText,
+        completion: result.completion,
+        latency: Date.now() - startTime,
+        timestamp: new Date(),
+      });
+
+      return result.completion;
+    } catch (error) {
+      await this.llmMonitor.logCall({
+        model: this.model,
+        provider: "openai",
+        prompt: promptText,
+        completion: null,
+        latency: Date.now() - startTime,
+        timestamp: new Date(),
+        error: error instanceof Error ? error.message : String(error),
+      });
+      throw error;
+    }
+  }
+
+  async parse(nl: string): Promise<string | null> {
+    const startTime = Date.now();
+    const promptVariants: Array<{ label: string; text: string }> = [];
+    const defaultPrompt = this.prompt.zero_shot_prompt(nl);
+    promptVariants.push({ label: "filtered", text: defaultPrompt });
+
+    try {
+      const result = await this.runStructuredCommand("parse", promptVariants, nl);
+      await this.llmMonitor.logCall({
+        model: this.model,
+        provider: "openai",
+        promptTokens: result.response.usage?.prompt_tokens,
+        completionTokens: result.response.usage?.completion_tokens,
+        totalTokens: result.response.usage?.total_tokens,
+        ...extractCostMetrics(result.response),
+        prompt: result.usedPromptText,
+        completion: result.completion,
+        latency: Date.now() - startTime,
+        timestamp: new Date(),
+      });
+      return result.completion;
+    } catch (error) {
+      await this.llmMonitor.logCall({
+        model: this.model,
+        provider: "openai",
+        prompt: promptVariants[0]?.text || "",
+        completion: null,
+        latency: Date.now() - startTime,
+        timestamp: new Date(),
+        error: error instanceof Error ? error.message : String(error),
+      });
+      throw error;
+    }
+  }
+
+  async parseGpt4(nl: string): Promise<string | null> {
+    const promptText = this.prompt.prompt(nl);
+    const startTime = Date.now();
+
+      const runParse = async (strict: boolean) => {
+        return this.runStructuredCommandMessages(
+          [
+            {
+              role: "system",
+              content: "only generate one line of code",
+            },
+            {
+              role: "user",
+              content: promptText,
+            },
+          ],
+          strict
+        );
+      };
+
+    try {
+      let result;
+      try {
+        result = await this.llmMonitor.traceCall(
+          "parseGpt4",
+          { model: this.model, provider: "openai" },
+          async () => runParse(false)
+        );
+      } catch {
+        result = await this.llmMonitor.traceCall(
+          "parseGpt4",
+          { model: this.model, provider: "openai" },
+          async () => runParse(true)
+        );
+      }
+
+      const validatedCommand = await this.validateOrRepairCommand(
+        "parseGpt4",
+        "legacy_prompt",
+        nl,
+        result.command,
+        promptText
+      );
+
+      await this.llmMonitor.logCall({
+        model: this.model,
+        provider: "openai",
+        promptTokens: result.response.usage?.prompt_tokens,
+        completionTokens: result.response.usage?.completion_tokens,
+        totalTokens: result.response.usage?.total_tokens,
+        ...extractCostMetrics(result.response),
+        prompt: promptText,
+        completion: validatedCommand,
+        latency: Date.now() - startTime,
+        timestamp: new Date(),
+      });
+
+      return validatedCommand;
+    } catch (error) {
+      await this.llmMonitor.logCall({
+        model: this.model,
+        provider: "openai",
+        prompt: promptText,
+        completion: null,
+        latency: Date.now() - startTime,
+        timestamp: new Date(),
+        error: error instanceof Error ? error.message : String(error),
+      });
+      throw error;
+    }
+  }
+
+  async respond(
+    nl: string,
+    parsed: string,
+    result: string
+  ): Promise<AgentResponsePayload | null> {
+    const promptText = this.prompt.response_prompt(nl, parsed, result);
+    const startTime = Date.now();
+
+    const runRespond = async (strict: boolean) => {
+      const response = await this.createChatCompletion({
+        model: this.model,
+        messages: [
+          {
+            role: "system",
+            content: "Return only a strict JSON object for agent_response_json.",
+          },
+          {
+            role: "user",
+            content: promptText,
+          },
+        ],
+        temperature: this.samplingParams.temperature,
+        max_tokens: 256,
+        top_p: this.samplingParams.top_p,
+        frequency_penalty: this.samplingParams.frequency_penalty,
+        presence_penalty: this.samplingParams.presence_penalty,
+        response_format: getStructuredAgentResponseFormat(strict),
+      });
+
+      const raw = response.choices[0]?.message?.content ?? null;
+      const payload = extractStructuredAgentResponse(raw);
+      if (!payload) {
+        throw new Error("OpenAI structured output missing agent response payload");
+      }
+      return { response, payload };
+    };
+
+    try {
+      let result;
+      try {
+        result = await this.llmMonitor.traceCall(
+          "respond",
+          { model: this.model, provider: "openai" },
+          async () => runRespond(false)
+        );
+      } catch {
+        result = await this.llmMonitor.traceCall(
+          "respond",
+          { model: this.model, provider: "openai" },
+          async () => runRespond(true)
+        );
+      }
+
+      const normalizedCompletion = normalizeAgentResponse(result.payload);
+
+      await this.llmMonitor.logCall({
+        model: this.model,
+        provider: "openai",
+        promptTokens: result.response.usage?.prompt_tokens,
+        completionTokens: result.response.usage?.completion_tokens,
+        totalTokens: result.response.usage?.total_tokens,
+        ...extractCostMetrics(result.response),
+        prompt: promptText,
+        completion: JSON.stringify(normalizedCompletion),
+        latency: Date.now() - startTime,
+        timestamp: new Date(),
+      });
+
+      return normalizedCompletion;
+    } catch (error) {
+      const fallbackResponse = buildFallbackAgentResponse(parsed, result);
+      await this.llmMonitor.logCall({
+        model: this.model,
+        provider: "openai",
+        prompt: promptText,
+        completion: JSON.stringify(fallbackResponse),
+        latency: Date.now() - startTime,
+        timestamp: new Date(),
+        error: error instanceof Error ? error.message : String(error),
+      });
+      return fallbackResponse;
+    }
+  }
+}