@prostgles/prostgles 2.3.6-dev.1786548258

package/server/src/serverFunctions/askLLM/getLLMRequestBody.ts

@@ -0,0 +1,368 @@
+import { filterArr } from "@common/llmUtils";
+import {
+  includes,
+  isDefined,
+  omitKeys,
+  tryCatchV2,
+  type AnyObject,
+} from "prostgles-types";
+import type { LLMMessage } from "./askLLM";
+import type { FetchLLMResponseArgs } from "./fetchLLMResponse";
+import { getCompactedMessages } from "./getCompactedMessages";
+
+export const getLLMRequestBody = ({
+  llm_provider,
+  llm_credential,
+  messages: maybeEmptyMessages,
+  tools: maybeEmptyTools,
+  llm_model,
+  llm_chat,
+}: FetchLLMResponseArgs) => {
+  /**
+   * Sending an empty array of tools or messages produces an error in openrouter: tool_choice may only be specified while providing tools
+   */
+  const tools =
+    maybeEmptyTools && maybeEmptyTools.length ? maybeEmptyTools : undefined;
+
+  const nonEmptyMessages = maybeEmptyMessages
+    .map((m) => {
+      const nonEmptyMessageContent = m.content
+        .filter((m) => m.type !== "text" || !("text" in m) || m.text.trim())
+        .map((c) => {
+          if (c.type === "text-document") {
+            return {
+              type: "text",
+              text: `<document name=${JSON.stringify(c.fileName)}>\n${c.text}\n</document>`,
+            } as const;
+          }
+          return c;
+        });
+      return {
+        ...m,
+        content: nonEmptyMessageContent,
+      };
+    })
+    .filter((m) => m.content.length);
+
+  const toolNames = new Set<string>();
+  tools?.forEach((tool) => {
+    if (toolNames.has(tool.name)) {
+      throw new Error(
+        `Duplicate tool name ${JSON.stringify(tool.name)} in tools array`,
+      );
+    }
+    toolNames.add(tool.name);
+  });
+
+  const { messagesAfterCompaction } = getCompactedMessages({
+    nonEmptyMessages,
+  });
+  const systemMessage = messagesAfterCompaction.filter(
+    (m) => m.role === "system",
+  );
+  const [systemMessageObj, ...otherSM] = systemMessage;
+  if (otherSM.length) throw "Multiple prompts found";
+  const { api_key } = llm_credential;
+  const model = llm_model.name;
+  const provider = llm_provider.id as
+    | "OpenAI"
+    | "Anthropic"
+    | "Google"
+    | "Prostgles"
+    | "OpenRouter"
+    | "Ollama";
+  const messages =
+    includes(["OpenAI", "OpenRouter", "Prostgles", "Ollama"], provider) ?
+      messagesAfterCompaction
+    : messagesAfterCompaction.filter((m) => m.role !== "system");
+  const headers: RequestInit["headers"] =
+    provider === "Anthropic" ?
+      {
+        "content-type": "application/json",
+        "x-api-key": api_key,
+        "anthropic-version": "2023-06-01",
+      }
+    : provider === "Google" ?
+      {
+        "content-type": "application/json",
+      }
+    : {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${api_key}`,
+      };
+
+  const body =
+    provider === "Anthropic" ?
+      {
+        model,
+        system: systemMessageObj?.content.map((c, i, arr) =>
+          i === arr.length - 1 ?
+            {
+              ...c,
+              cache_control: { type: "ephemeral" },
+            }
+          : c,
+        ),
+        messages: messages.map((m) => ({
+          ...m,
+          content: m.content.map((c) => {
+            return (
+              c.type === "image" ?
+                {
+                  type: "image",
+                  source: {
+                    ...c.source,
+                    data: removeBase64Prefix(c.source.data),
+                  },
+                }
+              : c.type === "tool_result" ? omitKeys(c, ["tool_name"])
+              : c
+            );
+          }),
+        })),
+        tools,
+      }
+    : provider === "Google" ?
+      {
+        system_instruction: systemMessageObj && {
+          parts: systemMessageObj.content
+            .map((c) => {
+              if (c.type !== "text" || !("text" in c)) return undefined;
+              return {
+                text: c.text,
+              };
+            })
+            .filter(isDefined),
+        },
+        /**
+         * https://ai.google.dev/gemini-api/docs/text-generation?lang=rest
+         */
+        contents: messages.map((m) => ({
+          role:
+            m.content.some((c) => c.type === "tool_result") ? "function"
+            : m.content.some((c) => c.type === "tool_use") ? "model"
+            : m.role,
+          parts: m.content
+            .map((c) => {
+              // if (c.type === "image") {
+              //   return {
+              //     inlineData: {
+              //       mimeType: c.source.media_type,
+              //       data: removeBase64Prefix(c.source.data),
+              //     },
+              //   };
+              // }
+              if (c.type === "text" && "text" in c) return [{ text: c.text }];
+              if (c.type === "tool_use") {
+                return [
+                  {
+                    functionCall: {
+                      name: c.name,
+                      args: c.input,
+                    },
+                  },
+                ];
+              }
+              const toolResults = getToolResults([c]);
+              if (toolResults.length) {
+                return toolResults.map((toolResult) => {
+                  const resultText = toolResult.content;
+                  const funcName = toolResult.tool_name; //arr[i - 1]?.content
+                  // .map((c) =>
+                  //   c.type === "tool_use" && c.name ? c.name : undefined,
+                  // )
+                  // .find(isDefined);
+                  const resultObject = tryCatchV2(
+                    () => JSON.parse(resultText || "{}") as AnyObject,
+                  );
+                  return {
+                    functionResponse: {
+                      name: funcName,
+                      response: {
+                        name: funcName,
+                        content: resultObject.data ?? {
+                          parsingError: "Could not parse tool result as JSON",
+                          toolResult: resultText,
+                        },
+
+                        // JSON.parse(
+                        //   (resultText as string | undefined) ?? "{}",
+                        // ),
+                      },
+                    },
+                  };
+                });
+              }
+            })
+            .flat()
+            .filter(isDefined),
+        })),
+        ...(tools && {
+          tools: [
+            {
+              functionDeclarations: tools.map((t) => ({
+                name: t.name,
+                description: t.description,
+                parameters: omitKeys(t.input_schema, ["$schema", "$id"]),
+              })),
+            },
+          ],
+        }),
+      }
+    : /**  "OpenAI"  */
+      {
+        model,
+        messages: messages
+          .map((m) => {
+            const toolUseContent = filterArr(m.content, {
+              type: "tool_use" as const,
+            });
+            const textContent = filterArr(m.content, { type: "text" as const });
+
+            if (toolUseContent.length) {
+              return {
+                role: "assistant",
+                content: textContent.map((t) => t.text).join("\n") || "",
+                tool_calls: toolUseContent.map((tc) => ({
+                  id: tc.id,
+                  type: "function",
+                  function: {
+                    name: tc.name,
+                    arguments: JSON.stringify(tc.input),
+                  },
+                })),
+              };
+            }
+            const toolUseResults = getToolResults(m.content);
+            if (toolUseResults.length) {
+              return toolUseResults.map((toolUseResult) => {
+                return {
+                  role: "tool",
+                  tool_call_id: toolUseResult.tool_use_id,
+                  content: toolUseResult.content,
+                  ...(toolUseResult.is_error && {
+                    is_error: toolUseResult.is_error,
+                  }),
+                };
+              });
+            }
+            return {
+              ...m,
+              content: m.content.map((c) => {
+                if (c.type === "image") {
+                  return {
+                    type: "image_url",
+                    image_url: { url: c.source.data },
+                  };
+                }
+                if (c.type === "tool_result") {
+                  return {
+                    type: "function_call_output",
+                    call_id: c.tool_use_id,
+                    output:
+                      typeof c.content === "string" ?
+                        c.content
+                      : (filterArr(c.content, { type: "text" as const })[0]
+                          ?.text ??
+                        "?? Internal issue in tool result parsing in prostgles ui"),
+                  };
+                }
+                return c;
+              }),
+              ...(m.content.some((c) => c.type === "tool_result") && {
+                role: "tool",
+              }),
+            };
+          })
+          .flat(),
+        tools: tools?.map((t) => ({
+          type: "function",
+          function: {
+            name: t.name,
+            description: t.description,
+            parameters: omitKeys(t.input_schema, ["$schema", "$id"]),
+          },
+        })),
+        ...(provider === "Ollama" && {
+          stream: false,
+        }),
+      };
+
+  const toolCalls = new Map<string, { resultCount: number }>();
+  messages.forEach((message) => {
+    message.content.forEach((m) => {
+      if (m.type === "tool_use") {
+        const match = toolCalls.get(m.id);
+        if (match) {
+          throw new Error(
+            `Duplicate tool_use id ${m.id} for ${m.name}, this should not happen`,
+          );
+        }
+        toolCalls.set(m.id, { resultCount: 0 });
+      } else if (m.type === "tool_result" && m.tool_use_id) {
+        const toolCall = toolCalls.get(m.tool_use_id);
+        if (!toolCall) {
+          throw new Error(
+            `tool_result with tool_use_id for ${m.tool_name} has no matching tool_use, this should not happen`,
+          );
+        }
+        if (toolCall.resultCount) {
+          throw new Error(
+            `Multiple tool_result with the same tool_use_id for ${m.tool_name}, this should not happen`,
+          );
+        }
+        toolCall.resultCount++;
+      }
+    });
+  });
+
+  const bodyWithExtras = {
+    ...body,
+    ...llm_provider.extra_body,
+    ...llm_credential.extra_body,
+    ...llm_model.extra_body,
+    ...llm_chat.extra_body,
+  };
+  return {
+    body: JSON.stringify(bodyWithExtras),
+    headers: {
+      ...headers,
+      ...llm_provider.extra_headers,
+      ...llm_credential.extra_headers,
+      ...llm_model.extra_headers,
+      ...llm_chat.extra_headers,
+    },
+  };
+};
+
+const removeBase64Prefix = (data: string) => {
+  const base64Prefix = "base64,";
+  if (data.includes(base64Prefix)) {
+    return data.substring(data.indexOf("base64,") + 7);
+  }
+  return data;
+};
+
+const getToolResults = (content: LLMMessage) => {
+  const toolUseResults = filterArr(content, {
+    type: "tool_result" as const,
+  });
+
+  return toolUseResults.map((toolUseResult) => {
+    const { content, ...otherProps } = toolUseResult;
+    const contentText =
+      typeof content === "string" ? content : (
+        filterArr(content, { type: "text" as const })
+          .map((c) => c.text)
+          .join("\n")
+      );
+    return {
+      ...otherProps,
+      role: "tool",
+      tool_call_id: toolUseResult.tool_use_id,
+      content:
+        contentText ||
+        "?? Internal issue in tool result parsing in prostgles ui",
+    };
+  });
+};

package/server/src/serverFunctions/askLLM/getLLMToolsAllowedInThisChat.ts

@@ -0,0 +1,179 @@
+import {
+  getJSONBSchemaAsJSONSchema,
+  isDefined,
+  isEmpty,
+  type FullFilter,
+} from "prostgles-types";
+import type { DBS } from "../..";
+
+import { getJsonSchemaAsTs } from "@common/getJsonSchemaAsTs";
+import {
+  getMCPFullToolName,
+  getMCPToolNameParts,
+  getProstglesMCPFullToolName,
+  type AllowedChatTool,
+} from "@common/mcpUtils";
+import type { DBSSchema } from "@common/publishUtils";
+import type { AuthClientRequest } from "prostgles-server/dist/Auth/AuthTypes";
+import { getAgentGoalTools } from "./agentConstants";
+import { getMCPServerTools } from "./prostglesLLMTools/getMCPServerTools";
+import { getMcpToolsWithDynamicDescription } from "./prostglesLLMTools/getMcpToolsWithDynamicDescription";
+import type { PROSTGLES_MCP_SERVERS_AND_TOOLS } from "@common/prostglesMcp";
+import type { uiMcpSchema } from "@common/mcp/ui.mcp.schema";
+
+export type GetLLMToolsArgs = {
+  userType: string;
+  chat: DBSSchema["llm_chats"];
+  dbs: DBS;
+  clientReq: AuthClientRequest;
+};
+
+export type MCPToolSchema = {
+  name: string;
+  description: string;
+  input_schema: ReturnType<typeof getJSONBSchemaAsJSONSchema>;
+};
+
+export const getLLMToolsAllowedInThisChat = async ({
+  userType,
+  dbs,
+  chat,
+  clientReq,
+}: GetLLMToolsArgs): Promise<undefined | AllowedChatTool[]> => {
+  const { id: chatId } = chat;
+
+  const isAgentChat =
+    chat.agent_info && chat.agent_info.type !== "orchestrator";
+  const llm_chats_allowed_mcp_tools =
+    await dbs.llm_chats_allowed_mcp_tools.find({
+      chat_id: chatId,
+    });
+  const allowedToolsFilter = (() => {
+    const filter = {
+      id: { $in: llm_chats_allowed_mcp_tools.map(({ tool_id }) => tool_id) },
+      // $existsJoined: {
+      //   llm_chats_allowed_mcp_tools: {
+      //     chat_id: chatId,
+      //   },
+      // },
+    } as const;
+    if (isAgentChat) {
+      const { serverName, toolName } = getMCPToolNameParts(
+        getProstglesMCPFullToolName("prostgles-ui", "compact_context"),
+      )!;
+      return {
+        $or: [filter, { name: toolName, server_name: serverName }],
+      } as const satisfies FullFilter<DBSSchema["mcp_server_tools"], void>;
+    }
+    return filter;
+  })();
+  const { mcpTools: mcpToolsWithoutExtraInfo } = await getMCPServerTools(
+    dbs,
+    allowedToolsFilter,
+  );
+  const allowedMcpToolsWithInfo = mcpToolsWithoutExtraInfo
+    .map(({ id, ...tool }) => {
+      const info = llm_chats_allowed_mcp_tools.find(
+        ({ tool_id }) => tool_id === id,
+      );
+      if (!info) return;
+      return {
+        ...tool,
+        ...info,
+        auto_approve: Boolean(info.auto_approve),
+      } satisfies AllowedChatTool;
+    })
+    .filter(isDefined);
+
+  const tools: Map<string, AllowedChatTool> = new Map();
+
+  const { mcpTools } = await getMcpToolsWithDynamicDescription({
+    dbs,
+    allowedMcpToolsWithInfo,
+  });
+
+  /** Check for name collisions */
+  [
+    ...mcpTools.map((t) => {
+      const toolNameParts = getMCPToolNameParts(t.name);
+      if (!toolNameParts) {
+        throw new Error(`Could not parse tool name parts for ${t.name}`);
+      }
+      return {
+        ...t,
+        tool_name: toolNameParts.toolName,
+        server_name: toolNameParts.serverName,
+      } satisfies AllowedChatTool;
+    }),
+  ].forEach((tool) => {
+    const { name } = tool;
+    if (tools.has(name)) {
+      throw new Error(
+        `Tool name collision: ${name} is used by both MCP tool and/or other function`,
+      );
+    }
+    tools.set(name, tool);
+  });
+
+  if (chat.agent_info && chat.agent_info.type !== "orchestrator") {
+    const agentGoalTools = getAgentGoalTools(chat.agent_info);
+    agentGoalTools.forEach((agentGoalTool, index) => {
+      tools.set(agentGoalTool.name, {
+        ...agentGoalTool,
+        auto_approve: true,
+        mode: "auto-approved-user-actionable",
+        server_name: "",
+        tool_name: agentGoalTool.name,
+        tool_id: -1 - index, // avoid collision with normal tools
+      });
+    });
+  }
+
+  const toolList = Array.from(tools.values());
+
+  const { mcpToolSchemaMode } = chat.options ?? {};
+  if (mcpToolSchemaMode) {
+    return toolList.map((toolSchema) => {
+      const { input_schema, description, ...toolInfo } = toolSchema;
+
+      const nonHiddenSchemaToolNames = [
+        "get_specific_tool_schemas",
+        "ask_user_questions",
+        "request_tool_access",
+        "compact_context",
+        "get_tool_list",
+      ] as const satisfies (keyof typeof uiMcpSchema)[];
+      if (
+        nonHiddenSchemaToolNames.some(
+          (toolName) =>
+            toolInfo.name ===
+            getProstglesMCPFullToolName("prostgles-ui", toolName),
+        )
+      ) {
+        return toolSchema;
+      }
+      return {
+        ...toolInfo,
+        input_schema: {},
+        description:
+          mcpToolSchemaMode === "hide-schemas-and-descriptions" ?
+            "Must use 'get_specific_tool_schemas' tool to see description and input schema for this tool"
+          : isEmpty(input_schema) ? description
+          : [
+              description,
+              `\nThe tool acceppts the following input (must be valid json) expressed in typescript types:`,
+              "```typescript",
+              getJsonSchemaAsTs(input_schema),
+              "```",
+            ].join("\n"),
+      };
+    });
+  }
+  return toolList;
+};
+
+export const getAllToolNames = async (dbs: DBS): Promise<string[]> => {
+  const mcpTools = await dbs.mcp_server_tools.find();
+
+  return mcpTools.map((t) => getMCPFullToolName(t.server_name, t.name));
+};

package/server/src/serverFunctions/askLLM/getLLMUsageCost.ts

@@ -0,0 +1,91 @@
+import type { DBSSchema } from "@common/publishUtils";
+import type {
+  AnthropicChatCompletionResponse,
+  GoogleGeminiChatCompletionResponse,
+  OpenAIChatCompletionResponse,
+} from "./LLMResponseTypes";
+
+export const getLLMUsageCost = (
+  model: DBSSchema["llm_models"],
+  meta:
+    | { type: "OpenAI"; meta: Pick<OpenAIChatCompletionResponse, "usage"> }
+    | {
+        type: "Gemini";
+        meta: Pick<GoogleGeminiChatCompletionResponse, "usageMetadata">;
+      }
+    | {
+        type: "Anthropic";
+        meta: Pick<AnthropicChatCompletionResponse, "usage">;
+      },
+) => {
+  const cacheReadTokens =
+    meta.type === "OpenAI" ?
+      (meta.meta.usage?.prompt_tokens_details?.cached_tokens ?? 0)
+    : meta.type === "Anthropic" ? meta.meta.usage.cache_read_input_tokens
+    : 0;
+  const cacheWriteTokens =
+    meta.type === "Anthropic" ? meta.meta.usage.cache_creation_input_tokens : 0;
+
+  const inputTokens =
+    meta.type === "Gemini" ? meta.meta.usageMetadata.promptTokenCount
+      /**
+       * https://community.openai.com/t/will-cached-prompt-be-charged-in-each-api-call/977999/2
+       */
+    : meta.type === "OpenAI" ?
+      (meta.meta.usage?.prompt_tokens ?? 0) - cacheReadTokens
+    : meta.meta.usage.input_tokens;
+  const outputTokens =
+    meta.type === "Gemini" ? meta.meta.usageMetadata.candidatesTokenCount
+    : meta.type === "OpenAI" ? (meta.meta.usage?.completion_tokens ?? 0)
+    : meta.meta.usage.output_tokens;
+
+  const total_tokens =
+    cacheReadTokens + cacheWriteTokens + inputTokens + outputTokens;
+
+  const cost = getLlmMessageCost(model, {
+    cacheReadTokens,
+    cacheWriteTokens,
+    inputTokens,
+    outputTokens,
+  });
+  return { cost, total_tokens };
+};
+
+export const getLlmMessageCost = (
+  model: DBSSchema["llm_models"],
+  message: {
+    cacheReadTokens: number;
+    cacheWriteTokens: number;
+    inputTokens: number;
+    outputTokens: number;
+  },
+) => {
+  if (!model.pricing_info) return;
+  const {
+    input,
+    output,
+    threshold,
+    cachedInput = 0,
+    cachedOutput = 0,
+  } = model.pricing_info;
+
+  const { cacheReadTokens, cacheWriteTokens, inputTokens, outputTokens } =
+    message;
+
+  const inputPrice =
+    threshold && inputTokens > threshold.tokenLimit ? threshold.input : input;
+  const outputPrice =
+    threshold && outputTokens > threshold.tokenLimit ?
+      threshold.output
+    : output;
+
+  const cachePrice =
+    (cacheReadTokens / 1e6) * cachedInput +
+    (cacheWriteTokens / 1e6) * cachedOutput;
+
+  return (
+    cachePrice +
+    (inputPrice / 1e6) * inputTokens +
+    (outputPrice / 1e6) * outputTokens
+  );
+};

package/server/src/serverFunctions/askLLM/getPastMessages.ts

@@ -0,0 +1,9 @@
+import type { DBS } from "@src/index";
+
+export const getPastMessages = async (dbs: DBS, chatId: number) => {
+  const messages = await dbs.llm_messages.find(
+    { chat_id: chatId },
+    { orderBy: { created: 1 } },
+  );
+  return messages;
+};

package/server/src/serverFunctions/askLLM/getUserMessageCost.ts

@@ -0,0 +1,21 @@
+import type { DBSSchema } from "@common/publishUtils";
+import type { LLMMessage } from "./askLLM";
+import { getLlmMessageCost } from "./getLLMUsageCost";
+
+const TOKENS_PER_CHARACTER = 0.25;
+
+export const getUserMessageCost = (
+  userMessage: LLMMessage,
+  model: DBSSchema["llm_models"],
+): number => {
+  const messageTextLength = JSON.stringify(userMessage).length;
+
+  return (
+    getLlmMessageCost(model, {
+      cacheReadTokens: 0,
+      cacheWriteTokens: 0,
+      inputTokens: messageTextLength * TOKENS_PER_CHARACTER,
+      outputTokens: 0,
+    }) ?? 0
+  );
+};