@saltcorn/large-language-model 0.8.10 → 0.9.0

package/generate.js

@@ -1,4 +1,4 @@
-const fetch = require("node-fetch");
+const node_fetch = require("node-fetch");
 const util = require("util");
 const exec = util.promisify(require("child_process").exec);
 const db = require("@saltcorn/data/db");
@@ -207,7 +207,10 @@ const getCompletionOpenAICompatible = async (
     )
       body.temperature = 0.7;
   }
-
+  if (rest.streamCallback) {
+    body.stream = true;
+    delete body.streamCallback;
+  }
   if (responses_api) {
     for (const tool of body.tools || []) {
       if (tool.type !== "function") continue;
@@ -303,18 +306,115 @@ const getCompletionOpenAICompatible = async (
       )} to ${chatCompleteEndpoint} headers ${JSON.stringify(headers)}`
     );
   if (debugCollector) debugCollector.request = body;
-
-  const rawResponse = await fetch(chatCompleteEndpoint, {
+  const reqTimeStart = Date.now();
+  const rawResponse = await (global.fetch || node_fetch)(chatCompleteEndpoint, {
     method: "POST",
     headers,
     body: JSON.stringify(body),
   });
+  let streamParts = [];
+  let streamToolCalls = null;
+
+  if (rest.streamCallback) {
+    // https://stackoverflow.com/a/75751803/19839414
+    // https://stackoverflow.com/a/57664622/19839414
+    await new Promise((resolve, reject) => {
+      let dataDone = false;
+      let stashed = "";
+
+      const process_stream_data = (value) => {
+        const arr = value.split("\n");
+        arr.forEach((data) => {
+          if (data.length === 0) return; // ignore empty message
+          if (data.startsWith(":")) return; // ignore sse comment message
+          if (data === "data: [DONE]") {
+            dataDone = true;
+            resolve();
+            return;
+          }
+          try {
+            const json = JSON.parse(stashed + data.substring(6));
+            stashed = "";
+            console.log(json.choices[0]);
+
+            // callback
+
+            //answer store
+            if (json.choices?.[0]?.content)
+              streamParts.push(json.choices[0].content);
+            if (json.choices?.[0]?.delta?.content)
+              streamParts.push(json.choices[0].delta.content);
+            if (json.choices?.[0]?.delta?.tool_calls) {
+              if (!streamToolCalls) streamToolCalls = json.choices?.[0]?.delta;
+              else
+                json.choices?.[0]?.delta?.tool_calls.forEach((tc, ix) => {
+                  streamToolCalls.tool_calls[ix].function.arguments +=
+                    tc.function.arguments;
+                });
+            }
+            rest.streamCallback(json);
+          } catch (e) {
+            //console.error(e);
+            stashed = data.substring(6);
+          }
+        });
+      };
+      if (global.fetch) {
+        const reader = rawResponse.body
+          ?.pipeThrough(new TextDecoderStream())
+          .getReader();
+        if (!reader) return;
+        // eslint-disable-next-line no-constant-condition
+        (async () => {
+          while (!dataDone) {
+            // eslint-disable-next-line no-await-in-loop
+
+            const { value, done } = await reader.read();
+
+            if (done) {
+              dataDone = true;
+              resolve();
+              break;
+            }
+            process_stream_data(value);
+            if (dataDone) break;
+          }
+        })().catch((e) => {
+          //console.error(e);
+          dataDone = true;
+          reject(e);
+        });
+      } else
+        rawResponse.body.on("readable", () => {
+          let chunk;
+          while (null !== (chunk = rawResponse.body.read())) {
+            let value = chunk.toString();
+            process_stream_data(value);
+            if (dataDone) break;
+          }
+        });
+    });
+    if (debugCollector) {
+      //TODO get the full response
+      if (streamToolCalls) debugCollector.response = streamToolCalls;
+      debugCollector.response_time_ms = Date.now() - reqTimeStart;
+    }
+    return streamToolCalls
+      ? {
+          content: streamParts.join(""),
+          tool_calls: streamToolCalls.tool_calls,
+        }
+      : streamParts.join("");
+  }
   const results = await rawResponse.json();
   //console.log("results", results);
   if (debugResult)
     console.log("OpenAI response", JSON.stringify(results, null, 2));
   else getState().log(6, `OpenAI response ${JSON.stringify(results)}`);
-  if (debugCollector) debugCollector.response = results;
+  if (debugCollector) {
+    debugCollector.response = results;
+    debugCollector.response_time_ms = Date.now() - reqTimeStart;
+  }
 
   if (results.error) throw new Error(`OpenAI error: ${results.error.message}`);
   if (responses_api) {
@@ -395,7 +495,7 @@ const getImageGenOpenAICompatible = async (
   if (debugResult) console.log("OpenAI image request", imageEndpoint, body);
   if (debugCollector) debugCollector.request = body;
 
-  const rawResponse = await fetch(imageEndpoint, {
+  const rawResponse = await (global.fetch || node_fetch)(imageEndpoint, {
     method: "POST",
     headers,
     body: JSON.stringify(body),
@@ -424,7 +524,7 @@ const getEmbeddingOpenAICompatible = async (
     input: prompt,
   };
 
-  const rawResponse = await fetch(embeddingsEndpoint, {
+  const rawResponse = await (global.fetch || node_fetch)(embeddingsEndpoint, {
     method: "POST",
     headers,
     body: JSON.stringify(body),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saltcorn/large-language-model",
-  "version": "0.8.10",
+  "version": "0.9.0",
   "description": "Large language models and functionality for Saltcorn",
   "main": "index.js",
   "dependencies": {