automation_model 1.0.788-dev → 1.0.788-stage

package/lib/network.js

@@ -3,6 +3,60 @@ import fs from "fs";
 import { _stepNameToTemplate } from "./route.js";
 import crypto from "crypto";
 import { tmpdir } from "os";
+import createDebug from "debug";
+const debug = createDebug("automation_model:network");
+class SaveQueue {
+    queue = [];
+    isProcessing = false;
+    maxRetries = 3;
+    async enqueueSave(current) {
+        this.queue.push({ current, retryCount: 0 });
+        if (!this.isProcessing) {
+            await this.processQueue();
+        }
+    }
+    async processQueue() {
+        this.isProcessing = true;
+        while (this.queue.length > 0) {
+            const item = this.queue.shift();
+            try {
+                await this.saveSafely(item.current);
+            }
+            catch (error) {
+                console.error(`Save failed for ${item.current ? "current" : "previous"} step:`, error);
+                // Retry logic
+                if (item.retryCount < this.maxRetries) {
+                    item.retryCount++;
+                    this.queue.unshift(item); // Put it back at the front
+                    await new Promise((resolve) => setTimeout(resolve, 100 * item.retryCount)); // Exponential backoff
+                }
+            }
+        }
+        this.isProcessing = false;
+    }
+    async saveSafely(current) {
+        const stepHash = current ? executionState.currentStepHash : executionState.previousStepHash;
+        if (!stepHash)
+            return;
+        const file = path.join(detailedNetworkFolder, `${stepHash}.json`);
+        const entries = current
+            ? Array.from(executionState.liveRequestsMap.values())
+            : Array.from(executionState.liveRequestsMapPrevious.values());
+        // Ensure all entries are JSON-serializable
+        const validEntries = entries.filter((entry) => {
+            try {
+                JSON.stringify(entry);
+                return true;
+            }
+            catch {
+                console.warn(`Skipping non-serializable entry: ${entry.requestId}`);
+                return false;
+            }
+        });
+        const jsonString = JSON.stringify(validEntries, null, 2);
+        await fs.promises.writeFile(file, jsonString, "utf8");
+    }
+}
 function _getNetworkFile(world = null, web = null, context = null) {
     let networkFile = null;
     if (world && world.reportFolder) {
@@ -50,7 +104,12 @@ function registerNetworkEvents(world, web, context, page) {
     const networkFile = _getNetworkFile(world, web, context);
     function saveNetworkData() {
         if (context && context.networkData) {
-            fs.writeFileSync(networkFile, JSON.stringify(context.networkData, null, 2), "utf8");
+            try {
+                fs.writeFileSync(networkFile, JSON.stringify(context.networkData, null, 2), "utf8");
+            }
+            catch (error) {
+                console.error("Error saving network data:", error);
+            }
         }
     }
     if (!context) {
@@ -65,109 +124,99 @@ function registerNetworkEvents(world, web, context, page) {
             const networkData = context.networkData;
             // Event listener for when a request is made
             page.on("request", (request) => {
-                // console.log("Request started:", request.url());
-                const requestId = requestIdCounter++;
-                request.requestId = requestId; // Assign a unique ID to the request
-                handleRequest(request);
-                const startTime = Date.now();
-                requestTimes.set(requestId, startTime);
-                // Initialize data for this request
-                networkData.push({
-                    requestId,
-                    requestStart: startTime,
-                    requestUrl: request.url(),
-                    method: request.method(),
-                    status: "Pending",
-                    responseTime: null,
-                    responseReceived: null,
-                    responseEnd: null,
-                    size: null,
-                });
-                saveNetworkData();
-            });
-            // Event listener for when a response is received
-            page.on("response", async (response) => {
-                const request = response.request();
-                const requestId = request.requestId;
-                const receivedTime = Date.now();
-                // await handleRequestFinishedOrFailed(request, false);
-                // Find the corresponding data object
-                const data = networkData.find((item) => item.requestId === requestId);
-                if (data) {
-                    data.status = response.status();
-                    data.responseReceived = receivedTime;
+                try {
+                    // console.log("Request started:", request.url());
+                    const requestId = requestIdCounter++;
+                    request.requestId = requestId; // Assign a unique ID to the request
+                    handleRequest(request, context);
+                    const startTime = Date.now();
+                    requestTimes.set(requestId, startTime);
+                    // Initialize data for this request
+                    networkData.push({
+                        requestId,
+                        requestStart: startTime,
+                        requestUrl: request.url(),
+                        method: request.method(),
+                        status: "Pending",
+                        responseTime: null,
+                        responseReceived: null,
+                        responseEnd: null,
+                        size: null,
+                    });
                     saveNetworkData();
                 }
-                else {
-                    // console.error("No data found for request ID", requestId);
+                catch (error) {
+                    // console.error("Error handling request:", error);
                 }
             });
-            // Event listener for when a request is finished
-            page.on("requestfinished", async (request) => {
-                const requestId = request.requestId;
-                const endTime = Date.now();
-                const startTime = requestTimes.get(requestId);
-                await handleRequestFinishedOrFailed(request, false);
-                const response = await request.response();
-                const timing = request.timing();
-                // Find the corresponding data object
-                const data = networkData.find((item) => item.requestId === requestId);
-                if (data) {
-                    data.responseEnd = endTime;
-                    data.responseTime = endTime - startTime;
-                    // Get response size
-                    try {
-                        const body = await response.body();
-                        data.size = body.length;
-                    }
-                    catch (e) {
-                        data.size = 0;
+            // Event listener for when a response is received
+            page.on("response", async (response) => {
+                try {
+                    const request = response.request();
+                    const requestId = request.requestId;
+                    const receivedTime = Date.now();
+                    // await handleRequestFinishedOrFailed(request, false);
+                    // Find the corresponding data object
+                    const data = networkData.find((item) => item.requestId === requestId);
+                    if (data) {
+                        data.status = response.status();
+                        data.responseReceived = receivedTime;
+                        saveNetworkData();
                     }
-                    const type = request.resourceType();
-                    /*
-                    domainLookupStart: 80.655,
-                    domainLookupEnd: 80.668,
-                    connectStart: 80.668,
-                    secureConnectionStart: 106.688,
-                    connectEnd: 129.69,
-                    requestStart: 129.81,
-                    responseStart: 187.006,
-                    responseEnd: 188.209
-                    */
-                    data.type = type;
-                    data.domainLookupStart = timing.domainLookupStart;
-                    data.domainLookupEnd = timing.domainLookupEnd;
-                    data.connectStart = timing.connectStart;
-                    data.secureConnectionStart = timing.secureConnectionStart;
-                    data.connectEnd = timing.connectEnd;
-                    data.requestStart = timing.requestStart;
-                    data.responseStart = timing.responseStart;
-                    data.responseEnd = timing.responseEnd;
-                    saveNetworkData();
-                    if (world && world.attach) {
-                        world.attach(JSON.stringify(data), { mediaType: "application/json+network" });
+                    else {
+                        // console.error("No data found for request ID", requestId);
                     }
                 }
-                else {
-                    // console.error("No data found for request ID", requestId);
+                catch (error) {
+                    // console.error("Error handling response:", error);
                 }
             });
-            // Event listener for when a request fails
-            page.on("requestfailed", async (request) => {
-                const requestId = request.requestId;
-                const endTime = Date.now();
-                const startTime = requestTimes.get(requestId);
-                await handleRequestFinishedOrFailed(request, true);
+            // Event listener for when a request is finished
+            page.on("requestfinished", async (request) => {
                 try {
-                    const res = await request.response();
-                    const statusCode = res ? res.status() : request.failure().errorText;
+                    const requestId = request.requestId;
+                    const endTime = Date.now();
+                    const startTime = requestTimes.get(requestId);
+                    await handleRequestFinishedOrFailed(request, false, context);
+                    const response = await request.response();
+                    const timing = request.timing();
                     // Find the corresponding data object
                     const data = networkData.find((item) => item.requestId === requestId);
                     if (data) {
                         data.responseEnd = endTime;
                         data.responseTime = endTime - startTime;
-                        data.status = statusCode;
-                        data.size = 0;
+                        // Get response size
+                        try {
+                            let size = 0;
+                            if (responseHasBody(response)) {
+                                const buf = await response.body();
+                                size = buf?.length ?? 0;
+                            }
+                            data.size = size;
+                        }
+                        catch {
+                            data.size = 0;
+                        }
+                        const type = request.resourceType();
+                        /*
+                        domainLookupStart: 80.655,
+                        domainLookupEnd: 80.668,
+                        connectStart: 80.668,
+                        secureConnectionStart: 106.688,
+                        connectEnd: 129.69,
+                        requestStart: 129.81,
+                        responseStart: 187.006,
+                        responseEnd: 188.209
+                        */
+                        data.type = type;
+                        data.domainLookupStart = timing.domainLookupStart;
+                        data.domainLookupEnd = timing.domainLookupEnd;
+                        data.connectStart = timing.connectStart;
+                        data.secureConnectionStart = timing.secureConnectionStart;
+                        data.connectEnd = timing.connectEnd;
+                        data.requestStart = timing.requestStart;
+                        data.responseStart = timing.responseStart;
+                        data.responseEnd = timing.responseEnd;
                         saveNetworkData();
                         if (world && world.attach) {
                             world.attach(JSON.stringify(data), { mediaType: "application/json+network" });
@@ -178,7 +227,41 @@ function registerNetworkEvents(world, web, context, page) {
                     }
                 }
                 catch (error) {
-                    // ignore
+                    // console.error("Error handling request finished:", error);
+                }
+            });
+            // Event listener for when a request fails
+            page.on("requestfailed", async (request) => {
+                try {
+                    const requestId = request.requestId;
+                    const endTime = Date.now();
+                    const startTime = requestTimes.get(requestId);
+                    await handleRequestFinishedOrFailed(request, true, context);
+                    try {
+                        const res = await request.response();
+                        const statusCode = res ? res.status() : request.failure().errorText;
+                        // Find the corresponding data object
+                        const data = networkData.find((item) => item.requestId === requestId);
+                        if (data) {
+                            data.responseEnd = endTime;
+                            data.responseTime = endTime - startTime;
+                            data.status = statusCode;
+                            data.size = 0;
+                            saveNetworkData();
+                            if (world && world.attach) {
+                                world.attach(JSON.stringify(data), { mediaType: "application/json+network" });
+                            }
+                        }
+                        else {
+                            // console.error("No data found for request ID", requestId);
+                        }
+                    }
+                    catch (error) {
+                        // ignore
+                    }
+                }
+                catch (error) {
+                    // console.error("Error handling request failed:", error);
                 }
             });
         }
@@ -188,7 +271,11 @@ function registerNetworkEvents(world, web, context, page) {
     }
 }
 async function appendEntryToStepFile(stepHash, entry) {
+    if (!stepHash)
+        return;
+    const debug = createDebug("network:appendEntryToStepFile");
     const file = path.join(detailedNetworkFolder, `${stepHash}.json`);
+    debug("appending to step file:", file);
     let data = [];
     try {
         /* read if it already exists */
@@ -199,16 +286,31 @@ async function appendEntryToStepFile(stepHash, entry) {
         /* ignore – file does not exist or cannot be parsed */
     }
     data.push(entry);
-    await fs.promises.writeFile(file, JSON.stringify(data, null, 2), "utf8");
+    try {
+        debug("writing to step file:", file);
+        await fs.promises.writeFile(file, JSON.stringify(data, null, 2), "utf8");
+    }
+    catch (error) {
+        debug("Error writing to step file:", error);
+    }
 }
 const detailedNetworkFolder = path.join(tmpdir(), "blinq_network_events");
+let outOfStep = true;
+let timeoutId = null;
 const executionState = {
     currentStepHash: null,
+    previousStepHash: null,
     liveRequestsMap: new Map(),
+    liveRequestsMapPrevious: new Map(),
 };
-export function networkBeforeStep(stepName) {
-    const storeDetailedNetworkData = process.env.STORE_DETAILED_NETWORK_DATA === "true";
-    if (!storeDetailedNetworkData) {
+const storeDetailedNetworkData = (context) => context && context.STORE_DETAILED_NETWORK_DATA === true;
+export function networkBeforeStep(stepName, context) {
+    if (timeoutId) {
+        clearTimeout(timeoutId);
+        timeoutId = null;
+    }
+    outOfStep = false;
+    if (!storeDetailedNetworkData(context)) {
         return;
     }
     // check if the folder exists, if not create it
@@ -217,15 +319,10 @@ export function networkBeforeStep(stepName) {
     }
     // const stepHash = stepNameToHash(stepName);
     let stepHash = "";
-    executionState.liveRequestsMap.clear();
-    if (process.env.CURRENT_STEP_ID) {
-        // console.log("Using CURRENT_STEP_ID from environment variables:", process.env.CURRENT_STEP_ID);
-        // If CURRENT_STEP_ID is set, use it as the step hash
-        stepHash = process.env.CURRENT_STEP_ID;
-    }
-    else {
-        stepHash = stepNameToHash(stepName);
-    }
+    executionState.liveRequestsMapPrevious = executionState.liveRequestsMap;
+    executionState.liveRequestsMap = new Map();
+    stepHash = stepNameToHash(stepName);
+    executionState.previousStepHash = executionState.currentStepHash; // ➊ NEW
     executionState.currentStepHash = stepHash;
     // check if the file exists, if exists delete it
     const networkFile = path.join(detailedNetworkFolder, `${stepHash}.json`);
@@ -236,30 +333,35 @@ export function networkBeforeStep(stepName) {
         // Ignore error if file does not exist
     }
 }
-export async function networkAfterStep(stepName) {
-    const storeDetailedNetworkData = process.env.STORE_DETAILED_NETWORK_DATA === "true";
-    if (!storeDetailedNetworkData) {
+const saveQueue = new SaveQueue();
+async function saveMap(current) {
+    await saveQueue.enqueueSave(current);
+}
+export async function networkAfterStep(stepName, context) {
+    if (!storeDetailedNetworkData(context)) {
         return;
     }
-    await new Promise((r) => setTimeout(r, 1000));
-    const entries = Array.from(executionState.liveRequestsMap.values());
-    const file = path.join(detailedNetworkFolder, `${executionState.currentStepHash}.json`);
-    await fs.promises.writeFile(file, JSON.stringify(entries, null, 2), "utf8");
+    //await new Promise((r) => setTimeout(r, 1000));
+    await saveMap(true);
     /* reset for next step */
-    executionState.liveRequestsMap.clear();
-    executionState.previousStepHash = executionState.currentStepHash; // ➋ NEW
-    executionState.currentStepHash = null;
+    //executionState.previousStepHash = executionState.currentStepHash; // ➋ NEW
+    //executionState.liveRequestsMap.clear();
+    outOfStep = true;
+    // set a timer of 60 seconds to the outOfStep, after that it will be set to false so no network collection will happen
+    timeoutId = setTimeout(() => {
+        outOfStep = false;
+        context.STORE_DETAILED_NETWORK_DATA = false;
+    }, 60000);
 }
 function stepNameToHash(stepName) {
     const templateName = _stepNameToTemplate(stepName);
     // create hash from the template name
     return crypto.createHash("sha256").update(templateName).digest("hex");
 }
-function handleRequest(request) {
-    const storeDetailedNetworkData = process.env.STORE_DETAILED_NETWORK_DATA === "true";
-    if (!storeDetailedNetworkData || !executionState.currentStepHash) {
+function handleRequest(request, context) {
+    const debug = createDebug("automation_model:network:handleRequest");
+    if (!storeDetailedNetworkData(context))
         return;
-    }
     const entry = {
         requestId: request.requestId,
         url: request.url(),
@@ -270,48 +372,33 @@ function handleRequest(request) {
         stepHash: executionState.currentStepHash,
     };
     executionState.liveRequestsMap.set(request, entry);
-    // console.log("Request started:", entry);
+    debug("Request to", request.url(), "with", request.requestId, "added to current step map at", Date.now());
 }
-function saveNetworkDataToFile(requestData) {
-    const storeDetailedNetworkData = process.env.STORE_DETAILED_NETWORK_DATA === "true";
-    if (!storeDetailedNetworkData) {
+async function handleRequestFinishedOrFailed(request, failed, context) {
+    const debug = createDebug("automation_model:network:handleRequestFinishedOrFailed");
+    if (!storeDetailedNetworkData(context))
         return;
-    }
-    const networkFile = path.join(detailedNetworkFolder, `${requestData.stepHash}.json`);
-    // read the existing data if it exists (should be an array)
-    let existingData = [];
-    if (fs.existsSync(networkFile)) {
-        const data = fs.readFileSync(networkFile, "utf8");
-        try {
-            existingData = JSON.parse(data);
-        }
-        catch (e) {
-            // console.error("Failed to parse existing network data:", e);
-        }
-    }
-    // Add the live requests to the existing data
-    existingData.push(requestData);
-    // Save the updated data back to the file
-    // console.log("Saving network data to file:", networkFile);
-    fs.writeFileSync(networkFile, JSON.stringify(existingData, null, 2), "utf8");
-}
-async function handleRequestFinishedOrFailed(request, failed) {
-    const storeDetailedNetworkData = process.env.STORE_DETAILED_NETWORK_DATA === "true";
-    if (!storeDetailedNetworkData) {
-        return;
-    }
+    const requestId = request.requestId;
+    debug("Request id in handleRequestFinishedOrFailed:", requestId, "at", Date.now());
     // const response = await request.response(); // This may be null if the request failed
     let entry = executionState.liveRequestsMap.get(request);
+    debug("Request entry found in current map:", entry?.requestId || false);
     if (!entry) {
-        entry = {
-            requestId: request.requestId,
-            url: request.url(),
-            method: request.method?.() ?? "GET",
-            headers: request.headers?.() ?? {},
-            postData: request.postData?.() ?? undefined,
-            stepHash: executionState.previousStepHash ?? "unknown",
-            requestTimestamp: Date.now(),
-        };
+        // check if the request is in the previous step's map
+        entry = executionState.liveRequestsMapPrevious.get(request);
+        debug("Request entry found in previous map:", entry?.requestId || false);
+        if (!entry) {
+            debug("No entry, creating fallback! for url:", request.url());
+            entry = {
+                requestId: request.requestId,
+                url: request.url(),
+                method: request.method?.() ?? "GET",
+                headers: request.headers?.() ?? {},
+                postData: request.postData?.() ?? undefined,
+                stepHash: executionState.previousStepHash ?? "unknown",
+                requestTimestamp: Date.now(),
+            };
+        }
     }
     // Remove the request from the live requests map
     let respData;
@@ -330,28 +417,35 @@ async function handleRequestFinishedOrFailed(request, failed) {
     }
     else {
         const response = await request.response();
-        // console.log("Response received for request:", request.url());
-        const headers = response?.headers() || {};
-        const contentType = headers["content-type"] || null;
+        const headers = response?.headers?.() || {};
+        let contentType = headers["content-type"] || null;
         let body = null;
         try {
-            if (contentType && contentType.includes("application/json")) {
-                const text = await response.text();
-                body = JSON.parse(text);
-                // console.log("Parsed JSON response body:", body);
-            }
-            else if (contentType && contentType.includes("text")) {
-                body = await response.text();
-                // console.log("Parsed text response body:", body);
+            if (responseHasBody(response)) {
+                if (contentType && contentType.includes("application/json")) {
+                    body = JSON.parse(await response.text());
+                }
+                else if ((contentType && contentType.includes("text")) ||
+                    (contentType && contentType.includes("application/csv"))) {
+                    body = await response.text();
+                    if (contentType.includes("application/csv"))
+                        contentType = "text/csv";
+                }
+                else {
+                    // If you want binary, you could read it here—but only when responseHasBody(response) is true
+                    // const buffer = await response.body();
+                    // body = buffer.toString("base64");
+                }
             }
             else {
-                // Optionally handle binary here
-                // const buffer = await response.body();
-                // body = buffer.toString("base64"); // if you want to store binary safely
+                // For redirects / no-body statuses, it's useful to keep redirect info
+                // e.g., include Location header if present
+                // body stays null
             }
         }
         catch (err) {
             console.error("Error reading response body:", err);
+            body = await response.text();
         }
         respData = {
             status: response.status(),
@@ -365,42 +459,36 @@ async function handleRequestFinishedOrFailed(request, failed) {
     if (executionState.liveRequestsMap.has(request)) {
         /* “normal” path – keep it in the buffer */
         entry.response = respData;
+        if (outOfStep && executionState.currentStepHash) {
+            await saveMap(true);
+        }
     }
     else {
-        /* orphan response – append directly to the previous step file */
-        entry.response = respData;
-        await appendEntryToStepFile(entry.stepHash, entry); // ➍ NEW
+        if (executionState.liveRequestsMapPrevious.has(request)) {
+            entry.response = respData;
+            await saveMap(false);
+        }
+        else {
+            /* orphan response – append directly to the previous step file */
+            entry.response = respData;
+            await appendEntryToStepFile(entry.stepHash, entry); // ➍ NEW
+        }
     }
     entry.response = respData;
     return;
 }
-// Handle successful request with a response
-// const headers = response.headers();
-// const contentType = headers["content-type"] || "";
-// let body = null;
-// try {
-//   if (contentType.includes("application/json")) {
-//     const text = await response.text();
-//     body = JSON.parse(text);
-//   } else if (contentType.includes("text")) {
-//     body = await response.text();
-//   } else {
-//     // Optionally handle binary here
-//     // const buffer = await response.body();
-//     // body = buffer.toString("base64"); // if you want to store binary safely
-//   }
-// } catch (err) {
-//   console.error("Error reading response body:", err);
-// }
-//   requestData.response = {
-//     status: response.status(),
-//     headers,
-//     url: response.url(),
-//     timestamp: Date.now(),
-//     body,
-//     contentType,
-//   };
-//   saveNetworkDataToFile(requestData);
-// }
+function responseHasBody(response) {
+    if (!response)
+        return false;
+    const s = response.status?.() ?? 0;
+    // RFC 7231: 1xx, 204, 205, 304 have no body. Playwright: 3xx (redirect) body is unavailable.
+    if ((s >= 100 && s < 200) || s === 204 || s === 205 || s === 304 || (s >= 300 && s < 400))
+        return false;
+    // HEAD responses have no body by definition
+    const method = response.request?.().method?.() ?? "GET";
+    if (method === "HEAD")
+        return false;
+    return true;
+}
 export { registerNetworkEvents, registerDownloadEvent };
 //# sourceMappingURL=network.js.map