@pipelab/core-node 1.0.1-beta.25 → 1.0.1-beta.27

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipelab/core-node",
-  "version": "1.0.1-beta.25",
+  "version": "1.0.1-beta.27",
   "private": false,
   "description": "The Pipelab automation engine for Node.js",
   "license": "FSL-1.1-MIT",
@@ -28,6 +28,7 @@
   "dependencies": {
     "adm-zip": "0.5.16",
     "archiver": "7.0.1",
+    "check-disk-space": "3.4.0",
     "es-toolkit": "1.25.2",
     "execa": "9.5.2",
     "nanoid": "5.0.7",
@@ -38,8 +39,8 @@
     "type-fest": "4.39.0",
     "ws": "8.18.3",
     "yauzl": "2.10.0",
-    "@pipelab/constants": "1.0.1-beta.21",
-    "@pipelab/shared": "2.0.1-beta.22"
+    "@pipelab/constants": "1.0.1-beta.23",
+    "@pipelab/shared": "2.0.1-beta.24"
   },
   "devDependencies": {
     "@types/adm-zip": "0.5.7",
@@ -52,7 +53,7 @@
     "@types/yauzl": "2.10.3",
     "tsdown": "0.21.2",
     "typescript": "^5.0.0",
-    "@pipelab/tsconfig": "1.0.1-beta.21"
+    "@pipelab/tsconfig": "1.0.1-beta.23"
   },
   "scripts": {
     "build": "tsdown",

package/src/handlers/build-history.ts

@@ -2,9 +2,10 @@ import { PipelabContext } from "../context";
 import { join } from "node:path";
 import { writeFile, readFile, unlink, mkdir, stat, readdir } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { BuildHistoryEntry, IBuildHistoryStorage, AppConfig } from "@pipelab/shared";
-import { useLogger } from "@pipelab/shared";
+import { useLogger, BuildHistoryEntry, IBuildHistoryStorage, AppConfig } from "@pipelab/shared";
 import { setupConfigFile } from "../config";
+import checkDiskSpace from "check-disk-space";
+import { getFolderSize } from "../utils/fs-extras";
 
 // Simplified storage - one file per pipeline containing array of build entries
 
@@ -128,6 +129,11 @@ export class BuildHistoryStorage implements IBuildHistoryStorage {
       this.logger
         .logger()
         .info(`Saved build history entry: ${entry.id} for pipeline: ${entry.pipelineId}`);
+
+      // Apply retention policy after each save
+      this.applyRetentionPolicy().catch((err) => {
+        this.logger.logger().error("Failed to apply retention policy after save:", err);
+      });
     } catch (error) {
       this.logger.logger().error("Failed to save build history entry:", error);
       throw new Error(`Failed to save build history entry: ${error}`);
@@ -280,24 +286,31 @@ export class BuildHistoryStorage implements IBuildHistoryStorage {
     oldestEntry?: number;
     newestEntry?: number;
     numberOfPipelines: number;
-    cachePath: string;
     userDataPath: string;
     retentionPolicy: {
       enabled: boolean;
       maxEntries: number;
       maxAge: number;
     };
+    disk: {
+      total: number;
+      free: number;
+      pipelab: number;
+    };
   }> {
     try {
       const allEntries = await this.getAll();
       const files = await this.getAllPipelineFiles();
 
+      const diskSpace = await checkDiskSpace(this.context.userDataPath);
+      const pipelabSize = await getFolderSize(this.context.userDataPath);
+
       const settings = await setupConfigFile<AppConfig>("settings", { context: this.context });
       const config = await settings.getConfig();
       const policy = config?.buildHistory?.retentionPolicy || {
         enabled: false,
-        maxEntries: 0,
-        maxAge: 0,
+        maxEntries: 50,
+        maxAge: 30,
       };
 
       if (allEntries.length === 0) {
@@ -305,13 +318,17 @@ export class BuildHistoryStorage implements IBuildHistoryStorage {
           totalEntries: 0,
           totalSize: 0,
           numberOfPipelines: files.length,
-          cachePath: config?.cacheFolder || tmpdir(),
           userDataPath: this.context.userDataPath,
           retentionPolicy: {
             enabled: policy.enabled,
             maxEntries: policy.maxEntries,
             maxAge: policy.maxAge,
           },
+          disk: {
+            total: diskSpace.size,
+            free: diskSpace.free,
+            pipelab: pipelabSize,
+          },
         };
       }
 
@@ -334,13 +351,17 @@ export class BuildHistoryStorage implements IBuildHistoryStorage {
         oldestEntry: sortedEntries[0]?.createdAt,
         newestEntry: sortedEntries[sortedEntries.length - 1]?.createdAt,
         numberOfPipelines: files.length,
-        cachePath: config?.cacheFolder || tmpdir(),
         userDataPath: this.context.userDataPath,
         retentionPolicy: {
           enabled: policy.enabled,
           maxEntries: policy.maxEntries,
           maxAge: policy.maxAge,
         },
+        disk: {
+          total: diskSpace.size,
+          free: diskSpace.free,
+          pipelab: pipelabSize,
+        },
       };
     } catch (error) {
       this.logger.logger().error("Failed to get storage info:", error);

package/src/handlers/history.ts

@@ -226,6 +226,36 @@ export const registerHistoryHandlers = (context: PipelabContext) => {
     }
   });
 
+  handle("build-history:clear-by-pipeline", async (event, { send, value }) => {
+    try {
+      await checkBuildHistoryAuthorization(event);
+
+      await buildHistoryStorage.clearByPipeline(value.pipelineId);
+      send({
+        type: "end",
+        data: { type: "success", result: { result: "ok" } },
+      });
+    } catch (error) {
+      logger().error(`Failed to clear build history for pipeline ${value.pipelineId}:`, error);
+
+      if (error instanceof SubscriptionRequiredError) {
+        send({
+          type: "end",
+          data: { type: "error", ipcError: error.userMessage, code: error.code },
+        });
+        return;
+      }
+
+      send({
+        type: "end",
+        data: {
+          type: "error",
+          ipcError: error instanceof Error ? error.message : "Failed to clear build history for pipeline",
+        },
+      });
+    }
+  });
+
   handle("build-history:get-storage-info", async (event, { send }) => {
     try {
       await checkBuildHistoryAuthorization(event);

package/src/runner.ts

@@ -69,7 +69,8 @@ export async function runPipelineCommand(file: string, options: RunOptions, vers
 
   const settings = await setupConfigFile<AppConfig>("settings", { context });
   const config = await settings.getConfig();
-  const cachePath = config?.cacheFolder || tmpdir();
+  const cachePath = join(context.userDataPath, "cache");
+  await mkdir(cachePath, { recursive: true });
 
   const abortController = new AbortController();
 

package/src/utils/fs-extras.ts

@@ -1,6 +1,6 @@
 import { mkdir, createWriteStream, createReadStream } from "node:fs";
 import { execa, Options, Subprocess } from "execa";
-import { mkdir as mkdirP, access, writeFile, realpath, mkdtemp, chmod } from "node:fs/promises";
+import { mkdir as mkdirP, access, writeFile, realpath, mkdtemp, chmod, stat, readdir } from "node:fs/promises";
 import { join, dirname } from "node:path";
 import { tmpdir } from "node:os";
 import tar from "tar";
@@ -180,3 +180,32 @@ export const runWithLiveLogs = async (
     throw new Error(`Command failed with exit code ${code}: ${error.message}`);
   }
 };
+
+/**
+ * Calculates the total size of a directory recursively.
+ */
+export async function getFolderSize(dirPath: string): Promise<number> {
+  try {
+    const files = await readdir(dirPath, { withFileTypes: true });
+    const ArrayOfPromises = files.map(async (file) => {
+      const path = join(dirPath, file.name);
+      if (file.isDirectory()) {
+        try {
+          return await getFolderSize(path);
+        } catch {
+          return 0;
+        }
+      }
+      try {
+        const { size } = await stat(path);
+        return size;
+      } catch {
+        return 0;
+      }
+    });
+    const results = await Promise.all(ArrayOfPromises);
+    return results.reduce((acc, size) => acc + size, 0);
+  } catch {
+    return 0;
+  }
+}

package/src/utils.ts

@@ -125,6 +125,9 @@ export const executeGraphWithHistory = async ({
   const logs: any[] = [];
   const sandboxPath = await generateTempFolder(cachePath);
   const { logger } = useLogger();
+  let completedSteps = 0;
+  let failedSteps = 0;
+  let cancelledSteps = 0;
 
   // Ensure all plugins used in the graph are downloaded and registered
   const { registerPlugins, plugins: registeredPlugins } = usePlugins();
@@ -156,7 +159,7 @@ export const executeGraphWithHistory = async ({
   logger().info(`[Sandbox] Execution sandbox created at: ${sandboxPath}`);
   const settingsFile = await setupConfigFile<AppConfig>("settings", { context: ctx });
   const config = await settingsFile.getConfig();
-  const shouldCleanup = config?.clearTemporaryFoldersOnPipelineEnd ?? true;
+  const shouldCleanup = true;
 
   try {
     const result = await processGraph({
@@ -175,7 +178,8 @@ export const executeGraphWithHistory = async ({
             async (data) => {
               if (data.type === "log") {
                 const logEntry = {
-                  type: data.type,
+                  id: nanoid(),
+                  level: "info",
                   message: data.data.message,
                   timestamp: data.data.time,
                   nodeUid: node?.uid,
@@ -195,7 +199,14 @@ export const executeGraphWithHistory = async ({
       onNodeEnter: (node) => {
         onNodeEnter?.(node);
       },
-      onNodeExit: (node) => {
+      onNodeExit: async (node) => {
+        completedSteps++;
+        if (!shouldDisableHistory) {
+          // Update progress in the background
+          buildHistoryStorage.update(buildId, { completedSteps }, pipelineId).catch((err) => {
+            logger().error(`Failed to update progress for build ${buildId}:`, err);
+          });
+        }
         onNodeExit?.(node);
       },
       abortSignal,
@@ -211,6 +222,7 @@ export const executeGraphWithHistory = async ({
           duration: endTime - startTime,
           output: result.steps,
           logs,
+          completedSteps,
         },
         pipelineId,
       );
@@ -234,6 +246,9 @@ export const executeGraphWithHistory = async ({
             timestamp: endTime,
           },
           logs,
+          completedSteps,
+          failedSteps: isCanceled ? 0 : 1,
+          cancelledSteps: isCanceled ? 1 : 0,
         },
         pipelineId,
       );