@malloy-publisher/server 0.0.198-dev2 → 0.0.198-dev3

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.198-dev2",
+   "version": "0.0.198-dev3",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"
@@ -51,7 +51,6 @@
       "@opentelemetry/sdk-metrics": "^2.0.0",
       "@opentelemetry/sdk-node": "^0.200.0",
       "@opentelemetry/sdk-trace-node": "^2.0.0",
-      "adm-zip": "^0.5.16",
       "async-mutex": "^0.5.0",
       "aws-sdk": "^2.1692.0",
       "body-parser": "^1.20.2",
@@ -61,6 +60,7 @@
       "cors": "^2.8.5",
       "duckdb": "1.4.4",
       "express": "^4.21.0",
+      "extract-zip": "^2.0.1",
       "globals": "^15.9.0",
       "handlebars": "^4.7.8",
       "http-proxy-middleware": "^3.0.5",
@@ -76,7 +76,6 @@
       "@eslint/eslintrc": "^3.3.1",
       "@eslint/js": "^9.23.0",
       "@faker-js/faker": "^9.4.0",
-      "@types/adm-zip": "^0.5.7",
       "@types/bun": "^1.2.20",
       "@types/cors": "^2.8.12",
       "@types/express": "^4.17.14",

package/src/service/environment_store.ts

@@ -1,8 +1,8 @@
 import { GetObjectCommand, S3 } from "@aws-sdk/client-s3";
 import { Storage } from "@google-cloud/storage";
-import AdmZip from "adm-zip";
 import { Mutex } from "async-mutex";
 import crypto from "crypto";
+import extract from "extract-zip";
 import * as fs from "fs";
 import * as path from "path";
 import simpleGit from "simple-git";
@@ -893,6 +893,7 @@ export class EnvironmentStore {
    }
 
    public async unzipEnvironment(absoluteEnvironmentPath: string) {
+      const startedAt = Date.now();
       logger.info(
          `Detected zip file at "${absoluteEnvironmentPath}". Unzipping...`,
       );
@@ -906,8 +907,28 @@ export class EnvironmentStore {
       });
       await fs.promises.mkdir(unzippedEnvironmentPath, { recursive: true });
 
-      const zip = new AdmZip(absoluteEnvironmentPath);
-      zip.extractAllTo(unzippedEnvironmentPath, true);
+      // Stream-extract via yauzl (wrapped by extract-zip). Each entry's
+      // inflate and write are dispatched to the libuv thread pool, so the
+      // main event loop stays responsive even for very large archives.
+      // The previous adm-zip path used fs.readFileSync + zlib.inflateRawSync
+      // on the main thread, which parked the loop long enough on multi-
+      // hundred-MB packages to fail Kubernetes liveness probes mid-extract.
+      let entryCount = 0;
+      let totalUncompressedBytes = 0;
+      await extract(absoluteEnvironmentPath, {
+         dir: path.resolve(unzippedEnvironmentPath),
+         onEntry: (entry) => {
+            entryCount += 1;
+            totalUncompressedBytes += entry.uncompressedSize ?? 0;
+         },
+      });
+
+      const mib = (totalUncompressedBytes / (1024 * 1024)).toFixed(1);
+      logger.info(
+         `Unzipped "${absoluteEnvironmentPath}" -> "${unzippedEnvironmentPath}" ` +
+            `(${entryCount} entries, ${mib} MiB uncompressed) in ` +
+            `${formatDuration(Date.now() - startedAt)}`,
+      );
 
       return unzippedEnvironmentPath;
    }

package/src/service/schema_worker_pool.ts

@@ -184,11 +184,14 @@ export class SchemaWorkerPool {
          },
       );
 
+      // Lifecycle: `error` fires first (if it fires) and reports the
+      // crash; `exit` always fires next and is the single point where
+      // we replace the slot. Splitting it this way avoids a class of
+      // bugs where `error` respawns the slot and then `exit` respawns
+      // it again, leaking the worker created in between (alive Worker
+      // instance with a DuckDB connection — exactly what this pool
+      // exists to prevent).
       worker.on("error", (err) => {
-         // A native crash inside the worker — fail the in-flight request
-         // attributed to this slot, then respawn the worker so the pool
-         // self-heals. Without respawn, one crash silently shrinks
-         // capacity and concurrent loads would queue forever.
          const inFlightId = this.workerCurrentId.get(index);
          if (inFlightId !== undefined) {
             const req = this.inFlight.get(inFlightId);
@@ -198,39 +201,45 @@ export class SchemaWorkerPool {
             }
             this.workerCurrentId.delete(index);
          }
-         logger.error("SchemaWorkerPool: worker errored, respawning", {
+         logger.error("SchemaWorkerPool: worker errored", {
             workerIndex: index,
             error: err,
          });
-         if (!this.stopped) {
-            this.workers[index] = this.spawn(index);
-            this.drain();
-         }
+         // Don't respawn here — `exit` will, after the worker has
+         // fully torn down its native resources.
       });
 
       worker.on("exit", (code) => {
          if (this.stopped) return;
+         // If `error` already fired, workerCurrentId is empty and this
+         // is a no-op. If the worker exited without firing `error`
+         // (e.g. process.exit inside the worker, or a clean exit while
+         // mid-request), reject any in-flight request so the caller
+         // doesn't hang forever.
+         const inFlightId = this.workerCurrentId.get(index);
+         if (inFlightId !== undefined) {
+            const req = this.inFlight.get(inFlightId);
+            if (req) {
+               this.inFlight.delete(inFlightId);
+               req.reject(new Error(`SchemaWorker exited with code ${code}`));
+            }
+            this.workerCurrentId.delete(index);
+         }
          if (code !== 0) {
             logger.warn("SchemaWorkerPool: worker exited unexpectedly", {
                workerIndex: index,
                code,
             });
-            // Treat unexpected exit like an error: respawn so the pool
-            // doesn't silently lose capacity.
-            const inFlightId = this.workerCurrentId.get(index);
-            if (inFlightId !== undefined) {
-               const req = this.inFlight.get(inFlightId);
-               if (req) {
-                  this.inFlight.delete(inFlightId);
-                  req.reject(
-                     new Error(`SchemaWorker exited with code ${code}`),
-                  );
-               }
-               this.workerCurrentId.delete(index);
-            }
-            this.workers[index] = this.spawn(index);
-            this.drain();
+         } else {
+            // A clean exit while the pool is still running is also
+            // unexpected — workers are supposed to live as long as
+            // the pool does. Respawn so capacity isn't silently lost.
+            logger.info("SchemaWorkerPool: worker exited cleanly, respawning", {
+               workerIndex: index,
+            });
          }
+         this.workers[index] = this.spawn(index);
+         this.drain();
       });
 
       return slot;