@underpostnet/cyberia 3.3.73 → 3.3.76

package/src/server/ops/logger.js

@@ -48,6 +48,13 @@ const SENSITIVE_WORDS = new Set([
   'token',
 ]);
 const SENSITIVE_COMPOUNDS = ['apikey', 'privatekey', 'accesskey', 'sessionid', 'mysqlpwd', 'mariadbpwd'];
+/**
+ * A value adjacent to a sensitive name, separated by nothing but whitespace, is only treated as
+ * the secret when it is shaped like one: long enough, and carrying a digit, a capital, or a
+ * separator. Prose reads the same way — `token verification failed` — and redacting the next
+ * word of every sentence would cost more than it protects.
+ */
+const CREDENTIAL_SHAPED = /^(?=.{12,}$)(?=.*[0-9A-Z_-])[A-Za-z0-9_-]+$/;
 const sensitiveField = (key) => {
   const words = `${key || ''}`
     .replace(/([a-z0-9])([A-Z])/g, '$1_$2')
@@ -61,43 +68,71 @@ const sensitiveField = (key) => {
   );
 };
 
+/**
+ * Redacts a value that follows a sensitive name with nothing but whitespace between them.
+ *
+ * Service errors report credentials this way — `Invalid api_key 4778…` — so the pair reaches a
+ * log through a message rather than through a command or a structured field, which is what every
+ * other rule matches on. The candidate is looked ahead rather than consumed: a plain match would
+ * take `Invalid api_key` as its own name/value pair and advance past the real one behind it.
+ * @param {string} text - Text to scan.
+ * @returns {string} The text with adjacent credentials redacted.
+ * @memberof Logger
+ */
+const redactAdjacentCredentials = (text) => {
+  const pairs = new RegExp(`(${CONFIG_NAME_SOURCE})([ \\t]+)(?=([A-Za-z0-9_-]+)(?![\\w-]))`, 'g');
+  let out = '';
+  let cursor = 0;
+  for (const match of text.matchAll(pairs)) {
+    const [, key, separator, candidate] = match;
+    const valueStart = match.index + key.length + separator.length;
+    if (valueStart < cursor) continue;
+    if (!sensitiveField(key) || !CREDENTIAL_SHAPED.test(candidate)) continue;
+    out += text.slice(cursor, valueStart) + REDACTED;
+    cursor = valueStart + candidate.length;
+  }
+  return cursor === 0 ? text : out + text.slice(cursor);
+};
+
 const redactSensitiveText = (value = '') =>
-  `${value ?? ''}`
-    .replace(/-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY-----/g, REDACTED)
-    .replace(
-      /\b(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)\b/g,
-      REDACTED,
-    )
-    .replace(/([a-z][a-z0-9+.-]*\\?:(?:\\?\/){2})(?:(?:\\.)|[^/\\\s@])+(\\?@)/gi, '$1***$2')
-    .replace(/\b(x-access-token\\?:)[^\s"']+?(?=\\?@github\.com\b)/gi, '$1***')
-    .replace(
-      /((?:"|')?(?:authorization|proxy-authorization)(?:"|')?\s*:\s*)(?:"[^"]*"|'[^']*'|(?:bearer|basic)\s+\S+|\S+)/gi,
-      `$1"${REDACTED}"`,
-    )
-    .replace(
-      new RegExp(
-        `(\\b(?:host|app|state|secret)\\s+set\\s+)(${CONFIG_NAME_SOURCE})(\\s+)(${SENSITIVE_VALUE_SOURCE})`,
-        'gi',
+  redactAdjacentCredentials(
+    `${value ?? ''}`
+      .replace(/-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY-----/g, REDACTED)
+      .replace(
+        /\b(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)\b/g,
+        REDACTED,
+      )
+      .replace(/([a-z][a-z0-9+.-]*\\?:(?:\\?\/){2})(?:(?:\\.)|[^/\\\s@])+(\\?@)/gi, '$1***$2')
+      .replace(/\b(x-access-token\\?:)[^\s"']+?(?=\\?@github\.com\b)/gi, '$1***')
+      .replace(
+        /((?:"|')?(?:authorization|proxy-authorization)(?:"|')?\s*:\s*)(?:"[^"]*"|'[^']*'|(?:bearer|basic)\s+\S+|\S+)/gi,
+        `$1"${REDACTED}"`,
+      )
+      .replace(
+        new RegExp(
+          `(\\b(?:host|app|state|secret)\\s+set\\s+)(${CONFIG_NAME_SOURCE})(\\s+)(${SENSITIVE_VALUE_SOURCE})`,
+          'gi',
+        ),
+        (match, prefix, key, separator) => (sensitiveField(key) ? `${prefix}${key}${separator}${REDACTED}` : match),
+      )
+      .replace(
+        new RegExp(`(\\s--?)(${CONFIG_NAME_SOURCE})(=|\\s+)(${SENSITIVE_VALUE_SOURCE})`, 'gi'),
+        (match, prefix, key, separator) => (sensitiveField(key) ? `${prefix}${key}${separator}${REDACTED}` : match),
+      )
+      .replace(
+        new RegExp(`(^|\\s)(-p)(\\s*)(${SENSITIVE_VALUE_SOURCE})`, 'gi'),
+        (match, prefix, flag, separator, secret) =>
+          secret.startsWith('-') ? match : `${prefix}${flag}${separator}${REDACTED}`,
+      )
+      .replace(
+        new RegExp(`\\b(${CONFIG_NAME_SOURCE})(\\s*=\\s*)(${SENSITIVE_VALUE_SOURCE})`, 'gi'),
+        (match, key, separator) => (sensitiveField(key) ? `${key}${separator}${REDACTED}` : match),
+      )
+      .replace(
+        new RegExp(`((?:"|')?)(${CONFIG_NAME_SOURCE})((?:"|')?\\s*:\\s*)(${SENSITIVE_VALUE_SOURCE})`, 'gi'),
+        (match, quote, key, separator) => (sensitiveField(key) ? `${quote}${key}${separator}"${REDACTED}"` : match),
       ),
-      (match, prefix, key, separator) => (sensitiveField(key) ? `${prefix}${key}${separator}${REDACTED}` : match),
-    )
-    .replace(
-      new RegExp(`(\\s--?)(${CONFIG_NAME_SOURCE})(=|\\s+)(${SENSITIVE_VALUE_SOURCE})`, 'gi'),
-      (match, prefix, key, separator) => (sensitiveField(key) ? `${prefix}${key}${separator}${REDACTED}` : match),
-    )
-    .replace(
-      new RegExp(`(^|\\s)(-p)(\\s*)(${SENSITIVE_VALUE_SOURCE})`, 'gi'),
-      (match, prefix, flag, separator, secret) =>
-        secret.startsWith('-') ? match : `${prefix}${flag}${separator}${REDACTED}`,
-    )
-    .replace(
-      new RegExp(`\\b(${CONFIG_NAME_SOURCE})(\\s*=\\s*)(${SENSITIVE_VALUE_SOURCE})`, 'gi'),
-      (match, key, separator) => (sensitiveField(key) ? `${key}${separator}${REDACTED}` : match),
-    )
-    .replace(
-      new RegExp(`((?:"|')?)(${CONFIG_NAME_SOURCE})((?:"|')?\\s*:\\s*)(${SENSITIVE_VALUE_SOURCE})`, 'gi'),
-      (match, quote, key, separator) => (sensitiveField(key) ? `${quote}${key}${separator}"${REDACTED}"` : match),
-    );
+  );
 
 const serializeLogValue = (value) => {
   const seen = new WeakSet();

package/src/server/runtime/conf.js

@@ -2986,8 +2986,10 @@ const syncPrivateConf = (deployId, extraPaths = []) => {
   if (fs.existsSync(replicaSrcDir))
     for (const entry of fs.readdirSync(replicaSrcDir))
       if (entry.match(deployId)) fs.copySync(`${replicaSrcDir}/${entry}`, `${privateRepoPath}/replica/${entry}`);
-
-  for (const extraPath of extraPaths) fs.copySync(`./engine-private/${extraPath}`, `${privateRepoPath}/${extraPath}`);
+  for (const extraPath of extraPaths) {
+    fs.removeSync(`${privateRepoPath}/${extraPath}`);
+    fs.copySync(`./engine-private/${extraPath}`, `${privateRepoPath}/${extraPath}`);
+  }
 
   shellExec(
     `cd ${privateRepoPath}` +

package/src/server/storage/data-query.js

@@ -15,6 +15,22 @@
  * @memberof DataQuery
  */
 class DataQuery {
+  /**
+   * Filter that addresses one document by its natural key.
+   *
+   * The natural key is the value people navigate by: an object layer item id, an
+   * instance code. A key shaped like a document id matches either field, so a
+   * document id still resolves.
+   *
+   * @param {string} field - Path of the natural key, e.g. 'data.item.id'.
+   * @param {string} key - The key taken from the request.
+   * @memberof DataQuery
+   * @returns {Object} A Mongoose filter selecting at most one document.
+   */
+  static naturalKeyFilter(field, key) {
+    return /^[0-9a-f]{24}$/i.test(String(key)) ? { $or: [{ [field]: key }, { _id: key }] } : { [field]: key };
+  }
+
   /**
    * Parse request query parameters into Mongoose query options
    * @param {Object} params - The request query parameters (req.query)

package/src/server/storage/downloader.js

@@ -31,31 +31,79 @@ class Downloader {
    * @memberof Downloader
    */
   static downloadFile(url, fullPath, options = { method: 'get', responseType: 'stream' }) {
-    return new Promise((resolve, reject) =>
+    /**
+     * Renders an error response body, whatever transport shape it arrived in, capped for a log
+     * line. A stream request carries even its errors as a stream, so the body has to be read
+     * before it can be reported.
+     */
+    const responseBody = async (data) => {
+      if (!data) return undefined;
+      try {
+        if (Buffer.isBuffer(data)) return data.toString('utf8').slice(0, 500);
+        if (typeof data === 'string') return data.slice(0, 500);
+        if (typeof data.on === 'function') {
+          let text = '';
+          for await (const chunk of data) {
+            text += chunk;
+            if (text.length > 500) break;
+          }
+          return text.slice(0, 500) || undefined;
+        }
+        return JSON.stringify(data).slice(0, 500);
+      } catch {
+        return undefined;
+      }
+    };
+
+    return new Promise((resolve, reject) => {
+      let settled = false;
+      const fail = (error) => {
+        if (settled) return;
+        settled = true;
+        logger.error('Error downloading the file', { fullPath, error: error?.message });
+        // A partial file is worse than none: the caller cannot tell it apart from a whole one.
+        if (fs.existsSync(fullPath)) fs.unlinkSync(fullPath);
+        return reject(error);
+      };
+
       axios({
         url,
         ...options,
       })
         .then((response) => {
-          // Create a write stream to save the file to the specified path
           const writer = fs.createWriteStream(fullPath);
-          response.data.pipe(writer);
+          const expectedBytes = Number(response.headers?.['content-length'] ?? NaN);
+
+          // A response stream that ends early still ends, so the writer emits `finish` and the
+          // download reports success over a truncated file. The declared length is what proves
+          // the transfer whole — without this check a short part reached the caller intact-looking
+          // and only surfaced much later, as a corrupt archive assembled from it.
           writer.on('finish', () => {
-            logger.info('Download completet');
+            if (settled) return;
+            const receivedBytes = writer.bytesWritten;
+            if (Number.isFinite(expectedBytes) && receivedBytes !== expectedBytes)
+              return fail(new Error(`Truncated download: expected ${expectedBytes} bytes, received ${receivedBytes}`));
+            settled = true;
+            logger.info('Download complete', { fullPath, bytes: receivedBytes });
             return resolve(fullPath);
           });
-          writer.on('error', (error) => {
-            logger.error('Error downloading the file');
-            // Cleanup incomplete file if possible
-            if (fs.existsSync(fullPath)) fs.unlinkSync(fullPath);
-            return reject(error);
-          });
+          writer.on('error', fail);
+          response.data.on('error', fail);
+          response.data.pipe(writer);
         })
-        .catch((error) => {
-          logger.error('Error in the request');
+        .catch(async (error) => {
+          // The status alone names nothing. A storage service explains a refusal in the body —
+          // which delivery type it looked under, which id it could not find — and without it a
+          // caller sees `status code 400` and has to reproduce the request by hand to learn why.
+          logger.error('Error in the request', {
+            fullPath,
+            error: error?.message,
+            status: error?.response?.status,
+            body: await responseBody(error?.response?.data),
+          });
           return reject(error);
-        }),
-    );
+        });
+    });
   }
 }
 

package/src/server/storage/zip.js

@@ -0,0 +1,153 @@
+/**
+ * ZIP archive helpers built on JSZip, shared by the file-storage CLI and the client builder.
+ * @module src/server/storage/zip.js
+ * @namespace Zip
+ */
+
+import JSZip from 'jszip';
+import * as dir from 'path';
+import fs from 'fs-extra';
+
+const ZIP_LOCAL_FILE_HEADER = Buffer.from([0x50, 0x4b, 0x03, 0x04]);
+
+/**
+ * Reports whether a buffer starts with the ZIP local file header signature.
+ * @function isZipBuffer
+ * @param {Buffer} buffer - Candidate archive bytes.
+ * @returns {boolean} True when the buffer looks like a ZIP archive.
+ * @memberof Zip
+ */
+const isZipBuffer = (buffer) => Buffer.isBuffer(buffer) && buffer.subarray(0, 4).equals(ZIP_LOCAL_FILE_HEADER);
+
+/**
+ * Loads a ZIP archive from a buffer or a local path.
+ * @function loadZip
+ * @param {Buffer|string} source - Archive bytes or a path to the archive.
+ * @returns {Promise<JSZip>} The loaded archive.
+ * @memberof Zip
+ */
+const loadZip = async (source) => JSZip.loadAsync(Buffer.isBuffer(source) ? source : fs.readFileSync(source));
+
+/**
+ * Resolves an archive entry by exact name, then by trailing path segment, then by
+ * falling back to the only entry of a single-file archive.
+ * @function findZipEntry
+ * @param {JSZip} zip - The loaded archive.
+ * @param {string} entryName - Entry name or basename to look up.
+ * @returns {object|null} The matching JSZip entry, or null when nothing matches.
+ * @memberof Zip
+ */
+const findZipEntry = (zip, entryName) => {
+  const files = Object.values(zip.files).filter((entry) => !entry.dir);
+  return (
+    files.find((entry) => entry.name === entryName) ??
+    files.find((entry) => entry.name.endsWith(`/${entryName}`)) ??
+    (files.length === 1 ? files[0] : null)
+  );
+};
+
+/**
+ * Reads a single entry out of an archive.
+ * @function readZipEntry
+ * @param {Buffer|string} source - Archive bytes or a path to the archive.
+ * @param {string} entryName - Entry name or basename to read.
+ * @returns {Promise<Buffer|null>} The entry contents, or null when the entry is absent.
+ * @memberof Zip
+ */
+const readZipEntry = async (source, entryName) => {
+  const entry = findZipEntry(await loadZip(source), entryName);
+  return entry ? await entry.async('nodebuffer') : null;
+};
+
+/**
+ * Builds an archive buffer from local files.
+ * @function zipFromLocalFiles
+ * @param {Array<{localPath: string, entryName: string}>} entries - Files to archive.
+ * @returns {Promise<Buffer>} The generated archive bytes.
+ * @memberof Zip
+ */
+const zipFromLocalFiles = async (entries) => {
+  const zip = new JSZip();
+  for (const { localPath, entryName } of entries) {
+    const stats = fs.statSync(localPath);
+    zip.file(entryName, fs.readFileSync(localPath), {
+      date: stats.mtime,
+      unixPermissions: stats.mode & 0o777,
+    });
+  }
+  return await zip.generateAsync({
+    type: 'nodebuffer',
+    platform: 'UNIX',
+    compression: 'DEFLATE',
+    compressionOptions: { level: 6 },
+  });
+};
+
+/**
+ * Resolves an entry name against an output directory, rejecting archive paths that
+ * escape it (zip slip).
+ * @function resolveEntryTarget
+ * @param {string} outputPath - Extraction root.
+ * @param {string} entryName - Entry name from the archive.
+ * @returns {string} The absolute target path.
+ * @throws {Error} When the entry resolves outside the extraction root.
+ * @memberof Zip
+ */
+const resolveEntryTarget = (outputPath, entryName) => {
+  const root = dir.resolve(outputPath);
+  const target = dir.resolve(root, entryName);
+  if (target !== root && !target.startsWith(root + dir.sep)) {
+    throw new Error(`Blocked zip entry outside extraction directory: ${entryName}`);
+  }
+  return target;
+};
+
+/**
+ * Extracts every entry of an archive into a directory.
+ * @function extractZipTo
+ * @param {Buffer|string} source - Archive bytes or a path to the archive.
+ * @param {string} outputPath - Extraction root; created when missing.
+ * @returns {Promise<string[]>} The written file paths.
+ * @memberof Zip
+ */
+const extractZipTo = async (source, outputPath) => {
+  const zip = await loadZip(source);
+  fs.mkdirSync(outputPath, { recursive: true });
+
+  const writtenPaths = [];
+  for (const entry of Object.values(zip.files)) {
+    const target = resolveEntryTarget(outputPath, entry.name);
+    if (entry.dir) {
+      fs.mkdirSync(target, { recursive: true });
+      continue;
+    }
+    fs.mkdirSync(dir.dirname(target), { recursive: true });
+    fs.writeFileSync(target, await entry.async('nodebuffer'));
+    if (entry.unixPermissions) fs.chmodSync(target, entry.unixPermissions & 0o777);
+    writtenPaths.push(target);
+  }
+  return writtenPaths;
+};
+
+/**
+ * Extracts a single archive entry into a directory, flattening its archive path.
+ * @function extractZipEntryTo
+ * @param {Buffer|string} source - Archive bytes or a path to the archive.
+ * @param {string} entryName - Entry name or basename to extract.
+ * @param {string} targetDir - Destination directory; created when missing.
+ * @returns {Promise<string>} The written file path.
+ * @throws {Error} When the entry is not present in the archive.
+ * @memberof Zip
+ */
+const extractZipEntryTo = async (source, entryName, targetDir) => {
+  const entry = findZipEntry(await loadZip(source), entryName);
+  if (!entry) throw new Error(`Zip entry not found: ${entryName}`);
+
+  const target = resolveEntryTarget(targetDir, dir.basename(entryName));
+  fs.mkdirSync(dir.dirname(target), { recursive: true });
+  fs.writeFileSync(target, await entry.async('nodebuffer'));
+  if (entry.unixPermissions) fs.chmodSync(target, entry.unixPermissions & 0o777);
+  return target;
+};
+
+export { isZipBuffer, loadZip, findZipEntry, readZipEntry, zipFromLocalFiles, extractZipTo, extractZipEntryTo };