@whyour/qinglong 2.21.0-13 → 2.21.0-15

package/docker/Dockerfile

@@ -1,11 +1,26 @@
-FROM python:3.11-alpine3.18 AS builder
+# Run Node package installation natively on the builder. Node/npm can spin at
+# 100% CPU when Alpine s390x is emulated through QEMU.
+FROM --platform=$BUILDPLATFORM node:18-alpine3.18 AS builder
+
+ARG TARGETARCH
+ENV NPM_CONFIG_PREFIX=/opt/node-global
+ENV PATH=/opt/node-global/bin:${PATH}
+
 COPY package.json .npmrc pnpm-lock.yaml /tmp/build/
 RUN set -x \
-  && apk update \
-  && apk add nodejs npm git \
+  && apk add --no-cache git \
   && npm i -g pnpm@8.3.1 pm2 ts-node typescript@5 \
   && cd /tmp/build \
-  && pnpm install --prod
+  && case "${TARGETARCH}" in \
+    amd64) NODE_ARCH=x64 ;; \
+    386) NODE_ARCH=ia32 ;; \
+    ppc64le) NODE_ARCH=ppc64 ;; \
+    *) NODE_ARCH="${TARGETARCH}" ;; \
+  esac \
+  && npm_config_target_platform=linux \
+    npm_config_target_arch="${NODE_ARCH}" \
+    npm_config_target_libc=musl \
+    pnpm install --prod
 
 FROM python:3.11-alpine
 
@@ -26,8 +41,8 @@ VOLUME /ql/data
 
 EXPOSE 5700
 
-COPY --from=builder /usr/local/lib/node_modules/. /usr/local/lib/node_modules/
-COPY --from=builder /usr/local/bin/. /usr/local/bin/
+COPY --from=builder /opt/node-global/lib/node_modules/. /usr/local/lib/node_modules/
+COPY --from=builder /opt/node-global/bin/. /usr/local/bin/
 
 RUN set -x \
   && apk update -f \

package/docker/Dockerfile.310

@@ -1,11 +1,26 @@
-FROM python:3.10-alpine3.18 AS builder
+# Run Node package installation natively on the builder. Node/npm can spin at
+# 100% CPU when Alpine s390x is emulated through QEMU.
+FROM --platform=$BUILDPLATFORM node:18-alpine3.18 AS builder
+
+ARG TARGETARCH
+ENV NPM_CONFIG_PREFIX=/opt/node-global
+ENV PATH=/opt/node-global/bin:${PATH}
+
 COPY package.json .npmrc pnpm-lock.yaml /tmp/build/
 RUN set -x \
-  && apk update \
-  && apk add nodejs npm git \
+  && apk add --no-cache git \
   && npm i -g pnpm@8.3.1 pm2 ts-node typescript@5 \
   && cd /tmp/build \
-  && pnpm install --prod
+  && case "${TARGETARCH}" in \
+    amd64) NODE_ARCH=x64 ;; \
+    386) NODE_ARCH=ia32 ;; \
+    ppc64le) NODE_ARCH=ppc64 ;; \
+    *) NODE_ARCH="${TARGETARCH}" ;; \
+  esac \
+  && npm_config_target_platform=linux \
+    npm_config_target_arch="${NODE_ARCH}" \
+    npm_config_target_libc=musl \
+    pnpm install --prod
 
 FROM python:3.10-alpine
 
@@ -26,8 +41,8 @@ VOLUME /ql/data
 
 EXPOSE 5700
 
-COPY --from=builder /usr/local/lib/node_modules/. /usr/local/lib/node_modules/
-COPY --from=builder /usr/local/bin/. /usr/local/bin/
+COPY --from=builder /opt/node-global/lib/node_modules/. /usr/local/lib/node_modules/
+COPY --from=builder /opt/node-global/bin/. /usr/local/bin/
 
 RUN set -x \
   && apk update -f \

package/docker/Dockerfile.debian

@@ -1,4 +1,6 @@
-FROM node:22-slim AS nodebuilder
+# Node 20 Bookworm is the latest official Node image variant that covers the
+# full Debian build matrix, including arm/v7, ppc64le, and s390x.
+FROM node:20-bookworm-slim AS nodebuilder
 
 FROM python:3.11-slim-bookworm AS builder
 COPY package.json .npmrc pnpm-lock.yaml /tmp/build/

package/docker/Dockerfile.debian310

@@ -1,4 +1,6 @@
-FROM node:22-slim AS nodebuilder
+# Node 20 Bookworm is the latest official Node image variant that covers the
+# full Debian build matrix, including arm/v7, ppc64le, and s390x.
+FROM node:20-bookworm-slim AS nodebuilder
 
 FROM python:3.10-slim-bookworm AS builder
 COPY package.json .npmrc pnpm-lock.yaml /tmp/build/

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@whyour/qinglong",
   "packageManager": "pnpm@8.3.1",
-  "version": "2.21.0-13",
+  "version": "2.21.0-15",
   "description": "Timed task management platform supporting Python3, JavaScript, Shell, Typescript",
   "repository": {
     "type": "git",
@@ -18,6 +18,7 @@
     "start:front": "max dev",
     "build:front": "max build",
     "build:back": "tsc -p back/tsconfig.json",
+    "test:race": "node -r ts-node/register test/race-conditions.test.js",
     "panel": "npm run build:back && node static/build/app.js",
     "gen:proto": "protoc --experimental_allow_proto3_optional --plugin=./node_modules/.bin/protoc-gen-ts_proto ./back/protos/*.proto --ts_proto_out=./ --ts_proto_opt=outputServices=grpc-js,env=node,esModuleInterop=true,snakeToCamel=false",
     "prettier": "prettier --write '**/*.{js,jsx,tsx,ts,less,md,json}'",
@@ -65,7 +66,7 @@
       }
     },
     "overrides": {
-      "sqlite3": "git+https://github.com/whyour/node-sqlite3.git#v1.0.3",
+      "sqlite3": "git+https://github.com/whyour/node-sqlite3.git#v1.1.0",
       "@codemirror/state": "6.5.4",
       "@codemirror/view": "6.39.16"
     }
@@ -109,7 +110,7 @@
     "request-ip": "3.3.0",
     "sequelize": "^6.37.5",
     "sockjs": "^0.3.24",
-    "sqlite3": "git+https://github.com/whyour/node-sqlite3.git#v1.0.3",
+    "sqlite3": "git+https://github.com/whyour/node-sqlite3.git#v1.1.0",
     "toad-scheduler": "^3.0.1",
     "typedi": "^0.10.0",
     "undici": "^7.9.0",

package/static/build/config/util.js

@@ -232,13 +232,34 @@ async function readDirs(dir, baseDir = '', blacklist = [], sort = dirSort) {
     const result = [];
     for (const file of files) {
         const subPath = path.join(dir, file);
-        const stats = await fs.lstat(subPath);
+        let stats;
+        try {
+            stats = await fs.lstat(subPath);
+        }
+        catch (error) {
+            // Files can disappear between readdir and lstat while logs are pruned.
+            // Skip only that expected race and keep surfacing all other I/O errors.
+            if (error.code === 'ENOENT') {
+                continue;
+            }
+            throw error;
+        }
         const key = path.join(relativePath, file);
         if (blacklist.includes(file) || stats.isSymbolicLink()) {
             continue;
         }
         if (stats.isDirectory()) {
-            const children = await readDirs(subPath, baseDir, blacklist, sort);
+            let children;
+            try {
+                children = await readDirs(subPath, baseDir, blacklist, sort);
+            }
+            catch (error) {
+                // A directory may also be removed after lstat but before recursion.
+                if (error.code === 'ENOENT') {
+                    continue;
+                }
+                throw error;
+            }
             result.push({
                 title: file,
                 key,

package/static/build/services/config.js

@@ -40,7 +40,7 @@ const path_1 = __importStar(require("path"));
 const config_1 = __importDefault(require("../config"));
 const util_1 = require("../config/util");
 const i18n_1 = require("../shared/i18n");
-const undici_1 = require("undici");
+const sampleConfig_1 = require("./sampleConfig");
 let ConfigService = class ConfigService {
     constructor() { }
     async getFile(filePath, res) {
@@ -65,8 +65,7 @@ let ConfigService = class ConfigService {
             return res.send({ code: 403, message: (0, i18n_1.t)('文件无法访问') });
         }
         if (filePath.startsWith('sample/')) {
-            const res = await (0, undici_1.request)(`https://gitlab.com/whyour/qinglong/-/raw/master/${filePath}`);
-            content = await res.body.text();
+            content = await (0, sampleConfig_1.loadSampleFile)(filePath);
         }
         else if (filePath.startsWith('data/scripts/')) {
             content = await (0, util_1.getFileContentByName)((0, path_1.join)(config_1.default.rootPath, filePath));

package/static/build/services/sampleConfig.js

@@ -0,0 +1,73 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadSampleFile = void 0;
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const undici_1 = require("undici");
+const config_1 = __importDefault(require("../config"));
+const util_1 = require("../config/util");
+const logger_1 = __importDefault(require("../loaders/logger"));
+const sampleRequestDispatcher = new undici_1.RetryAgent(new undici_1.Agent({
+    connect: { timeout: 10000 },
+    headersTimeout: 10000,
+    bodyTimeout: 10000,
+    keepAliveTimeout: 10000,
+    keepAliveMaxTimeout: 30000,
+}), {
+    maxRetries: 2,
+    minTimeout: 200,
+    maxTimeout: 1000,
+    timeoutFactor: 2,
+    statusCodes: [429, 500, 502, 503, 504],
+    errorCodes: [
+        'ECONNRESET',
+        'ECONNREFUSED',
+        'ENOTFOUND',
+        'ENETDOWN',
+        'ENETUNREACH',
+        'EHOSTDOWN',
+        'EHOSTUNREACH',
+        'EPIPE',
+        'UND_ERR_CONNECT_TIMEOUT',
+        'UND_ERR_HEADERS_TIMEOUT',
+        'UND_ERR_BODY_TIMEOUT',
+        'UND_ERR_SOCKET',
+    ],
+});
+const pendingSampleRequests = new Map();
+async function loadSampleFile(filePath) {
+    const pendingRequest = pendingSampleRequests.get(filePath);
+    if (pendingRequest) {
+        return pendingRequest;
+    }
+    const requestPromise = (async () => {
+        const remoteUrl = `https://gitlab.com/whyour/qinglong/-/raw/master/${filePath}`;
+        try {
+            const response = await (0, undici_1.request)(remoteUrl, {
+                dispatcher: sampleRequestDispatcher,
+                maxRedirections: 3,
+            });
+            const content = await response.body.text();
+            if (response.statusCode < 200 || response.statusCode >= 300) {
+                throw new Error(`GitLab returned HTTP ${response.statusCode} for ${filePath}`);
+            }
+            return content;
+        }
+        catch (error) {
+            logger_1.default.warn(`[config] Failed to fetch ${filePath}, using bundled sample: ${(0, util_1.errStack)(error)}`);
+            return (0, promises_1.readFile)((0, path_1.join)(config_1.default.rootPath, filePath), 'utf8');
+        }
+    })();
+    pendingSampleRequests.set(filePath, requestPromise);
+    try {
+        return await requestPromise;
+    }
+    finally {
+        pendingSampleRequests.delete(filePath);
+    }
+}
+exports.loadSampleFile = loadSampleFile;
+//# sourceMappingURL=sampleConfig.js.map

package/test/race-conditions.test.js

@@ -0,0 +1,175 @@
+const assert = require('node:assert/strict');
+const fs = require('node:fs/promises');
+const os = require('node:os');
+const path = require('node:path');
+
+require('reflect-metadata');
+
+async function testReadDirsSkipsDisappearedEntries() {
+  const temporaryRoot = await fs.mkdtemp(
+    path.join(os.tmpdir(), 'qinglong-read-dirs-'),
+  );
+  const disappearingFile = path.join(temporaryRoot, 'disappearing.log');
+  const disappearingDirectory = path.join(temporaryRoot, 'disappearing-dir');
+  const originalLstat = fs.lstat;
+  const originalReaddir = fs.readdir;
+
+  await fs.writeFile(path.join(temporaryRoot, 'stable.log'), 'stable');
+  await fs.writeFile(disappearingFile, 'gone');
+  await fs.mkdir(disappearingDirectory);
+
+  fs.lstat = async (filePath, ...args) => {
+    if (filePath === disappearingFile) {
+      const error = new Error('file disappeared');
+      error.code = 'ENOENT';
+      throw error;
+    }
+    return originalLstat(filePath, ...args);
+  };
+  fs.readdir = async (directoryPath, ...args) => {
+    if (directoryPath === disappearingDirectory) {
+      const error = new Error('directory disappeared');
+      error.code = 'ENOENT';
+      throw error;
+    }
+    return originalReaddir(directoryPath, ...args);
+  };
+
+  try {
+    const { readDirs } = require('../back/config/util');
+    const entries = await readDirs(temporaryRoot, temporaryRoot);
+    assert.deepEqual(
+      entries.map((entry) => entry.title),
+      ['stable.log'],
+    );
+  } finally {
+    fs.lstat = originalLstat;
+    fs.readdir = originalReaddir;
+    await fs.rm(temporaryRoot, { recursive: true, force: true });
+  }
+}
+
+async function testReadDirsPreservesUnexpectedErrors() {
+  const temporaryRoot = await fs.mkdtemp(
+    path.join(os.tmpdir(), 'qinglong-read-dirs-error-'),
+  );
+  const blockedFile = path.join(temporaryRoot, 'blocked.log');
+  const originalLstat = fs.lstat;
+
+  await fs.writeFile(blockedFile, 'blocked');
+  fs.lstat = async (filePath, ...args) => {
+    if (filePath === blockedFile) {
+      const error = new Error('permission denied');
+      error.code = 'EACCES';
+      throw error;
+    }
+    return originalLstat(filePath, ...args);
+  };
+
+  try {
+    const { readDirs } = require('../back/config/util');
+    await assert.rejects(
+      readDirs(temporaryRoot, temporaryRoot),
+      (error) => error.code === 'EACCES',
+    );
+  } finally {
+    fs.lstat = originalLstat;
+    await fs.rm(temporaryRoot, { recursive: true, force: true });
+  }
+}
+
+async function testSampleRequestsAreCoalescedAndFallbackLocally() {
+  const undici = require('undici');
+  const originalRequest = undici.request;
+  let releaseRequest;
+  let requestCount = 0;
+
+  undici.request = async () => {
+    requestCount += 1;
+    await new Promise((resolve) => {
+      releaseRequest = resolve;
+    });
+    return {
+      statusCode: 200,
+      body: { text: async () => 'remote sample' },
+    };
+  };
+
+  const { loadSampleFile } = require('../back/services/sampleConfig');
+  const pendingLoads = [
+    loadSampleFile('sample/config.sample.sh'),
+    loadSampleFile('sample/config.sample.sh'),
+    loadSampleFile('sample/config.sample.sh'),
+  ];
+
+  await new Promise((resolve) => setImmediate(resolve));
+  assert.equal(requestCount, 1);
+  releaseRequest();
+  assert.deepEqual(await Promise.all(pendingLoads), [
+    'remote sample',
+    'remote sample',
+    'remote sample',
+  ]);
+
+  const temporaryRoot = await fs.mkdtemp(
+    path.join(os.tmpdir(), 'qinglong-sample-fallback-'),
+  );
+  const bundledSample = path.join(temporaryRoot, 'sample', 'config.sample.sh');
+  const config = require('../back/config').default;
+  const originalRootPath = config.rootPath;
+
+  await fs.mkdir(path.dirname(bundledSample), { recursive: true });
+  await fs.writeFile(bundledSample, 'bundled sample');
+  config.rootPath = temporaryRoot;
+  undici.request = async () => {
+    const error = new Error('other side closed');
+    error.code = 'UND_ERR_SOCKET';
+    throw error;
+  };
+
+  try {
+    assert.equal(
+      await loadSampleFile('sample/config.sample.sh'),
+      'bundled sample',
+    );
+
+    undici.request = async () => ({
+      statusCode: 503,
+      body: { text: async () => 'upstream unavailable' },
+    });
+    assert.equal(
+      await loadSampleFile('sample/config.sample.sh'),
+      'bundled sample',
+    );
+  } finally {
+    undici.request = originalRequest;
+    config.rootPath = originalRootPath;
+    await fs.rm(temporaryRoot, { recursive: true, force: true });
+  }
+}
+
+async function main() {
+  const testRoot = await fs.mkdtemp(
+    path.join(os.tmpdir(), 'qinglong-test-root-'),
+  );
+  process.env.QL_DIR = testRoot;
+  await fs.writeFile(path.join(testRoot, '.env'), 'NODE_ENV=test\n');
+  await fs.mkdir(path.join(testRoot, 'data', 'syslog'), { recursive: true });
+
+  try {
+    await testReadDirsSkipsDisappearedEntries();
+    await testReadDirsPreservesUnexpectedErrors();
+    await testSampleRequestsAreCoalescedAndFallbackLocally();
+    console.log('race condition regression tests passed');
+  } finally {
+    const loggerModule =
+      require.cache[require.resolve('../back/loaders/logger')];
+    loggerModule?.exports?.default?.close();
+    await fs.rm(testRoot, { recursive: true, force: true });
+  }
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});