@whyour/qinglong 0.21.2 → 2.17.8-beta.1

package/docker/310.Dockerfile

@@ -54,7 +54,7 @@ RUN set -x && \
   git config --global user.email "qinglong@@users.noreply.github.com" && \
   git config --global user.name "qinglong" && \
   git config --global http.postBuffer 524288000 && \
-  npm install -g pnpm@8.3.1 pm2 tsx && \
+  npm install -g pnpm@8.3.1 pm2 ts-node && \
   rm -rf /root/.pnpm-store && \
   rm -rf /root/.local/share/pnpm/store && \
   rm -rf /root/.cache && \

package/docker/Dockerfile

@@ -55,7 +55,7 @@ RUN set -x && \
   git config --global user.email "qinglong@@users.noreply.github.com" && \
   git config --global user.name "qinglong" && \
   git config --global http.postBuffer 524288000 && \
-  npm install -g pnpm@8.3.1 pm2 tsx && \
+  npm install -g pnpm@8.3.1 pm2 ts-node && \
   rm -rf /root/.pnpm-store && \
   rm -rf /root/.local/share/pnpm/store && \
   rm -rf /root/.cache && \

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@whyour/qinglong",
-  "version": "0.21.2",
+  "version": "2.17.8-beta.1",
   "description": "Timed task management platform supporting Python3, JavaScript, Shell, Typescript",
   "repository": {
     "type": "git",
@@ -179,10 +179,9 @@
     "react-intl-universal": "^2.6.21",
     "react-split-pane": "^0.1.92",
     "sockjs-client": "^1.6.0",
-    "ts-node": "^10.6.0",
+    "ts-node": "^10.9.2",
     "ts-proto": "^1.146.0",
     "tslib": "^2.4.0",
-    "tsx": "^4.7.3",
     "typescript": "5.2.2",
     "vh-check": "^2.0.5",
     "virtualizedtableforantd4": "1.3.0",

package/sample/config.sample.sh

@@ -8,7 +8,7 @@ AutoAddCron="true"
 DefaultCronRule=""
 
 ## ql repo命令拉取脚本时需要拉取的文件后缀，直接写文件后缀名即可
-RepoFileExtensions="js py"
+RepoFileExtensions="js mjs py pyc"
 
 ## 代理地址，支持HTTP/SOCK5，例如 http://127.0.0.1:7890
 ProxyUrl=""

package/shell/api.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 create_token() {
-  local token_command="tsx ${dir_root}/back/token.ts"
+  local token_command="ts-node-transpile-only ${dir_root}/back/token.ts"
   local token_file="${dir_root}/static/build/token.js"
   if [[ -f $token_file ]]; then
     token_command="node ${token_file}"

package/shell/check.sh

@@ -73,7 +73,7 @@ check_pm2() {
 
 main() {
   echo -e "=====> 开始检测"
-  npm i -g pnpm@8.3.1 pm2 tsx
+  npm i -g pnpm@8.3.1 pm2 ts-node
   patch_version
 
   reset_env

package/shell/otask.sh

@@ -71,8 +71,17 @@ run_nohup() {
 
 
 env_str_to_array() {
+  . $file_env
   local IFS="&"
   read -ra array <<<"${!env_param}"
+  array_length=${#array[@]}
+  clear_env
+}
+
+clear_non_sh_env() {
+  if [[ $file_param != *.sh ]]; then
+    clear_env
+  fi
 }
 
 ## 正常运行单个脚本，$1：传入参数
@@ -89,37 +98,34 @@ run_normal() {
     file_param=${file_param/$relative_path\//}
   fi
 
-  $timeoutCmd $which_program $file_param "${script_params[@]}"
+  if [[ $isJsOrPythonFile == 'false' ]]; then
+    clear_non_sh_env
+  fi
+  configDir="${dir_config}" $timeoutCmd $which_program $file_param "${script_params[@]}"
+}
+
+handle_env_split() {
+  if [[ ! $num_param ]]; then
+    num_param="1-max"
+  fi
+
+  env_str_to_array
+  local tempArr=$(echo $num_param | sed "s/-max/-${array_length}/g" | sed "s/max-/${array_length}-/g" | perl -pe "s|(\d+)(-\|~\|_)(\d+)|{\1..\3}|g")
+  local runArr=($(eval echo $tempArr))
+  array_run=($(awk -v RS=' ' '!a[$1]++' <<<${runArr[@]}))
 }
 
 ## 并发执行时，设定的 RandomDelay 不会生效，即所有任务立即执行
 run_concurrent() {
   local file_param="$1"
   local env_param="$2"
-  local num_param=$(echo "$3" | perl -pe "s|.*$2(.*)|\1|")
+  local num_param=$(echo "$3" | perl -pe "s|.*$2(.*)|\1|" | awk '{$1=$1};1')
   if [[ ! $env_param ]]; then
     echo -e "\n 缺少并发运行的环境变量参数"
     exit 1
   fi
 
-  env_str_to_array
-  local tempArr=$(echo $num_param | sed "s/-max/-${#array[@]}/g" | sed "s/max-/${#array[@]}-/g" | perl -pe "s|(\d+)(-\|~\|_)(\d+)|{\1..\3}|g")
-  local runArr=($(eval echo $tempArr))
-  runArr=($(awk -v RS=' ' '!a[$1]++' <<<${runArr[@]}))
-
-  local n=0
-  for i in ${runArr[@]}; do
-    array_run[n]=${array[$i - 1]}
-    let n++
-  done
-
-  local cookieStr=$(
-    IFS="&"
-    echo "${array_run[*]}"
-  )
-  [[ ! -z $cookieStr ]] && export "${env_param}=${cookieStr}"
-
-  env_str_to_array
+  handle_env_split
   single_log_time=$(date "+%Y-%m-%d-%H-%M-%S.%3N")
 
   cd $dir_scripts
@@ -128,15 +134,24 @@ run_concurrent() {
     cd ${relative_path}
     file_param=${file_param/$relative_path\//}
   fi
-  for i in "${!array[@]}"; do
-    export "${env_param}=${array[i]}"
-    single_log_path="$dir_log/$log_dir/${single_log_time}_$((i + 1)).log"
-    eval $timeoutCmd $which_program $file_param "${script_params[@]}" &>$single_log_path &
+
+  local j=0
+  for i in ${array_run[@]}; do
+    single_log_path="$dir_log/$log_dir/${single_log_time}_$((j + 1)).log"
+    let j++
+
+    if [[ $isJsOrPythonFile == 'false' ]]; then
+      export "${env_param}=${array[$i - 1]}"
+      clear_non_sh_env
+    fi
+    eval configDir="${dir_config}" envParam="${env_param}" numParam="${i}" $timeoutCmd $which_program $file_param "${script_params[@]}" &>$single_log_path &
   done
 
   wait
-  for i in "${!array[@]}"; do
-    single_log_path="$dir_log/$log_dir/${single_log_time}_$((i + 1)).log"
+  local k=0
+  for i in ${array_run[@]}; do
+    single_log_path="$dir_log/$log_dir/${single_log_time}_$((k + 1)).log"
+    let k++
     cat $single_log_path
     [[ -f $single_log_path ]] && rm -f $single_log_path
   done
@@ -145,28 +160,27 @@ run_concurrent() {
 run_designated() {
   local file_param="$1"
   local env_param="$2"
-  local num_param=$(echo "$3" | perl -pe "s|.*$2(.*)|\1|")
-  if [[ ! $env_param ]] || [[ ! $num_param ]]; then
-    echo -e "\n 缺少单独运行的参数 task xxx.js desi Test 1 3"
+  local num_param=$(echo "$3" | perl -pe "s|.*$2(.*)|\1|" | awk '{$1=$1};1')
+  if [[ ! $env_param ]]; then
+    echo -e "\n 缺少单独运行的参数 task xxx.js desi Test"
     exit 1
   fi
 
-  env_str_to_array
-  local tempArr=$(echo $num_param | sed "s/-max/-${#array[@]}/g" | sed "s/max-/${#array[@]}-/g" | perl -pe "s|(\d+)(-\|~\|_)(\d+)|{\1..\3}|g")
-  local runArr=($(eval echo $tempArr))
-  runArr=($(awk -v RS=' ' '!a[$1]++' <<<${runArr[@]}))
-
-  local n=0
-  for i in ${runArr[@]}; do
-    array_run[n]=${array[$i - 1]}
-    let n++
-  done
+  handle_env_split
 
-  local cookieStr=$(
-    IFS="&"
-    echo "${array_run[*]}"
-  )
-  [[ ! -z $cookieStr ]] && export "${env_param}=${cookieStr}"
+  if [[ $isJsOrPythonFile == 'false' ]]; then
+    local n=0
+    for i in ${array_run[@]}; do
+      array_str[n]=${array[$i - 1]}
+      let n++
+    done
+    local envStr=$(
+      IFS="&"
+      echo "${array_str[*]}"
+    )
+    [[ ! -z $envStr ]] && export "${env_param}=${envStr}"
+    clear_non_sh_env
+  fi
 
   cd $dir_scripts
   local relative_path="${file_param%/*}"
@@ -174,7 +188,8 @@ run_designated() {
     cd ${relative_path}
     file_param=${file_param/$relative_path\//}
   fi
-  $timeoutCmd $which_program $file_param "${script_params[@]}"
+
+  configDir="${dir_config}" envParam="${env_param}" numParam="${num_param}" $timeoutCmd $which_program $file_param "${script_params[@]}"
 }
 
 ## 运行其他命令
@@ -190,12 +205,31 @@ run_else() {
 
   shift
 
+  clear_non_sh_env
   $timeoutCmd $which_program $file_param "$@"
 }
 
 ## 命令检测
 main() {
+  isJsOrPythonFile="false"
+  if [[ $1 == *.js ]] || [[ $1 == *.py ]] || [[ $1 == *.pyc ]] || [[ $1 == *.ts ]]; then
+    isJsOrPythonFile="true"
+  fi
+  if [[ -f $file_env ]]; then
+    get_env_array
+    if [[ $isJsOrPythonFile == 'true' ]]; then
+      export NODE_OPTIONS="${NODE_OPTIONS} -r ${preload_js_file}"
+      export PYTHONPATH="${PYTHONPATH}:${dir_preload}"
+    else
+      . $file_env
+    fi
+  fi
+
   if [[ $1 == *.js ]] || [[ $1 == *.py ]] || [[ $1 == *.pyc ]] || [[ $1 == *.sh ]] || [[ $1 == *.ts ]]; then
+    if [[ $1 == *.sh ]]; then
+      timeoutCmd=""
+    fi
+
     case $# in
     1)
       run_normal "$1"
@@ -229,4 +263,5 @@ handle_task_start "${task_shell_params[@]}"
 run_task_before "${task_shell_params[@]}"
 main "${task_shell_params[@]}"
 run_task_after "${task_shell_params[@]}"
+clear_env
 handle_task_end "${task_shell_params[@]}"

package/shell/preload/sitecustomize.js

@@ -0,0 +1,30 @@
+require(`./env.js`);
+
+function expandRange(rangeStr, max) {
+  const tempRangeStr = rangeStr
+    .trim()
+    .replace(/-max/g, `-${max}`)
+    .replace(/max-/g, `${max}-`);
+
+  return tempRangeStr.split(' ').flatMap((part) => {
+    const rangeMatch = part.match(/^(\d+)([-~_])(\d+)$/);
+    if (rangeMatch) {
+      const [, start, , end] = rangeMatch.map(Number);
+      return Array.from({ length: end - start + 1 }, (_, i) => start + i);
+    }
+    return Number(part);
+  });
+}
+
+function run() {
+  if (process.env.envParam && process.env.numParam) {
+    const { envParam, numParam } = process.env;
+    const array = (process.env[envParam] || '').split('&');
+    const runArr = expandRange(numParam, array.length);
+    const arrayRun = runArr.map((i) => array[i - 1]);
+    const envStr = arrayRun.join('&');
+    process.env[envParam] = envStr;
+  }
+}
+
+run();

package/shell/preload/sitecustomize.py

@@ -0,0 +1,44 @@
+import os
+import re
+import env
+
+
+def try_parse_int(value):
+    try:
+        return int(value)
+    except ValueError:
+        return None
+
+
+def expand_range(range_str, max_value):
+    temp_range_str = (
+        range_str.strip()
+        .replace("-max", f"-{max_value}")
+        .replace("max-", f"{max_value}-")
+    )
+
+    result = []
+    for part in temp_range_str.split(" "):
+        range_match = re.match(r"^(\d+)([-~_])(\d+)$", part)
+        if range_match:
+            start, _, end = map(try_parse_int, range_match.groups())
+            result.extend(range(start, end + 1))
+        else:
+            result.append(int(part))
+
+    return result
+
+
+def run():
+    env_param = os.getenv("envParam")
+    num_param = os.getenv("numParam")
+
+    if env_param and num_param:
+        array = (os.getenv(env_param) or "").split("&")
+        run_arr = expand_range(num_param, len(array))
+        array_run = [array[i - 1] for i in run_arr if i - 1 < len(array) and i > 0]
+        env_str = "&".join(array_run)
+        os.environ[env_param] = env_str
+
+
+run()

package/shell/pub.sh

@@ -11,7 +11,7 @@ echo -e "提交master代码"
 git push
 
 echo -e "更新cdn文件"
-tsx sample/tool.ts
+ts-node-transpile-only sample/tool.ts
 
 string=$(cat version.yaml | grep "version" | egrep "[^ ]*" -o | egrep "\d\.*")
 version="v$string"

package/shell/share.sh

@@ -10,6 +10,7 @@ if [[ $QL_DATA_DIR ]]; then
 fi
 
 dir_shell=$dir_root/shell
+dir_preload=$dir_shell/preload
 dir_sample=$dir_root/sample
 dir_static=$dir_root/static
 dir_config=$dir_data/config
@@ -25,7 +26,10 @@ ql_static_repo=$dir_repo/static
 
 ## 文件
 file_config_sample=$dir_sample/config.sample.sh
-file_env=$dir_config/env.sh
+file_env=$dir_preload/env.sh
+js_file_env=$dir_preload/env.js
+py_file_env=$dir_preload/env.py
+preload_js_file=$dir_preload/sitecustomize.js
 file_sharecode=$dir_config/sharecode.sh
 file_config_user=$dir_config/config.sh
 file_auth_sample=$dir_sample/auth.sample.json
@@ -74,9 +78,6 @@ init_env() {
 
 import_config() {
   [[ -f $file_config_user ]] && . $file_config_user
-  if [[ $LOAD_ENV != 'false' ]] && [[ -f $file_env ]]; then
-    . $file_env
-  fi
 
   ql_base_url=${QlBaseUrl:-"/"}
   ql_port=${QlPort:-"5700"}
@@ -439,6 +440,19 @@ init_nginx() {
   sed -i "s,IPV4_CONFIG,${ipv4Str},g" /etc/nginx/conf.d/front.conf
 }
 
+get_env_array() {
+  exported_variables=()
+  while IFS= read -r line; do
+    exported_variables+=("$line")
+  done < <(grep '^export ' $file_env | awk '{print $2}' | cut -d= -f1)
+}
+
+clear_env() {
+  for var in "${exported_variables[@]}"; do
+    unset "$var"
+  done
+}
+
 handle_task_start() {
   [[ $ID ]] && update_cron "\"$ID\"" "0" "$$" "$log_path" "$begin_timestamp"
   echo -e "## 开始执行... $begin_time\n"

package/shell/start.sh

@@ -53,7 +53,7 @@ else
   exit 1
 fi
 
-npm install -g pnpm@8.3.1 pm2 tsx
+npm install -g pnpm@8.3.1 pm2 ts-node
 
 cd ${QL_DIR}
 cp -f .env.example .env

package/shell/task.sh

@@ -19,13 +19,9 @@ define_program() {
   elif [[ $file_param == *.py ]] || [[ $file_param == *.pyc ]]; then
     which_program="python3"
   elif [[ $file_param == *.sh ]]; then
-    which_program="bash"
+    which_program="."
   elif [[ $file_param == *.ts ]]; then
-    if ! type tsx &>/dev/null; then
-      which_program="ts-node-transpile-only"
-    else
-      which_program="tsx"
-    fi
+    which_program="ts-node-transpile-only"
   else
     which_program=""
   fi

package/static/build/config/index.js

@@ -24,6 +24,7 @@ if (process.env.QL_DATA_DIR) {
     dataPath = process.env.QL_DATA_DIR.replace(/\/$/g, '');
 }
 const shellPath = path_1.default.join(rootPath, 'shell/');
+const preloadPath = path_1.default.join(shellPath, 'preload/');
 const tmpPath = path_1.default.join(rootPath, '.tmp/');
 const samplePath = path_1.default.join(rootPath, 'sample/');
 const configPath = path_1.default.join(dataPath, 'config/');
@@ -34,9 +35,9 @@ const dbPath = path_1.default.join(dataPath, 'db/');
 const uploadPath = path_1.default.join(dataPath, 'upload/');
 const sshdPath = path_1.default.join(dataPath, 'ssh.d/');
 const systemLogPath = path_1.default.join(dataPath, 'syslog/');
-const envFile = path_1.default.join(configPath, 'env.sh');
-const jsEnvFile = path_1.default.join(configPath, 'env.js');
-const pyEnvFile = path_1.default.join(configPath, 'env.py');
+const envFile = path_1.default.join(preloadPath, 'env.sh');
+const jsEnvFile = path_1.default.join(preloadPath, 'env.js');
+const pyEnvFile = path_1.default.join(preloadPath, 'env.py');
 const confFile = path_1.default.join(configPath, 'config.sh');
 const crontabFile = path_1.default.join(configPath, 'crontab.list');
 const authConfigFile = path_1.default.join(configPath, 'auth.json');

package/static/build/loaders/initFile.js

@@ -13,6 +13,7 @@ let dataPath = path_1.default.join(rootPath, 'data/');
 if (process.env.QL_DATA_DIR) {
     dataPath = process.env.QL_DATA_DIR.replace(/\/$/g, '');
 }
+const preloadPath = path_1.default.join(rootPath, 'shell/preload/');
 const configPath = path_1.default.join(dataPath, 'config/');
 const scriptPath = path_1.default.join(dataPath, 'scripts/');
 const logPath = path_1.default.join(dataPath, 'log/');
@@ -39,6 +40,7 @@ exports.default = async () => {
     const authFileExist = await (0, util_1.fileExist)(authConfigFile);
     const confFileExist = await (0, util_1.fileExist)(confFile);
     const scriptDirExist = await (0, util_1.fileExist)(scriptPath);
+    const preloadDirExist = await (0, util_1.fileExist)(preloadPath);
     const logDirExist = await (0, util_1.fileExist)(logPath);
     const configDirExist = await (0, util_1.fileExist)(configPath);
     const uploadDirExist = await (0, util_1.fileExist)(uploadPath);
@@ -57,6 +59,9 @@ exports.default = async () => {
     if (!scriptDirExist) {
         await promises_1.default.mkdir(scriptPath);
     }
+    if (!preloadDirExist) {
+        await promises_1.default.mkdir(preloadPath);
+    }
     if (!logDirExist) {
         await promises_1.default.mkdir(logPath);
     }

package/static/build/loaders/initTask.js

@@ -15,7 +15,7 @@ exports.default = async () => {
     const scheduleService = typedi_1.Container.get(schedule_1.default);
     const subscriptionService = typedi_1.Container.get(subscription_1.default);
     // 生成内置token
-    let tokenCommand = `tsx ${(0, path_1.join)(config_1.default.rootPath, 'back/token.ts')}`;
+    let tokenCommand = `ts-node-transpile-only ${(0, path_1.join)(config_1.default.rootPath, 'back/token.ts')}`;
     const tokenFile = (0, path_1.join)(config_1.default.rootPath, 'static/build/token.js');
     if (await (0, util_1.fileExist)(tokenFile)) {
         tokenCommand = `node ${tokenFile}`;

package/static/build/services/env.js

@@ -206,13 +206,12 @@ let EnvService = class EnvService {
                         .replace(/'/g, "'\\''")
                         .trim();
                     env_string += `export ${key}='${value}'\n`;
-                    const _env_value = `'${group
+                    const _env_value = `${group
                         .map((x) => x.value)
                         .join('&')
-                        .replace(/\\/g, '\\\\')
-                        .replace(/'/g, "\\'")}'`;
-                    js_env_string += `process.env.${key}=${_env_value};\n`;
-                    py_env_string += `os.environ['${key}']=${_env_value}\n`;
+                        .replace(/\\/g, '\\\\')}`;
+                    js_env_string += `process.env.${key}=\`${_env_value.replace(/\`/g, '\\`')}\`;\n`;
+                    py_env_string += `os.environ['${key}']='''${_env_value.replace(/\'/g, "\\'")}'''\n`;
                 }
             }
         }

package/static/build/services/system.js

@@ -31,6 +31,7 @@ const pLimit_1 = __importDefault(require("../shared/pLimit"));
 const notify_1 = __importDefault(require("./notify"));
 const schedule_1 = __importDefault(require("./schedule"));
 const sock_1 = __importDefault(require("./sock"));
+const os_1 = __importDefault(require("os"));
 const util_2 = require("../config/util");
 let SystemService = class SystemService {
     constructor(logger, scheduleService, sockService) {
@@ -156,6 +157,9 @@ let SystemService = class SystemService {
     }
     async updateLinuxMirror(info, res, onEnd) {
         const oDoc = await this.getSystemConfig();
+        if (os_1.default.platform() !== 'linux') {
+            return;
+        }
         const command = await (0, util_2.updateLinuxMirrorFile)(info.linuxMirror || '');
         let hasError = false;
         this.scheduleService.runTask(command, {

package/version.yaml

@@ -1,8 +1,7 @@
-version: 2.17.7
-changeLogLink: https://t.me/jiao_long/408
-publishTime: 2024-06-30 16:00
+version: 2.17.8
+changeLogLink: https://t.me/jiao_long/419
+publishTime: 2024-07-13 08:00
 changeLog: |
-  1. 修复更新任务状态错误
-  2. 修复查询系统 health 状态错误
-  3. 修复更新 linux 镜像源错误
-  4. npm 包支持 alpine 和 debian、ubuntu，修复 linux 依赖安装命令
+  1. 修改环境变量加载逻辑
+  2. 修复执行任务报错 Argument list too long
+