@whyour/qinglong 2.22.0 → 2.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/back/protos/cron.proto +4 -0
- package/docs/development/node-cron-migration.md +49 -0
- package/docs/development/task-startup.md +78 -0
- package/docs/releases/2.22.0.md +8 -0
- package/docs/releases/2.23.0.md +33 -0
- package/docs/script-history.md +53 -0
- package/package.json +9 -8
- package/scripts/benchmark-script-history.cjs +39 -0
- package/scripts/publish-static.sh +34 -0
- package/scripts/sync-static.sh +20 -0
- package/scripts/wait-for-mirror.sh +214 -0
- package/shell/api.sh +56 -70
- package/shell/node_path_cache.sh +10 -2
- package/shell/otask.sh +7 -9
- package/shell/preload/sitecustomize.js +6 -2
- package/shell/preload/sitecustomize.py +4 -2
- package/shell/share.sh +1 -1
- package/shell/task-timeout.sh +63 -0
- package/shell/task.sh +8 -6
- package/shell/update.sh +40 -95
- package/shell/upgrade.cjs +8 -0
- package/static/build/api/config.js +2 -6
- package/static/build/api/env.js +15 -14
- package/static/build/api/open.js +2 -1
- package/static/build/api/script.js +155 -9
- package/static/build/api/subscription.js +3 -6
- package/static/build/api/system.js +35 -3
- package/static/build/config/grpcCerts.js +89 -40
- package/static/build/config/subscription.js +19 -11
- package/static/build/config/util.js +6 -4
- package/static/build/loaders/express.js +2 -0
- package/static/build/loaders/initData.js +20 -30
- package/static/build/loaders/initTask.js +11 -1
- package/static/build/middlewares/uniqueConstraintError.js +15 -0
- package/static/build/protos/cron.js +63 -2
- package/static/build/schedule/addCron.js +45 -63
- package/static/build/schedule/client.js +19 -1
- package/static/build/schedule/delCron.js +1 -1
- package/static/build/schedule/setConcurrency.js +27 -0
- package/static/build/services/config.js +2 -3
- package/static/build/services/cron.js +36 -10
- package/static/build/services/env.js +1 -1
- package/static/build/services/grpc.js +2 -1
- package/static/build/services/log.js +1 -1
- package/static/build/services/open.js +106 -31
- package/static/build/services/schedule.js +20 -10
- package/static/build/services/script.js +1 -1
- package/static/build/services/scriptHistory.js +30 -0
- package/static/build/services/sshKey.js +39 -14
- package/static/build/services/subscription.js +104 -53
- package/static/build/services/system.js +67 -32
- package/static/build/services/user.js +15 -10
- package/static/build/shared/cronCommand.js +17 -0
- package/static/build/shared/cronSchedule.js +101 -0
- package/static/build/shared/cronScheduler.js +73 -0
- package/static/build/shared/i18n.js +3 -2
- package/static/build/shared/logPath.js +15 -0
- package/static/build/shared/loginThrottle.js +45 -0
- package/static/build/shared/pLimit.js +19 -6
- package/static/build/shared/runCron.js +13 -4
- package/static/build/shared/scriptHistory.js +445 -0
- package/static/build/shared/shellQuote.js +11 -0
- package/static/build/shared/subscriptionMutationLock.js +34 -0
- package/static/build/shared/subscriptionPath.js +41 -0
- package/static/build/shared/subscriptionStorage.js +98 -0
- package/static/build/shared/utils.js +2 -1
- package/static/build/token.js +3 -14
- package/static/build/validation/schedule.js +3 -18
- package/static/dist/292.264757f0.async.js +1 -0
- package/static/dist/3512.95e41255.async.js +1 -0
- package/static/dist/index.html +2 -2
- package/static/dist/preload_helper.01fcd534.js +1 -0
- package/static/dist/src__pages__dependence__modal.0fa8b421.async.js +1 -0
- package/static/dist/src__pages__diff__index.b94e5dc5.async.js +1 -0
- package/static/dist/src__pages__script__editModal.afabf073.async.js +1 -0
- package/static/dist/src__pages__script__editNameModal.789ad87b.async.js +1 -0
- package/static/dist/src__pages__script__historyDrawer.8c58d958.async.js +1 -0
- package/static/dist/src__pages__script__historyDrawer.a449193a.chunk.css +1 -0
- package/static/dist/src__pages__script__index.ab360776.chunk.css +1 -0
- package/static/dist/src__pages__script__index.fc575945.async.js +1 -0
- package/static/dist/src__pages__script__saveModal.8fca1a22.async.js +1 -0
- package/static/dist/src__pages__script__saveWithHistory.99ece830.async.js +1 -0
- package/static/dist/src__pages__subscription__modal.30868ba8.async.js +1 -0
- package/static/dist/{umi.fc019050.js → umi.3a24a9fa.js} +1 -1
- package/test/back/advisory-input-security.test.cjs +416 -0
- package/test/back/code-scanning-security.test.cjs +189 -0
- package/test/back/cron-macros.test.cjs +51 -0
- package/test/back/cron-schedule-compatibility.test.cjs +69 -0
- package/test/back/cron-scheduler-routing.test.cjs +59 -0
- package/test/back/draft-path-security.test.cjs +235 -0
- package/test/back/env-import-security.test.cjs +91 -0
- package/test/back/execution-lifecycle.test.cjs +1 -1
- package/test/back/file-access-scope.test.cjs +180 -0
- package/test/back/grpc-mtls-security.test.cjs +136 -0
- package/test/back/http-security.test.cjs +105 -5
- package/test/back/legacy-cron-recovery.test.cjs +76 -0
- package/test/back/linux-mirror-result.test.cjs +176 -0
- package/test/back/listen-security.test.cjs +1 -0
- package/test/back/mirror-status.test.cjs +257 -0
- package/test/back/node-cron-migration.test.cjs +65 -0
- package/test/back/node-path-cache.test.cjs +29 -0
- package/test/back/node-path-lock.test.cjs +6 -2
- package/test/back/preload-lazy-notify.test.cjs +168 -0
- package/test/back/queued-cron-invalidation.test.cjs +98 -0
- package/test/back/restart-instance-recovery.test.cjs +116 -0
- package/test/back/scheduler-bulk-recovery.test.cjs +40 -0
- package/test/back/scheduler-concurrency.test.cjs +411 -0
- package/test/back/scheduler-reconciliation.test.cjs +58 -4
- package/test/back/script-file-access.test.cjs +210 -0
- package/test/back/script-history-api.test.cjs +551 -0
- package/test/back/script-history.test.cjs +383 -0
- package/test/back/shell-api-parsing.test.cjs +4 -4
- package/test/back/shell-translation-output.test.cjs +16 -0
- package/test/back/shell-upgrade.test.cjs +312 -0
- package/test/back/shellApiToken.test.cjs +151 -0
- package/test/back/ssh-config-home.test.cjs +113 -0
- package/test/back/static-mirror-trigger.test.cjs +441 -0
- package/test/back/static-mirror.test.cjs +179 -0
- package/test/back/stop-race.test.cjs +5 -1
- package/test/back/subscription-mutation-lock.test.cjs +83 -0
- package/test/back/subscription-review-regression.test.cjs +607 -0
- package/test/back/subscription-storage-regression.test.cjs +553 -0
- package/test/back/system-dependence-cache.test.cjs +1 -0
- package/test/back/system-export-history.test.cjs +119 -0
- package/test/back/systemTokenRenewal.test.cjs +497 -0
- package/test/back/task-shell-lifecycle.test.cjs +194 -0
- package/test/back/unique-constraint-error.test.cjs +72 -0
- package/test/back/upgrade-guards.test.cjs +285 -0
- package/test/back/upgrade-lifecycle.test.cjs +288 -0
- package/test/back/upgrade-transaction.test.cjs +133 -0
- package/test/back/user-security.test.cjs +102 -1
- package/test/back/wpush-grpc.test.cjs +1 -0
- package/test/fixtures/legacy-cron.json +1371 -0
- package/test/front/diff-file-routing.test.cjs +126 -0
- package/test/front/script-history-save.test.cjs +121 -0
- package/test/helpers/upgrade-fixture.cjs +30 -0
- package/version.yaml +14 -19
- package/static/dist/2096.6d98fd12.async.js +0 -1
- package/static/dist/4859.97d6653c.async.js +0 -1
- package/static/dist/6707.801b886b.async.js +0 -1
- package/static/dist/preload_helper.cf7de3fa.js +0 -1
- package/static/dist/src__pages__dependence__modal.a29c1a0f.async.js +0 -1
- package/static/dist/src__pages__diff__index.1714b941.async.js +0 -1
- package/static/dist/src__pages__script__editModal.22acdf7c.async.js +0 -1
- package/static/dist/src__pages__script__editNameModal.1a521cc0.async.js +0 -1
- package/static/dist/src__pages__script__index.60b6f827.chunk.css +0 -1
- package/static/dist/src__pages__script__index.a18b2692.async.js +0 -1
- package/static/dist/src__pages__script__saveModal.8965b74c.async.js +0 -1
- package/static/dist/src__pages__subscription__modal.38991e1e.async.js +0 -1
- /package/static/dist/{4859.1b962b29.chunk.css → 292.1b962b29.chunk.css} +0 -0
package/back/protos/cron.proto
CHANGED
|
@@ -5,6 +5,7 @@ package com.ql.cron;
|
|
|
5
5
|
service Cron {
|
|
6
6
|
rpc addCron(AddCronRequest) returns (AddCronResponse);
|
|
7
7
|
rpc delCron(DeleteCronRequest) returns (DeleteCronResponse);
|
|
8
|
+
rpc setConcurrency(SetConcurrencyRequest) returns (AddCronResponse);
|
|
8
9
|
}
|
|
9
10
|
|
|
10
11
|
message ISchedule { string schedule = 1; }
|
|
@@ -27,3 +28,6 @@ message AddCronResponse {}
|
|
|
27
28
|
message DeleteCronRequest { repeated string ids = 1; }
|
|
28
29
|
|
|
29
30
|
message DeleteCronResponse {}
|
|
31
|
+
|
|
32
|
+
// Zero restores the default limit.
|
|
33
|
+
message SetConcurrencyRequest { int32 concurrency = 1; }
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# Node cron 调度器迁移
|
|
2
|
+
|
|
3
|
+
Node 调度路径使用固定版本 `node-cron@4.6.0`,普通任务(含额外规则)、订阅和系统 cron 共用 `back/shared/cronScheduler.ts`。启动/一次性任务、间隔任务、脚本并发限制和进程执行链路保持原有实现。系统模式只将语义一致的普通五字段规则交给 crond。
|
|
4
|
+
|
|
5
|
+
移除 `node-schedule`、其类型包以及兼容别名 `cron-parser-v4`。保留 `cron-parser@5` 做旧表达式的校验与标准化;计时器和日历推进由 node-cron 管理。
|
|
6
|
+
|
|
7
|
+
## 兼容行为
|
|
8
|
+
|
|
9
|
+
- 保留旧版 1–6 字段补全、五种宏、月份/星期名称、数字起点步长、L 和 #。另外明确新增 `@annually`(等同 `@yearly`)、`@midnight`(等同 `@daily`)、`@minutely`(每分钟第 0 秒)。宏按进程时区解释。`?` 仅允许日期和星期字段;裸 `/N`、H 和其他未列出的宏仍拒绝。
|
|
10
|
+
- 日期与星期同时指定时保留 OR 语义,不采用 node-cron 默认的 AND。两个原生任务分别处理日期和星期,并按计划时间过滤交集,避免重复执行;保留旧解析器的月份全日期范围及全局 nth-week 限制。
|
|
11
|
+
- 原生任务先暂停创建并验证,再替换旧任务。构造失败时销毁已准备的实例并保留旧快照。
|
|
12
|
+
- 进程内迟到的触发通过 `execution:missed` 加入既有执行队列,记录补执行日志。并发和单任务积压上限仍由青龙队列控制;这不提供进程退出期间的持久化补跑或分布式 exactly-once 保证。
|
|
13
|
+
- 标准化结果使用上限 512 条的只读缓存,仅缓存语法,不缓存运行实例或下次执行时间。避免大批相同表达式重复解析导致恢复 RPC 超时。
|
|
14
|
+
- 回调异常记录日志,后续调度继续;取消时调用原生 `destroy()` 释放计时器和注册表。
|
|
15
|
+
- 持久化无效任务在恢复时跳过并记录日志,无效订阅不会产生未处理的启动拒绝。API、导入和执行使用同一套表达式校验。
|
|
16
|
+
|
|
17
|
+
## 复审修复
|
|
18
|
+
|
|
19
|
+
- 移除没有生产调用的 `nextInvocation()` 接口,避免对 OR/# 等过滤规则报告不会实际执行的原生日期。回归测试直接观察实际执行和原生实例状态。
|
|
20
|
+
- 批量注册 RPC 时限为 5 秒加每条规则 5 毫秒,计入额外规则并封顶 120 秒;保留超时后的恢复机制,不立即重放结果不确定的写入。
|
|
21
|
+
- Alpine/系统模式下,数字起点步长(如 `0/5`)、星期包含 `7`、日期步长、日期和星期同时受限的表达式转入 Node 路径。BusyBox 的这些解析或通配语义与旧 Node 解析器不同,不能仅因是纯数字五字段就交给 crond。
|
|
22
|
+
- 这些规则在 `crontab.list` 中仍保留为注释,避免同时被系统和 Node 执行。其他可移植规则仍使用系统 crond。
|
|
23
|
+
|
|
24
|
+
## 夏令时差异
|
|
25
|
+
|
|
26
|
+
采用 node-cron 的当地时间语义:春季跳时不存在的时间跳过,秋季重复小时不再执行第二次。旧 cron-parser 对部分春季时间会顺延,并可能重复秋季高频任务。使用有夏令时的时区时需要考虑该行为变化;Asia/Shanghai 不受影响。回归测试固定 America/New_York 的 2026 年转换日期。
|
|
27
|
+
|
|
28
|
+
## 验证
|
|
29
|
+
|
|
30
|
+
`npm test` 和 `npm run build:back`。
|
|
31
|
+
|
|
32
|
+
- `test/fixtures/legacy-cron.json` 是迁移前使用 node-schedule 2.1.1 / cron-parser 4.9.0 保存的快照:269 条语法判定,18 组日历序列,每组 10 次执行。
|
|
33
|
+
- 真正的 node-cron 实例配合虚拟时钟验证日期/星期 OR、交集去重、闰年、L/#、8 秒迟到补执行、取消释放、回调拒绝和夏令时。
|
|
34
|
+
- 注册构造失败回滚、无效订阅隔离、空表达式立即运行,以及现有调度恢复/503/子进程测试覆盖接入流程。
|
|
35
|
+
- 隔离的正式 Debian 镜像使用实际编译产物验证 10,000 条有效任务和 2 条错误任务的恢复、300 个同时到期的 Python 脚本、并发上限 16 及 HTTP 健康检查。具体结果见本次测试报告;这是生产镜像隔离验证,不是线上长期运行证明。
|
|
36
|
+
|
|
37
|
+
### 本次验收结果(2026-09-27)
|
|
38
|
+
|
|
39
|
+
完整回归 260 通过、3 项原有跳过,后端编译成功。Node 20 生产运行时的 21 项兼容测试通过。隔离镜像中 300/300 脚本完成,无重复或遗漏,最大并发 16,270 个成功及 30 个预设失败均落库;完成耗时 52.36 秒。恢复后健康检查 1,870/1,870 成功,任务列表 38/38 成功。重启恢复阶段另有 50 次健康检查失败。
|
|
40
|
+
|
|
41
|
+
初版重复解析曾导致恢复超过 5 秒 RPC 时限,加入有上限的语法缓存后重跑通过。本次 10,000 个任务共享两个表达式,不能视为一万种复杂表达式的容量保证;实际脚本耗时也不代表纯调度性能。
|
|
42
|
+
|
|
43
|
+
新增宏在 UTC 和 Asia/Shanghai 下验证连续两次实际触发及不提前触发;原有语法快照仅对这三个新增宏允许预期变化。
|
|
44
|
+
|
|
45
|
+
### 复审后验收
|
|
46
|
+
|
|
47
|
+
完整回归 267 通过、3 项原有跳过;Node 20 的 10 项修复回归及 Alpine 镜像的 24 项兼容/宏测试通过。
|
|
48
|
+
|
|
49
|
+
额外使用 9,700 个不同表达式和 300 个同时到期任务验证恢复(另有 2 个无效任务),不再局限于共享两个表达式。300/300 脚本完成,无重复或遗漏,最大并发 16,270 个退出码 0、30 个预设退出码 3 均落库。恢复后健康检查 1,650/1,650 成功、任务列表 33/33 成功;启动恢复阶段另有 260 次健康检查失败,整批脚本耗时 60.27 秒。较大快照恢复仍需等待,不承诺启动阶段始终可用或任意规模都能在时限内恢复。
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# Task startup and notification loading
|
|
2
|
+
|
|
3
|
+
`task` performs shell configuration, status reporting and dependency-path discovery
|
|
4
|
+
before invoking the selected runtime. These costs are separate from cron callback
|
|
5
|
+
latency and time spent waiting for an execution slot.
|
|
6
|
+
|
|
7
|
+
## Dependency-path cache
|
|
8
|
+
|
|
9
|
+
Node scheduling usually starts in `/ql`, while Alpine crond starts in `/root`.
|
|
10
|
+
The pnpm discovery key includes the working directory, npm/pnpm configuration,
|
|
11
|
+
relevant environment and executable metadata. Each key now has its own cached
|
|
12
|
+
record, so alternating scheduler environments do not evict each other.
|
|
13
|
+
|
|
14
|
+
Records expire after 60 seconds. Refreshes share a persistent per-user lock;
|
|
15
|
+
records older than an hour are removed on refresh. Warm lookups do not scan the
|
|
16
|
+
cache directory. The lock is never removed while another process might use it.
|
|
17
|
+
The opt-out `QL_NODE_PATH_CACHE=0`, failed-discovery fallback, and package resolution
|
|
18
|
+
order remain available. This caches the dependency directory, not its contents.
|
|
19
|
+
|
|
20
|
+
## Language startup paths
|
|
21
|
+
|
|
22
|
+
| Invocation | Environment and before hooks | Automatic notification loading |
|
|
23
|
+
| --- | --- | --- |
|
|
24
|
+
| `task script.py` / `.pyc` | Python preload imports generated environment, runs the shell/command before hooks, imports `task_before.py`, then applies account selection | `QLAPI.notify(...)` imports `__ql_notify__` on first call |
|
|
25
|
+
| `task script.js` / `.mjs` / `.ts` | Node preload imports generated environment, runs shell/command before hooks, requires `task_before.js`, then applies account selection | `QLAPI.notify(...)` requires `__ql_notify__.js` on first call |
|
|
26
|
+
| `task script.sh` | Shell sources the generated environment and runs shell before hooks | No automatic Python/Node notification module |
|
|
27
|
+
| Explicit interpreter or other command, e.g. `task python3 script.py` | Retains generated environment and shell before-hook exports; it does not automatically install the Python/Node preload | No automatic notification module unless configured by the caller |
|
|
28
|
+
|
|
29
|
+
TypeScript uses `ts-node-transpile-only`; MJS uses the same Node preload plus the
|
|
30
|
+
ESM loader. There are no separate built-in Ruby, Go, Java or PHP preload modules.
|
|
31
|
+
All these task paths still pass through shared dependency-path setup: non-JS
|
|
32
|
+
scripts and their hooks may themselves call Node or use the QLAPI bridge.
|
|
33
|
+
|
|
34
|
+
The before hooks are deliberately still synchronous: they can export environment
|
|
35
|
+
variables or perform work required by the user's script. Python's preload still
|
|
36
|
+
starts a child shell/Python to capture that environment; Node's preload starts a
|
|
37
|
+
child shell/Node. Node also still loads its gRPC client eagerly. These are remaining
|
|
38
|
+
startup costs, not work performed by the cron library, and are not removed by this
|
|
39
|
+
change.
|
|
40
|
+
|
|
41
|
+
Shell scripts run in a child shell that inherits the wrapper's environment and
|
|
42
|
+
before-hook functions. This lets the wrapper run its after hooks and report the
|
|
43
|
+
exit status even when a script calls `exit`, replaces itself with `exec`, or fails
|
|
44
|
+
under `set -e`. The script's own EXIT trap still runs in the child. Changes made
|
|
45
|
+
inside the script (variables, working directory, functions and shell options)
|
|
46
|
+
do not propagate back to the wrapper or its after hooks.
|
|
47
|
+
|
|
48
|
+
## Lazy notifications
|
|
49
|
+
|
|
50
|
+
Scripts keep calling `QLAPI.notify` with the same arguments. When no notification
|
|
51
|
+
is sent, the notification provider and its channel dependencies are never imported
|
|
52
|
+
by the preload. First use loads the provider; subsequent uses reuse the standard
|
|
53
|
+
Python/Node module cache. JS forwards the receiver and return value, including
|
|
54
|
+
promises, and Python forwards positional and keyword arguments. Import/send errors
|
|
55
|
+
surface at the notification call rather than preventing unrelated scripts from
|
|
56
|
+
obtaining QLAPI during startup. A failed import can be retried after its cause is
|
|
57
|
+
resolved.
|
|
58
|
+
|
|
59
|
+
Provider initialization, including provider configuration reads, now happens at
|
|
60
|
+
first use. A script's explicit `import __ql_notify__`, `require(...)`, or independent
|
|
61
|
+
notification helper still loads that module immediately.
|
|
62
|
+
|
|
63
|
+
## Task timeouts
|
|
64
|
+
|
|
65
|
+
`task -m DURATION ...` also applies to sourced Shell scripts. The timeout worker
|
|
66
|
+
inherits before-hook variables, arrays and functions without serializing them.
|
|
67
|
+
At the deadline, the worker's process group receives TERM, followed by KILL after
|
|
68
|
+
0.5 seconds, so ordinary child processes cannot continue after the timeout.
|
|
69
|
+
Explicit interpreters and other commands use the same mechanism. Processes that
|
|
70
|
+
intentionally create a separate session/process group are outside this boundary.
|
|
71
|
+
The wrapper records status 124 and runs its after hook; concurrent account mode
|
|
72
|
+
retains its existing aggregate status behavior. Zero disables the timeout.
|
|
73
|
+
|
|
74
|
+
Cron executions waiting for a concurrency slot retain their schedule revision.
|
|
75
|
+
Disabling, deleting or replacing that revision prevents its queued executions
|
|
76
|
+
from starting when a slot becomes available. Already running executions retain
|
|
77
|
+
the existing stop behavior. Invalidated callbacks do not count against a new
|
|
78
|
+
revision's limit of five pending/running executions.
|
package/docs/releases/2.22.0.md
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# 青龙 2.22.0
|
|
2
2
|
|
|
3
|
+
## 2026-09-27 覆盖更新
|
|
4
|
+
|
|
5
|
+
- 修复历史任务中合法的 `?` 定时表达式被拒绝,导致启动恢复失败、面板健康检查返回 503 的问题(#3078)。主规则和附加规则均保留兼容性,无效规则仍在替换现有任务前拒绝。
|
|
6
|
+
- 修复 `ql check` 等 Shell 诊断输出以 `--` 开头时的 `printf: --: invalid option`,补全调度注册失败日志中的错误详情。
|
|
7
|
+
- 启动恢复跳过无效主规则或附加规则的任务,记录任务 ID、名称和错误规则;数据库原记录保留可编辑,其他任务与面板正常启动。服务连接故障仍返回 503。
|
|
8
|
+
- 校验使用与实际调度器一致的解析器,避免接受 `H` 等无法执行的语法;五字段扩展规则、宏和历史简写交给 Node 调度器,避免系统 crontab 拒绝整个任务快照。
|
|
9
|
+
- 已对 250 多个表达式样本与真实调度库做一致性校验,验证无效任务隔离和两种调度模式;本地回归 240 项,237 通过、3 项平台相关跳过,后端构建通过。
|
|
10
|
+
|
|
3
11
|
## 新增功能
|
|
4
12
|
|
|
5
13
|
- 独立远程命令行包 `@whyour/qinglong-cli`:覆盖任务、订阅、环境变量、配置、脚本、日志、依赖、应用、系统等 OpenAPI,支持认证、JSON 输出和中英文帮助。
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# 青龙 2.23.0
|
|
2
|
+
|
|
3
|
+
## 新增功能
|
|
4
|
+
|
|
5
|
+
- 脚本历史版本:查看、对比和恢复历史内容;每个脚本保留最多 20 个版本,支持不超过 1 MiB 的 UTF-8 文本脚本。恢复前校验当前内容哈希,发生并发修改时返回冲突,避免覆盖新内容。
|
|
6
|
+
- 独立远程 CLI 0.1.2 增加脚本历史查看、对比和恢复命令,并完善资源冲突及认证错误反馈。最低要求仍为 Node.js 22.12。
|
|
7
|
+
|
|
8
|
+
## 修复与优化
|
|
9
|
+
|
|
10
|
+
- 普通任务、额外规则和订阅的 Node 调度路径迁移到 node-cron。保留旧规则标准化、日期/星期 OR、L/# 等兼容行为,补充 `@annually`、`@midnight`、`@minutely`。
|
|
11
|
+
- 禁用、删除或修改任务后,旧规则对应的排队执行不会再启动。完善任务进程超时、停止回收、Shell before/after 环境传递和退出状态。
|
|
12
|
+
- 修复系统令牌未主动续期和认证缓存并发更新:定时生成与启动后重新计时会主动续期,避免长期运行后失效。Shell API 每次请求检查令牌,遇到 401 时刷新并重试一次;其他错误不自动重试。
|
|
13
|
+
- 降低任务启动开销;JS/Python 的 QLAPI 通知依赖在首次通知调用时加载,减少不发送通知的任务启动成本。
|
|
14
|
+
- 加固环境变量名和字面值、订阅参数、路径访问及 SSH 配置输入;保留合法的中文及分支别名兼容。
|
|
15
|
+
- 修复页面升级、`ql update` 和静态资源的版本同步,完善升级暂存校验及应用、完整备份与恢复流程。
|
|
16
|
+
- 修复 Debian 上的依赖安装、通知和调度回归,补齐前端缺失的中英文翻译,改进 Gitee/GitLab 静态资源镜像同步检查。
|
|
17
|
+
- 最小升级 compression、multer、@grpc/grpc-js、proxy-addr、websocket-driver、sequelize,修补现有 HTTP、上传、代理信任、WebSocket、gRPC 和 JSON 查询路径相关风险。完整依赖治理、Node/pnpm 迁移及工具打包调整延期。
|
|
18
|
+
|
|
19
|
+
## 升级说明
|
|
20
|
+
|
|
21
|
+
升级前停止面板与任务,备份完整持久化数据目录,包括数据库、脚本、配置、日志、依赖缓存及已有的 `script-history` 目录,并记录原镜像摘要和启动配置。使用面板备份接口时,确认选择了脚本、配置和日志;仅备份数据库不能恢复全部数据。
|
|
22
|
+
|
|
23
|
+
升级后试运行代表性的 JavaScript、Python、Shell 任务,检查订阅、环境变量、依赖安装以及通知。存在夏令时的时区采用 node-cron 行为:跳过春季不存在的当地时间,秋季重复小时不重复执行;Asia/Shanghai 不受影响。调度器不提供进程退出期间的持久化补跑保证。
|
|
24
|
+
|
|
25
|
+
任务超时会终止普通子进程所在的进程组并记录退出状态 124;主动创建独立会话/进程组的程序不在该边界内。并发账号执行模式保留既有汇总退出状态。
|
|
26
|
+
|
|
27
|
+
本次沿用现有 Dockerfile 和 pnpm 8.3.1。默认 Debian 镜像使用 Debian 12 与 Node.js 20,默认 Alpine 镜像的 Node.js 由 APK 提供;默认 Python 为 3.11。Node 20 与 Node 18 构建运行时已结束上游维护,相关迁移延期,功能验证通过不代表全部依赖风险清零。
|
|
28
|
+
|
|
29
|
+
2.23.0 是 Python 3.10 兼容镜像的最后一个版本。`2.23.0-python3.10` 和 `2.23.0-debian-python3.10` 发布后停止维护,不再提供后续更新;请验证脚本和依赖后迁移至默认 Python 3.11 镜像。
|
|
30
|
+
|
|
31
|
+
主包 `@whyour/qinglong` 与独立远程包 `@whyour/qinglong-cli` 的 `ql` 命令用途不同。在已有面板的主机上使用独立 npm prefix 或 `npm exec --package=@whyour/qinglong-cli -- ql ...`,避免覆盖面板内置入口。
|
|
32
|
+
|
|
33
|
+
发布时间:2026-10-11 12:27:02(Asia/Shanghai)。本版本已完成四类镜像验证及独立 dev 环境超过两小时的持续功能验收。
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# 脚本编辑历史
|
|
2
|
+
|
|
3
|
+
脚本管理工具栏的“历史版本”打开右侧抽屉。选择版本后,对比历史内容与服务器上当前已保存文件;桌面使用双栏对比,手机使用行内对比。恢复需要确认,有未保存修改时必须先保存或取消。恢复前保留当前版本,因此恢复操作也可以撤销。普通保存不再弹出不可恢复确认。
|
|
4
|
+
|
|
5
|
+
## 记录范围
|
|
6
|
+
|
|
7
|
+
- 编辑器 `PUT /api/scripts` 和调试窗口 `POST /api/scripts` 覆盖已有脚本时,在内容变化时留档,首次修改前的内容也保留;相同内容不写历史。
|
|
8
|
+
- 历史只包含保存前真实存在的内容;不把失败写入当成一个内容版本。
|
|
9
|
+
- 上传、新增、重命名、删除、订阅更新和外部编辑不主动留档。若下次面板保存时检测到外部变更,会保留那一刻的内容,标记为“面板外修改后”。这不代表外部修改之前的内容受到保护。
|
|
10
|
+
- 按脚本规范化后的相对路径隔离历史;不同目录同名文件不会混淆,指向同一脚本的安全符号链接共用历史。重命名后不会自动迁移历史。调试器另存为覆盖已有脚本时,目标旧内容进入历史,删除源文件前将其旧内容保留到现有 bak 目录;备份失败则不覆盖目标、不删除源文件。源和目标实际指向同一脚本时(如安全符号链接或不同路径写法),按原地保存处理,保留两个路径,不删除源文件。历史不能用于回收已删除的文件。
|
|
11
|
+
- 历史记录支持至多 1 MiB 的 UTF-8 文本文件(含空文件)。当前文件或新内容超出范围时,界面要求明确确认“不记录历史直接保存”;取消则保持文件和草稿不变。确认后以 `skipHistory: true` 和服务端返回的当前哈希重试,锁内检查哈希以防确认期间覆盖他人修改。成功后明确提示本次未留档。大文件可以通过此流程缩小,后续恢复普通留档。未确认的 API 请求返回 413,并包含 `historyUnavailable` 和 `currentHash`;历史读取损坏等其他错误不允许用此选项绕过。
|
|
12
|
+
|
|
13
|
+
## 存储与资源边界
|
|
14
|
+
|
|
15
|
+
位置:`<QL_DATA_DIR>/script-history/<SHA-256(规范相对路径)>.br`。目录位于脚本目录之外,避免进入脚本树、订阅和任务扫描。目录权限 `0700`,历史文件权限 `0600`。目录随整个数据目录备份。
|
|
16
|
+
|
|
17
|
+
每个脚本仅一个 Brotli 压缩 JSON 包,存储完整的旧内容快照以及时间、来源、内容哈希。新文件内容不再重复存储,只保存头版本元数据,供下一次留档标注来源。面板服务通过 TypeDI 注册为单例,路由使用 `Container.get(ScriptHistoryService)` 获取;构造实例只初始化配置,首次发生保存时才创建历史目录。包内重复文本共享压缩窗口,不依赖补丁链,也不需要额外数据库、文件监听、后台任务或常驻缓存。使用 Node 内置 zlib 异步压缩,Brotli quality=4、window=4 MiB,不新增依赖。
|
|
18
|
+
|
|
19
|
+
| 限制 | 默认值 |
|
|
20
|
+
| --- | --- |
|
|
21
|
+
| 每脚本历史版本 | 最多 20 个 |
|
|
22
|
+
| 单个脚本内容 | 1 MiB |
|
|
23
|
+
| 单包解压后的 JSON | 8 MiB |
|
|
24
|
+
| 单个压缩历史包 | 2 MiB |
|
|
25
|
+
| 所有压缩历史包合计 | 32 MiB |
|
|
26
|
+
| 有历史的脚本数 | 最多 1000 个 |
|
|
27
|
+
| 历史写入暂存文件 | 共享一个,最多 2 MiB |
|
|
28
|
+
| 脚本写入暂存文件 | 每次保存一个,与本次保存内容等大,结束后清理 |
|
|
29
|
+
|
|
30
|
+
达到单文件限制时删除最旧版本,至少保留本次覆盖前的内容;无法保留则拒绝覆盖。达到全局限制时,在脚本成功提交后优先清理最久未更新的脚本历史包(该文件的整份历史),不删除脚本本身。历史暂存、提交或脚本提交失败时不清理任何其他历史;脚本提交失败会回滚本文件历史。如果提交成功但清理失败,返回 `cleanupPending`,界面提示检查存储权限和空间,后续保存重新尝试清理。配额是正常清理成功后的目标,事务期间和清理失败时会暂时超出。32 MiB 是文件内容字节数,不包含文件系统块分配、目录和锁的开销。清理只在发生保存时执行,通常最多扫描 1000 个包的元数据(清理失败期间可能更多),不解压其他脚本的历史。读取仅解压当前脚本,限制解压输出长度。
|
|
31
|
+
|
|
32
|
+
文件保存复用现有锁路径,同时以独立跨进程目录锁串行化历史写入和配额清理;`finally` 释放锁。先准备并同步脚本暂存文件,保留文件权限和所有者;随后写并同步历史暂存文件、原子提交历史,再原子替换脚本。任一步准备失败不会截断原脚本;最后替换脚本失败时还原此前历史;成功后才清理配额。超限确认保存不创建新历史包,使用同样的脚本暂存和替换方式,并流式计算旧文件哈希。外部进程不遵守该锁时,不承诺与其并发写入的事务一致性。
|
|
33
|
+
|
|
34
|
+
恢复请求携带预览时的当前内容哈希,锁内检查后才写入;文件变化则返回 409。历史读写与脚本读取采用相同的路径和符号链接边界检查。面板配置目录中的敏感文件仍受黑名单保护;脚本目录中的 auth.json、token.json 等普通脚本名称可以正常保存和留档。
|
|
35
|
+
|
|
36
|
+
## 压缩样本
|
|
37
|
+
|
|
38
|
+
运行 `node scripts/benchmark-script-history.cjs` 可复现比较。模拟同一脚本连续 20 次小修改,比较独立 gzip、合并 gzip 与合并 Brotli。首次本机测量:
|
|
39
|
+
|
|
40
|
+
| 样本 | 独立 gzip | 合并 gzip | 合并 Brotli(quality 4) |
|
|
41
|
+
| --- | ---: | ---: | ---: |
|
|
42
|
+
| sample/notify.js | 242,430 B | 221,832 B | 13,930 B |
|
|
43
|
+
| shell/task.sh | 30,159 B | 2,317 B | 1,737 B |
|
|
44
|
+
|
|
45
|
+
这是文本重复度较高的模拟样本,不是所有工作负载的压缩保证。Brotli 合包特别适合大小超过 gzip 32 KiB 窗口、但每次只改少量内容的脚本。增量补丁可能在部分样本更小,但增加补丁生成、链式恢复、裁剪和损坏恢复的复杂度;本实现选择压缩合包与硬上限,在磁盘、CPU 和可靠性之间取平衡。
|
|
46
|
+
|
|
47
|
+
## API
|
|
48
|
+
|
|
49
|
+
- `GET /api/scripts/history?filename=...&path=...`:版本元数据、当前哈希、保留数量。
|
|
50
|
+
- `GET /api/scripts/history/detail?filename=...&path=...&id=...`:历史内容、当前内容及当前哈希。
|
|
51
|
+
- `PUT /api/scripts/history/restore`:`{filename, path, id, expectedHash}`。
|
|
52
|
+
|
|
53
|
+
沿用现有 scripts 路由的认证与权限。历史 ID 使用 UUID;不存在或已清理返回 404,文件已变更返回 409,文本或容量限制返回 413。
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@whyour/qinglong",
|
|
3
3
|
"packageManager": "pnpm@8.3.1",
|
|
4
|
-
"version": "2.
|
|
4
|
+
"version": "2.23.0",
|
|
5
5
|
"description": "Timed task management platform supporting Python3, JavaScript, Shell, Typescript",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
@@ -74,20 +74,22 @@
|
|
|
74
74
|
"overrides": {
|
|
75
75
|
"sqlite3": "npm:@whyour/sqlite3@1.1.2",
|
|
76
76
|
"@codemirror/state": "6.5.4",
|
|
77
|
-
"@codemirror/view": "6.39.16"
|
|
77
|
+
"@codemirror/view": "6.39.16",
|
|
78
|
+
"proxy-addr": "2.0.8",
|
|
79
|
+
"websocket-driver": "0.7.5"
|
|
78
80
|
}
|
|
79
81
|
},
|
|
80
82
|
"dependencies": {
|
|
81
83
|
"@ant-design/plots": "^2.6.8",
|
|
82
84
|
"@bufbuild/protobuf": "^2.10.0",
|
|
83
|
-
"@grpc/grpc-js": "
|
|
85
|
+
"@grpc/grpc-js": "1.14.5",
|
|
84
86
|
"@grpc/proto-loader": "^0.8.0",
|
|
85
87
|
"@keyv/sqlite": "^4.0.1",
|
|
86
88
|
"@otplib/preset-default": "^12.0.1",
|
|
87
89
|
"body-parser": "^1.20.3",
|
|
88
90
|
"celebrate": "^15.0.3",
|
|
89
91
|
"chokidar": "^4.0.1",
|
|
90
|
-
"compression": "
|
|
92
|
+
"compression": "1.8.2",
|
|
91
93
|
"cors": "^2.8.5",
|
|
92
94
|
"cron-parser": "^5.4.0",
|
|
93
95
|
"cross-spawn": "^7.0.6",
|
|
@@ -106,15 +108,15 @@
|
|
|
106
108
|
"jsonwebtoken": "^9.0.2",
|
|
107
109
|
"keyv": "^5.2.3",
|
|
108
110
|
"lodash": "^4.17.21",
|
|
109
|
-
"multer": "2.
|
|
110
|
-
"node-
|
|
111
|
+
"multer": "2.3.0",
|
|
112
|
+
"node-cron": "4.6.0",
|
|
111
113
|
"nodemailer": "^8.0.1",
|
|
112
114
|
"p-queue-cjs": "7.3.4",
|
|
113
115
|
"proper-lockfile": "^4.1.2",
|
|
114
116
|
"ps-tree": "^1.2.0",
|
|
115
117
|
"reflect-metadata": "^0.2.2",
|
|
116
118
|
"request-ip": "3.3.0",
|
|
117
|
-
"sequelize": "
|
|
119
|
+
"sequelize": "6.37.8",
|
|
118
120
|
"sockjs": "^0.3.24",
|
|
119
121
|
"sqlite3": "npm:@whyour/sqlite3@1.1.2",
|
|
120
122
|
"toad-scheduler": "^3.0.1",
|
|
@@ -144,7 +146,6 @@
|
|
|
144
146
|
"@types/lodash": "^4.14.185",
|
|
145
147
|
"@types/multer": "^1.4.7",
|
|
146
148
|
"@types/node": "^17.0.21",
|
|
147
|
-
"@types/node-schedule": "^1.3.2",
|
|
148
149
|
"@types/nodemailer": "^6.4.4",
|
|
149
150
|
"@types/proper-lockfile": "^4.1.4",
|
|
150
151
|
"@types/ps-tree": "^1.1.6",
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
// Reproducible storage comparison, without touching actual script history.
|
|
2
|
+
const fs = require('node:fs');
|
|
3
|
+
const path = require('node:path');
|
|
4
|
+
const zlib = require('node:zlib');
|
|
5
|
+
for (const file of ['sample/notify.js', 'shell/task.sh']) {
|
|
6
|
+
const source = fs.readFileSync(path.join(__dirname, '..', file), 'utf8');
|
|
7
|
+
const versions = Array.from({ length: 20 }, (_, i) => ({
|
|
8
|
+
content: source + '\n// edit ' + i,
|
|
9
|
+
}));
|
|
10
|
+
const raw = Buffer.from(JSON.stringify(versions));
|
|
11
|
+
const strategies = {
|
|
12
|
+
gzipSeparate: () =>
|
|
13
|
+
versions.reduce(
|
|
14
|
+
(sum, version) => sum + zlib.gzipSync(version.content).length,
|
|
15
|
+
0,
|
|
16
|
+
),
|
|
17
|
+
gzipBundle: () => zlib.gzipSync(raw).length,
|
|
18
|
+
brotliBundle: () =>
|
|
19
|
+
zlib.brotliCompressSync(raw, {
|
|
20
|
+
params: {
|
|
21
|
+
[zlib.constants.BROTLI_PARAM_QUALITY]: 4,
|
|
22
|
+
[zlib.constants.BROTLI_PARAM_LGWIN]: 22,
|
|
23
|
+
},
|
|
24
|
+
}).length,
|
|
25
|
+
};
|
|
26
|
+
for (const [strategy, compress] of Object.entries(strategies)) {
|
|
27
|
+
const start = performance.now();
|
|
28
|
+
const bytes = compress();
|
|
29
|
+
console.log(
|
|
30
|
+
JSON.stringify({
|
|
31
|
+
file,
|
|
32
|
+
strategy,
|
|
33
|
+
rawBytes: raw.length,
|
|
34
|
+
bytes,
|
|
35
|
+
ms: +(performance.now() - start).toFixed(2),
|
|
36
|
+
}),
|
|
37
|
+
);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
destination=$1
|
|
4
|
+
ref=$2
|
|
5
|
+
ref_type=$3
|
|
6
|
+
git check-ref-format "refs/heads/$ref"
|
|
7
|
+
script_dir=$(cd "$(dirname "$0")" && pwd)
|
|
8
|
+
# Publish only a complete artifact from the checked-out source commit.
|
|
9
|
+
node "$script_dir/../docker/verify-build.cjs"
|
|
10
|
+
if [[ "$ref_type" == 'tag' ]]; then
|
|
11
|
+
version=$(awk '$1 == "version:" { print $2; exit }' version.yaml)
|
|
12
|
+
if [[ "$ref" != "v$version" ]]; then
|
|
13
|
+
echo "::error::Release tag $ref does not match version.yaml ($version)"
|
|
14
|
+
exit 1
|
|
15
|
+
fi
|
|
16
|
+
elif [[ "$ref_type" != 'branch' ]]; then
|
|
17
|
+
echo "::error::Unsupported ref type: $ref_type"
|
|
18
|
+
exit 1
|
|
19
|
+
fi
|
|
20
|
+
workspace=$(mktemp -d)
|
|
21
|
+
trap 'rm -rf "$workspace"' EXIT
|
|
22
|
+
cp -a static/. "$workspace/"
|
|
23
|
+
git init -b "$ref" "$workspace"
|
|
24
|
+
git -C "$workspace" config user.name 'github-actions[bot]'
|
|
25
|
+
git -C "$workspace" config user.email 'github-actions[bot]@users.noreply.github.com'
|
|
26
|
+
git -C "$workspace" add -f .
|
|
27
|
+
git -C "$workspace" commit --allow-empty -m "Build static from $(git rev-parse HEAD)"
|
|
28
|
+
refs=("refs/heads/$ref:refs/heads/$ref")
|
|
29
|
+
if [[ "$ref_type" == 'tag' ]]; then
|
|
30
|
+
# Keep version branches for existing archive URLs and publish real Git tags.
|
|
31
|
+
git -C "$workspace" tag "$ref"
|
|
32
|
+
refs+=("refs/tags/$ref:refs/tags/$ref")
|
|
33
|
+
fi
|
|
34
|
+
git -C "$workspace" push --force --quiet "$destination" "${refs[@]}"
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Static branches are orphan snapshots. Fetch complete commits for the branch
|
|
3
|
+
# that was built: a shallow/default-branch clone is rejected by mirror servers.
|
|
4
|
+
set -euo pipefail
|
|
5
|
+
destination=$1
|
|
6
|
+
branch=$2
|
|
7
|
+
ref_type=${3:-branch}
|
|
8
|
+
git check-ref-format "refs/heads/$branch"
|
|
9
|
+
workspace=$(mktemp -d)
|
|
10
|
+
trap 'rm -rf "$workspace"' EXIT
|
|
11
|
+
git init --bare "$workspace"
|
|
12
|
+
fetch_refs=("+refs/heads/$branch:refs/heads/$branch")
|
|
13
|
+
push_refs=("refs/heads/$branch:refs/heads/$branch")
|
|
14
|
+
if [[ "$ref_type" == 'tag' ]]; then
|
|
15
|
+
fetch_refs+=("+refs/tags/$branch:refs/tags/$branch")
|
|
16
|
+
push_refs+=("refs/tags/$branch:refs/tags/$branch")
|
|
17
|
+
fi
|
|
18
|
+
git -C "$workspace" fetch --no-tags "${QL_STATIC_SOURCE:-https://github.com/whyour/qinglong-static.git}" \
|
|
19
|
+
"${fetch_refs[@]}"
|
|
20
|
+
git -C "$workspace" push --force "$destination" "${push_refs[@]}"
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Observe Pull mirrors; optionally notify them through a small GitHub push.
|
|
3
|
+
set -euo pipefail
|
|
4
|
+
|
|
5
|
+
if [[ $# -lt 3 || $# -gt 4 ]]; then
|
|
6
|
+
echo 'Usage: wait-for-mirror.sh SOURCE MIRROR REF [branch|tag]' >&2
|
|
7
|
+
exit 2
|
|
8
|
+
fi
|
|
9
|
+
source=$1
|
|
10
|
+
mirror=$2
|
|
11
|
+
ref=$3
|
|
12
|
+
ref_type=${4:-branch}
|
|
13
|
+
wait_seconds=${QL_MIRROR_WAIT_SECONDS:-600}
|
|
14
|
+
poll_interval=${QL_MIRROR_POLL_INTERVAL:-30}
|
|
15
|
+
query_timeout=${QL_MIRROR_QUERY_TIMEOUT:-20}
|
|
16
|
+
include_version_branch=${QL_MIRROR_INCLUDE_VERSION_BRANCH:-false}
|
|
17
|
+
notify_repo=${QL_MIRROR_NOTIFY_REPO:-}
|
|
18
|
+
retry_after=${QL_MIRROR_RETRY_AFTER_SECONDS:-310}
|
|
19
|
+
notify_on_start=${QL_MIRROR_NOTIFY_ON_START:-true}
|
|
20
|
+
|
|
21
|
+
timings=("$wait_seconds" "$poll_interval" "$query_timeout")
|
|
22
|
+
if [[ -n "$notify_repo" ]]; then timings+=("$retry_after"); fi
|
|
23
|
+
for value in "${timings[@]}"; do
|
|
24
|
+
if [[ ! "$value" =~ ^[1-9][0-9]{0,4}$ ]] || (( value > 86400 )); then
|
|
25
|
+
echo 'Mirror timing values must be integers between 1 and 86400 seconds' >&2
|
|
26
|
+
exit 2
|
|
27
|
+
fi
|
|
28
|
+
done
|
|
29
|
+
case "$ref_type" in
|
|
30
|
+
branch) refs=("refs/heads/$ref") ;;
|
|
31
|
+
tag)
|
|
32
|
+
refs=("refs/tags/$ref")
|
|
33
|
+
if [[ "$include_version_branch" == true ]]; then
|
|
34
|
+
refs+=("refs/heads/$ref")
|
|
35
|
+
fi
|
|
36
|
+
;;
|
|
37
|
+
*) echo 'Mirror ref type must be branch or tag' >&2; exit 2 ;;
|
|
38
|
+
esac
|
|
39
|
+
if [[ "$include_version_branch" != true && "$include_version_branch" != false ]]; then
|
|
40
|
+
echo 'QL_MIRROR_INCLUDE_VERSION_BRANCH must be true or false' >&2
|
|
41
|
+
exit 2
|
|
42
|
+
fi
|
|
43
|
+
if [[ "$notify_on_start" != true && "$notify_on_start" != false ]]; then
|
|
44
|
+
echo 'QL_MIRROR_NOTIFY_ON_START must be true or false' >&2
|
|
45
|
+
exit 2
|
|
46
|
+
fi
|
|
47
|
+
for requested_ref in "${refs[@]}"; do
|
|
48
|
+
git check-ref-format "$requested_ref"
|
|
49
|
+
done
|
|
50
|
+
command -v timeout >/dev/null || { echo 'GNU timeout is required' >&2; exit 2; }
|
|
51
|
+
if [[ -n "$notify_repo" ]]; then
|
|
52
|
+
if [[ ! "$notify_repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then
|
|
53
|
+
echo 'QL_MIRROR_NOTIFY_REPO must be a GitHub owner/repository' >&2
|
|
54
|
+
exit 2
|
|
55
|
+
fi
|
|
56
|
+
command -v gh >/dev/null || { echo 'GitHub CLI is required for notifications' >&2; exit 2; }
|
|
57
|
+
fi
|
|
58
|
+
export GIT_TERMINAL_PROMPT=0
|
|
59
|
+
polling=false
|
|
60
|
+
|
|
61
|
+
notification_command() {
|
|
62
|
+
local budget=$((notification_deadline - SECONDS))
|
|
63
|
+
if (( budget <= 0 )); then return 1; fi
|
|
64
|
+
if (( budget > 10 )); then budget=10; fi
|
|
65
|
+
timeout --signal=KILL "${budget}s" "$@"
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
notify_static_mirror() {
|
|
69
|
+
local notification_deadline=$((SECONDS + 30))
|
|
70
|
+
if [[ "$polling" == true ]] && (( notification_deadline > wait_seconds )); then
|
|
71
|
+
notification_deadline=$wait_seconds
|
|
72
|
+
fi
|
|
73
|
+
local trigger_ref=refs/heads/gitee-sync-trigger
|
|
74
|
+
local attempt remote_refs source_sha parent tree commit delay remaining
|
|
75
|
+
local commit_args
|
|
76
|
+
if ! remote_refs=$(notification_command git ls-remote --refs "$source" "${refs[0]}"); then
|
|
77
|
+
echo '::warning::Could not read the published source; continuing mirror verification'
|
|
78
|
+
return 1
|
|
79
|
+
fi
|
|
80
|
+
source_sha=${remote_refs%%$'\t'*}
|
|
81
|
+
if [[ ! "$source_sha" =~ ^[0-9a-f]{40}$ ]]; then
|
|
82
|
+
echo "::error::Published source ref ${refs[0]} does not exist"
|
|
83
|
+
return 2
|
|
84
|
+
fi
|
|
85
|
+
for attempt in 1 2 3; do
|
|
86
|
+
if (( attempt > 1 )); then
|
|
87
|
+
delay=$((attempt * 2 - 2))
|
|
88
|
+
remaining=$((notification_deadline - SECONDS))
|
|
89
|
+
if (( remaining <= delay )); then break; fi
|
|
90
|
+
sleep "$delay"
|
|
91
|
+
fi
|
|
92
|
+
# Refresh the parent on every attempt. REST avoids the Git push endpoint's
|
|
93
|
+
# transient 500s, and non-forced updates preserve concurrent notifications.
|
|
94
|
+
if parent=$(notification_command gh api "repos/$notify_repo/git/ref/heads/gitee-sync-trigger" --jq '.object.sha' 2>&1); then
|
|
95
|
+
if ! tree=$(notification_command gh api "repos/$notify_repo/git/commits/$parent" --jq '.tree.sha'); then
|
|
96
|
+
echo "::warning::Could not read notification history (attempt $attempt)"
|
|
97
|
+
continue
|
|
98
|
+
fi
|
|
99
|
+
elif [[ "$parent" == *'(HTTP 404)'* ]]; then
|
|
100
|
+
parent=''
|
|
101
|
+
# A new notification branch starts with one tiny marker, never artifacts.
|
|
102
|
+
# Later commits reuse this tree and have no changed files in the webhook.
|
|
103
|
+
if ! tree=$(printf '%s' '{"tree":[{"path":".gitee-sync-trigger","mode":"100644","type":"blob","content":"Gitee Pull mirror notifications.\n"}]}' |
|
|
104
|
+
notification_command gh api --method POST "repos/$notify_repo/git/trees" --input - --jq '.sha'); then
|
|
105
|
+
echo "::warning::Could not create notification tree (attempt $attempt)"
|
|
106
|
+
continue
|
|
107
|
+
fi
|
|
108
|
+
else
|
|
109
|
+
echo "::warning::Could not read notification ref (attempt $attempt)"
|
|
110
|
+
continue
|
|
111
|
+
fi
|
|
112
|
+
commit_args=(-f "tree=$tree" -f "message=Trigger Gitee mirror for ${refs[0]} at $source_sha")
|
|
113
|
+
if [[ -n "$parent" ]]; then commit_args+=(-f "parents[]=$parent"); fi
|
|
114
|
+
if ! commit=$(notification_command gh api --method POST "repos/$notify_repo/git/commits" "${commit_args[@]}" --jq '.sha'); then
|
|
115
|
+
echo "::warning::Could not create notification commit (attempt $attempt)"
|
|
116
|
+
continue
|
|
117
|
+
fi
|
|
118
|
+
if [[ -n "$parent" ]]; then
|
|
119
|
+
if notification_command gh api --method PATCH "repos/$notify_repo/git/refs/heads/gitee-sync-trigger" -f "sha=$commit" -F force=false --silent; then
|
|
120
|
+
echo '::notice::Sent a small GitHub push event via REST to trigger the Gitee Pull mirror'
|
|
121
|
+
return 0
|
|
122
|
+
fi
|
|
123
|
+
elif notification_command gh api --method POST "repos/$notify_repo/git/refs" -f "ref=$trigger_ref" -f "sha=$commit" --silent; then
|
|
124
|
+
echo '::notice::Sent a small GitHub push event via REST to trigger the Gitee Pull mirror'
|
|
125
|
+
return 0
|
|
126
|
+
fi
|
|
127
|
+
echo "::warning::Mirror notification update failed (attempt $attempt)"
|
|
128
|
+
done
|
|
129
|
+
echo '::warning::Could not send the mirror notification; continuing mirror verification'
|
|
130
|
+
return 1
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
if [[ -n "$notify_repo" && "$notify_on_start" == true ]]; then
|
|
134
|
+
if notify_static_mirror; then :; elif [[ $? -eq 2 ]]; then exit 1; fi
|
|
135
|
+
fi
|
|
136
|
+
|
|
137
|
+
SECONDS=0
|
|
138
|
+
polling=true
|
|
139
|
+
attempt=0
|
|
140
|
+
retried=false
|
|
141
|
+
last_source='(not read)'
|
|
142
|
+
last_mirror='(not read)'
|
|
143
|
+
while (( SECONDS < wait_seconds )); do
|
|
144
|
+
attempt=$((attempt + 1))
|
|
145
|
+
outputs=()
|
|
146
|
+
ready=true
|
|
147
|
+
# Re-read the source after the mirror so a moving source cannot pass with
|
|
148
|
+
# the previous SHA. --refs compares annotated tag objects as well as branches.
|
|
149
|
+
urls=("$source" "$mirror" "$source")
|
|
150
|
+
labels=('source' 'mirror' 'source confirmation')
|
|
151
|
+
for index in 0 1 2; do
|
|
152
|
+
remaining=$((wait_seconds - SECONDS))
|
|
153
|
+
if (( remaining <= 0 )); then
|
|
154
|
+
ready=false
|
|
155
|
+
break
|
|
156
|
+
fi
|
|
157
|
+
budget=$query_timeout
|
|
158
|
+
if (( budget > remaining )); then budget=$remaining; fi
|
|
159
|
+
if ! output=$(timeout --signal=KILL "${budget}s" git ls-remote --refs \
|
|
160
|
+
"${urls[$index]}" "${refs[@]}" | LC_ALL=C sort); then
|
|
161
|
+
echo "::warning::Could not read ${labels[$index]} refs on attempt $attempt; retrying"
|
|
162
|
+
ready=false
|
|
163
|
+
break
|
|
164
|
+
fi
|
|
165
|
+
if (( index == 0 )); then last_source=$output; fi
|
|
166
|
+
if (( index == 1 )); then last_mirror=$output; fi
|
|
167
|
+
for requested_ref in "${refs[@]}"; do
|
|
168
|
+
found=false
|
|
169
|
+
while IFS=$'\t' read -r sha returned_ref; do
|
|
170
|
+
if [[ "$returned_ref" == "$requested_ref" && "$sha" =~ ^[0-9a-f]{40}([0-9a-f]{24})?$ ]]; then
|
|
171
|
+
found=true
|
|
172
|
+
fi
|
|
173
|
+
done <<< "$output"
|
|
174
|
+
if [[ "$found" == false ]]; then
|
|
175
|
+
echo "::notice::${labels[$index]} is missing $requested_ref; waiting"
|
|
176
|
+
ready=false
|
|
177
|
+
break
|
|
178
|
+
fi
|
|
179
|
+
done
|
|
180
|
+
if [[ "$ready" == false ]]; then break; fi
|
|
181
|
+
outputs+=("$output")
|
|
182
|
+
# Only confirm a source when the initial source and mirror reads match.
|
|
183
|
+
if (( index == 1 )) && [[ "${outputs[0]}" != "${outputs[1]}" ]]; then
|
|
184
|
+
ready=false
|
|
185
|
+
break
|
|
186
|
+
fi
|
|
187
|
+
done
|
|
188
|
+
if [[ "$ready" == true && "${outputs[0]}" == "${outputs[2]}" ]]; then
|
|
189
|
+
echo "::notice::Mirror matches the latest source refs after ${SECONDS}s"
|
|
190
|
+
printf '%s\n' "${outputs[2]}"
|
|
191
|
+
exit 0
|
|
192
|
+
fi
|
|
193
|
+
echo "Waiting for mirror (attempt $attempt, ${SECONDS}/${wait_seconds}s)"
|
|
194
|
+
remaining=$((wait_seconds - SECONDS))
|
|
195
|
+
if (( remaining <= 0 )); then break; fi
|
|
196
|
+
# Gitee requires five minutes between sync requests. Retry once after that
|
|
197
|
+
# interval without restarting the polling deadline.
|
|
198
|
+
if [[ -n "$notify_repo" && "$retried" == false ]] && (( SECONDS >= retry_after )); then
|
|
199
|
+
echo '::notice::Retrying the GitHub notification after the Gitee sync interval'
|
|
200
|
+
if notify_static_mirror; then :; elif [[ $? -eq 2 ]]; then exit 1; fi
|
|
201
|
+
retried=true
|
|
202
|
+
continue
|
|
203
|
+
fi
|
|
204
|
+
delay=$poll_interval
|
|
205
|
+
if (( delay > remaining )); then delay=$remaining; fi
|
|
206
|
+
if [[ -n "$notify_repo" && "$retried" == false ]] && (( SECONDS < retry_after && delay > retry_after - SECONDS )); then
|
|
207
|
+
delay=$((retry_after - SECONDS))
|
|
208
|
+
fi
|
|
209
|
+
sleep "$delay"
|
|
210
|
+
done
|
|
211
|
+
|
|
212
|
+
echo "::error::Mirror did not match the latest source refs within ${wait_seconds}s"
|
|
213
|
+
printf 'Last source refs:\n%s\nLast mirror refs:\n%s\n' "$last_source" "$last_mirror"
|
|
214
|
+
exit 1
|