codebee 0.1.95 → 0.1.96
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +725 -720
- package/LICENSE +21 -21
- package/README.md +515 -509
- package/app/core/acceptance.py +100 -100
- package/app/core/agent_context.py +100 -0
- package/app/core/aiflavor.py +197 -197
- package/app/core/analytics.py +38 -38
- package/app/core/artifacts.py +148 -148
- package/app/core/assets.py +110 -110
- package/app/core/attachments.py +623 -623
- package/app/core/audit.py +30 -30
- package/app/core/backup.py +470 -470
- package/app/core/beekeeper.py +183 -183
- package/app/core/benchstore.py +102 -95
- package/app/core/board.py +209 -209
- package/app/core/bookmeta.py +1036 -1036
- package/app/core/bookmeta_catalog.py +180 -180
- package/app/core/branching.py +137 -137
- package/app/core/builtin_agent.py +1680 -1680
- package/app/core/capability.py +144 -144
- package/app/core/chaptersafe.py +32 -32
- package/app/core/checkpoints.py +143 -0
- package/app/core/cleanup.py +303 -303
- package/app/core/compaction.py +201 -201
- package/app/core/contracts.py +255 -255
- package/app/core/covergen.py +356 -356
- package/app/core/defectretro.py +43 -43
- package/app/core/diagnostics.py +114 -114
- package/app/core/dispatch.py +198 -191
- package/app/core/dispatch_log.py +128 -128
- package/app/core/env_scrub.py +85 -85
- package/app/core/errorlog.py +137 -137
- package/app/core/eval_matrix.py +82 -0
- package/app/core/evalbench.py +184 -91
- package/app/core/evaluation.py +174 -174
- package/app/core/failure_notify.py +290 -290
- package/app/core/flow_graph.py +33 -0
- package/app/core/goal_service.py +167 -167
- package/app/core/health.py +389 -389
- package/app/core/history.py +38 -38
- package/app/core/hooks.py +162 -162
- package/app/core/jobs.py +25 -4
- package/app/core/knowledge_pipeline.py +86 -0
- package/app/core/manager.py +2232 -2232
- package/app/core/mocks.py +64 -64
- package/app/core/notify.py +201 -201
- package/app/core/novel_quality.py +181 -189
- package/app/core/openapi.py +177 -154
- package/app/core/operations.py +228 -228
- package/app/core/paihang.py +145 -145
- package/app/core/pipeline.py +43 -19
- package/app/core/planner.py +17 -11
- package/app/core/policy.py +52 -0
- package/app/core/portguard.py +111 -111
- package/app/core/presence.py +77 -77
- package/app/core/preview.py +293 -293
- package/app/core/prompt_guard.py +78 -78
- package/app/core/publish/__init__.py +7 -7
- package/app/core/publish/auto.py +26 -3
- package/app/core/publish/browser.py +655 -655
- package/app/core/publish/chaprows.py +138 -0
- package/app/core/publish/fanqie.py +210 -165
- package/app/core/publish/flow.py +753 -640
- package/app/core/publish/flows-fanqie-calibrated.json +349 -336
- package/app/core/publish/ledger.py +13 -1
- package/app/core/publish/manager.py +1150 -1118
- package/app/core/publish/qimao.py +176 -176
- package/app/core/publish/volumes.py +108 -0
- package/app/core/publish/ws.py +139 -139
- package/app/core/quality_gate.py +194 -194
- package/app/core/redact.py +38 -38
- package/app/core/repeat_guard.py +129 -129
- package/app/core/retrieval.py +137 -137
- package/app/core/router.py +398 -398
- package/app/core/runner.py +2238 -2238
- package/app/core/selfupdate.py +384 -384
- package/app/core/session_log.py +161 -161
- package/app/core/sessions.py +312 -312
- package/app/core/settings.py +20 -0
- package/app/core/settings_schema.py +270 -270
- package/app/core/share_page.py +114 -114
- package/app/core/skill_scan.py +139 -139
- package/app/core/skillpacks/fanqie-novel.md +8 -6
- package/app/core/skillpacks/market/character-bible.md +66 -66
- package/app/core/skillpacks/market/code-risk-checklist.md +58 -58
- package/app/core/skillpacks/market/git-workflow.md +57 -57
- package/app/core/skillpacks/market/release-notes.md +72 -72
- package/app/core/skillpacks/market/weekly-report.md +71 -71
- package/app/core/skillpacks/market/worldview-consistency.md +70 -70
- package/app/core/skillpacks/qimao-signing.md +14 -11
- package/app/core/skills.py +32 -21
- package/app/core/step_runner.py +132 -132
- package/app/core/store.py +9 -0
- package/app/core/story_tracking.py +275 -275
- package/app/core/task_compile.py +199 -199
- package/app/core/telemetry.py +292 -292
- package/app/core/tlsctx.py +64 -64
- package/app/core/token_meter.py +147 -147
- package/app/core/tracing.py +207 -144
- package/app/core/upstream.py +41 -41
- package/app/core/usage.py +1038 -1038
- package/app/core/volumes.py +389 -389
- package/app/core/webhooks.py +89 -0
- package/app/core/wxdigest.py +858 -858
- package/app/core/zentao.py +2692 -2692
- package/app/main.py +104 -9
- package/app/mcp_server.py +244 -244
- package/app/pick_dialog.py +110 -110
- package/app/ui/app.js +286 -18
- package/app/ui/artdiff.js +74 -74
- package/app/ui/assets/community-qr.png +0 -0
- package/app/ui/assets/donate-alipay.jpg +0 -0
- package/app/ui/assets/donate-qr.jpg +0 -0
- package/app/ui/board.html +162 -162
- package/app/ui/board.js +305 -305
- package/app/ui/i18n.js +33 -0
- package/app/ui/icons/bee.svg +79 -79
- package/app/ui/index.html +32 -3
- package/app/ui/manifest.json +16 -16
- package/app/ui/qrcode.js +2297 -2297
- package/app/ui/style.css +196 -2
- package/bin/tutti.js +147 -147
- package/package.json +48 -48
- package/tools/wx_reader.py +261 -261
- package/app/.playwright-cli/console-2026-09-23T08-26-03-635Z.log +0 -88
- package/app/.playwright-cli/page-2026-09-23T08-26-03-845Z.yml +0 -126
- package/app/.playwright-cli/page-2026-09-23T08-29-44-203Z.png +0 -0
- package/app/.playwright-cli/page-2026-09-23T08-35-42-046Z.yml +0 -210
- package/app/.playwright-cli/page-2026-09-23T08-36-52-858Z.png +0 -0
- package/app/.playwright-cli/page-2026-09-23T08-37-33-938Z.png +0 -0
- package/app/.playwright-cli/run-detail-after.png +0 -0
- package/app/output/_createbook_body.json +0 -1
- package/app/output/_probe_fq_click.py +0 -50
- package/app/output/_probe_fq_form.py +0 -42
- package/app/output/_probe_fq_modal.py +0 -55
- package/app/output/_probe_fq_points.py +0 -71
- package/app/output/_probe_fq_submit.py +0 -34
- package/app/output/_probe_fq_watch.py +0 -62
package/app/core/backup.py
CHANGED
|
@@ -1,470 +1,470 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
"""整机数据备份:导出 / 导入(换电脑、换路径安装时的数据搬家)。
|
|
3
|
-
|
|
4
|
-
备份包是一个 zip,布局:
|
|
5
|
-
manifest.json 清单(app/schema/旧路径/计数/未打包目录)
|
|
6
|
-
data/… 整个数据目录(见下方排除清单)
|
|
7
|
-
workspace/<相对默认保存路径的路径> 任务工作目录(手稿/成品文件——真正的劳动成果)
|
|
8
|
-
|
|
9
|
-
排除不打包(易失或机器相关,导入后也无意义):
|
|
10
|
-
- 发布浏览器 profile(data/publish/profiles 迁移残留 + 家目录 ~/.codebee/
|
|
11
|
-
publish_profiles 本来就在数据目录外)——GB 级且登录态 cookie 与机器绑定;
|
|
12
|
-
- 日志(data 根 *.log)、pet.lock、chat_cache、pending(运行中信箱)、
|
|
13
|
-
exports/imports(备份模块自己的输出/暂存区)。
|
|
14
|
-
运行过程日志(runs/*/steps/*.log)默认打包,include_logs=False 可去掉减小体积。
|
|
15
|
-
|
|
16
|
-
导入路径重映射:任务记录/设置里存的是旧机器的绝对路径。manifest 记下旧
|
|
17
|
-
data 目录与旧默认保存路径,导入时把所有 JSON 值里以旧路径开头的字符串改写
|
|
18
|
-
为新机器对应路径(长前缀优先,避免父子前缀互吞),工作目录文件按相对路径
|
|
19
|
-
落到新默认保存路径下。导入前自动把当前小体积状态(根 *.json + tasks/*.json)
|
|
20
|
-
备份到 <data>/imports/pre-import-<ts>.zip 供反悔。
|
|
21
|
-
|
|
22
|
-
导入要求没有运行中/排队中的任务(记录与文件正被使用);完成后必须重启
|
|
23
|
-
服务(各模块的内存态与 import 期绑定只认启动时的数据目录)。
|
|
24
|
-
"""
|
|
25
|
-
from __future__ import annotations
|
|
26
|
-
|
|
27
|
-
import json
|
|
28
|
-
import os
|
|
29
|
-
import re
|
|
30
|
-
import shutil
|
|
31
|
-
import time
|
|
32
|
-
import zipfile
|
|
33
|
-
from pathlib import Path
|
|
34
|
-
|
|
35
|
-
from . import paths, settings as settings_mod
|
|
36
|
-
|
|
37
|
-
_SEG_SPLIT = re.compile(r"[\\/]")
|
|
38
|
-
_DRIVE_RE = re.compile(r"^[A-Za-z]:")
|
|
39
|
-
|
|
40
|
-
MANIFEST_NAME = "manifest.json"
|
|
41
|
-
SCHEMA = 1
|
|
42
|
-
APP_ID = "codebee"
|
|
43
|
-
|
|
44
|
-
# 数据目录内不打包的一级目录 / 文件名(易失、缓存或备份模块自留地)
|
|
45
|
-
_EXCLUDE_TOP_DIRS = {"exports", "imports", "chat_cache", "pending"}
|
|
46
|
-
_EXCLUDE_TOP_FILES = {"pet.lock"}
|
|
47
|
-
# publish/profiles 是 GB 级浏览器 profile 迁移残留(登录态与机器绑定),不打包
|
|
48
|
-
_LOG_SUFFIXES = (".log", ".tmp")
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
def _app_version():
|
|
52
|
-
try:
|
|
53
|
-
pkg = paths.ROOT / "package.json"
|
|
54
|
-
return str(json.loads(pkg.read_text(encoding="utf-8")).get("version") or "")
|
|
55
|
-
except Exception:
|
|
56
|
-
return ""
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
def _iter_files(base: Path):
|
|
60
|
-
"""yield (绝对路径, 相对 Path)。符号坏链等交给调用方逐文件兜底。"""
|
|
61
|
-
for cur, dirs, files in os.walk(base):
|
|
62
|
-
cur_p = Path(cur)
|
|
63
|
-
for f in files:
|
|
64
|
-
p = cur_p / f
|
|
65
|
-
try:
|
|
66
|
-
yield p, p.relative_to(base)
|
|
67
|
-
except ValueError:
|
|
68
|
-
continue
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
def _export_skip(rel: Path, include_logs=True):
|
|
72
|
-
"""导出时是否跳过该相对路径(rel 相对 DATA_DIR)。"""
|
|
73
|
-
parts = rel.parts
|
|
74
|
-
if not parts:
|
|
75
|
-
return True
|
|
76
|
-
if parts[0] in _EXCLUDE_TOP_DIRS:
|
|
77
|
-
return True
|
|
78
|
-
if parts[0] == "publish" and len(parts) > 1 and parts[1] == "profiles":
|
|
79
|
-
return True
|
|
80
|
-
if parts[0] in _EXCLUDE_TOP_FILES:
|
|
81
|
-
return True
|
|
82
|
-
if rel.suffix.lower() in _LOG_SUFFIXES:
|
|
83
|
-
# 运行过程日志按开关保留;数据根的服务日志一律不打包(易失)
|
|
84
|
-
if not (include_logs and parts[0] == "runs"):
|
|
85
|
-
return True
|
|
86
|
-
return False
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
def _collect_workspace_dirs():
|
|
90
|
-
"""收集该打包的任务工作目录。
|
|
91
|
-
|
|
92
|
-
只打包「默认保存路径」下的目录(当初由默认路径自动放置的 app 管辖区,
|
|
93
|
-
与 store.migrate_task_workdirs 同一口径);落在数据目录里的(如 demo)
|
|
94
|
-
随 data/ 打包;两者之外的是用户自己的目录(代码仓库等),不打包、只在
|
|
95
|
-
manifest.external_workdirs 里点名提醒。
|
|
96
|
-
返回 (根目录, [相对 Path…], [外部目录…])。
|
|
97
|
-
"""
|
|
98
|
-
root = Path(settings_mod.default_workdir())
|
|
99
|
-
from . import store
|
|
100
|
-
wds = set()
|
|
101
|
-
try:
|
|
102
|
-
for t in store.list_tasks(limit=10 ** 6):
|
|
103
|
-
wd = str(t.get("workdir") or "").strip()
|
|
104
|
-
if wd:
|
|
105
|
-
wds.add(Path(wd))
|
|
106
|
-
except Exception:
|
|
107
|
-
pass
|
|
108
|
-
try:
|
|
109
|
-
from . import automation
|
|
110
|
-
for at in automation.list_tasks():
|
|
111
|
-
wd = str(at.get("workdir") or "").strip()
|
|
112
|
-
if wd:
|
|
113
|
-
wds.add(Path(wd))
|
|
114
|
-
except Exception:
|
|
115
|
-
pass
|
|
116
|
-
inside, external = [], []
|
|
117
|
-
seen = set()
|
|
118
|
-
for wd in sorted(wds, key=lambda p: str(p).lower()):
|
|
119
|
-
s = str(wd)
|
|
120
|
-
if s.lower() in seen:
|
|
121
|
-
continue
|
|
122
|
-
seen.add(s.lower())
|
|
123
|
-
try:
|
|
124
|
-
rel = wd.resolve().relative_to(root.resolve())
|
|
125
|
-
except (ValueError, OSError):
|
|
126
|
-
try:
|
|
127
|
-
wd.resolve().relative_to(Path(paths.DATA_DIR).resolve())
|
|
128
|
-
continue # 数据目录内的随 data/ 打包,不重复收
|
|
129
|
-
except (ValueError, OSError):
|
|
130
|
-
external.append(s)
|
|
131
|
-
continue
|
|
132
|
-
if wd.is_dir():
|
|
133
|
-
inside.append(rel)
|
|
134
|
-
return root, inside, external
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
# ---------------------------------------------------------------- 导出
|
|
138
|
-
|
|
139
|
-
def export_data(target_dir="", include_logs=True, include_workspace=True):
|
|
140
|
-
"""打包当前数据目录 + 任务工作目录到 zip。返回信息 dict;参数非法抛 ValueError。"""
|
|
141
|
-
data_dir = Path(paths.DATA_DIR)
|
|
142
|
-
if not data_dir.is_dir():
|
|
143
|
-
raise ValueError("数据目录不存在: %s" % data_dir)
|
|
144
|
-
target = Path(target_dir).expanduser() if str(target_dir or "").strip() \
|
|
145
|
-
else (Path.home() / "Downloads")
|
|
146
|
-
if not target.is_absolute():
|
|
147
|
-
raise ValueError("导出位置必须是绝对路径")
|
|
148
|
-
try:
|
|
149
|
-
target.mkdir(parents=True, exist_ok=True)
|
|
150
|
-
except OSError as e:
|
|
151
|
-
raise ValueError("导出位置不可创建: %s(%s)" % (target, e))
|
|
152
|
-
|
|
153
|
-
root, ws_rels, external = _collect_workspace_dirs()
|
|
154
|
-
manifest = {
|
|
155
|
-
"app": APP_ID, "schema": SCHEMA,
|
|
156
|
-
"exported_at": time.strftime("%Y-%m-%d %H:%M:%S"),
|
|
157
|
-
"version": _app_version(),
|
|
158
|
-
"data_dir": str(data_dir),
|
|
159
|
-
"default_workdir": str(root),
|
|
160
|
-
"include_logs": bool(include_logs),
|
|
161
|
-
"include_workspace": bool(include_workspace),
|
|
162
|
-
"external_workdirs": external,
|
|
163
|
-
"workspace_dirs": [str(r) for r in ws_rels],
|
|
164
|
-
}
|
|
165
|
-
out = target / ("codebee-backup-" + time.strftime("%Y%m%d-%H%M%S") + ".zip")
|
|
166
|
-
counts = {"data_files": 0, "workspace_files": 0, "tasks": 0, "runs": 0}
|
|
167
|
-
warnings = []
|
|
168
|
-
try:
|
|
169
|
-
counts["tasks"] = sum(1 for _ in (data_dir / "tasks").glob("t-*.json"))
|
|
170
|
-
rdir = data_dir / "runs"
|
|
171
|
-
counts["runs"] = sum(1 for _ in rdir.iterdir()) if rdir.is_dir() else 0
|
|
172
|
-
except OSError:
|
|
173
|
-
pass
|
|
174
|
-
manifest["counts"] = counts
|
|
175
|
-
|
|
176
|
-
zf = None
|
|
177
|
-
try:
|
|
178
|
-
zf = zipfile.ZipFile(str(out), "w", zipfile.ZIP_DEFLATED)
|
|
179
|
-
for p, rel in _iter_files(data_dir):
|
|
180
|
-
if _export_skip(rel, include_logs=include_logs) or p == out:
|
|
181
|
-
continue
|
|
182
|
-
try:
|
|
183
|
-
zf.write(str(p), str(Path("data") / rel))
|
|
184
|
-
counts["data_files"] += 1
|
|
185
|
-
except OSError as e:
|
|
186
|
-
warnings.append("跳过 %s(%s)" % (rel, e.strerror or e))
|
|
187
|
-
if include_workspace:
|
|
188
|
-
for rel in ws_rels:
|
|
189
|
-
base = root / rel
|
|
190
|
-
for p, frel in _iter_files(base):
|
|
191
|
-
try:
|
|
192
|
-
zf.write(str(p), str(Path("workspace") / rel / frel))
|
|
193
|
-
counts["workspace_files"] += 1
|
|
194
|
-
except OSError as e:
|
|
195
|
-
warnings.append("跳过 %s(%s)" % (rel / frel, e.strerror or e))
|
|
196
|
-
# manifest 最后写:此刻 data/workspace 计数才是终值
|
|
197
|
-
zf.writestr(MANIFEST_NAME, json.dumps(manifest, ensure_ascii=False, indent=2))
|
|
198
|
-
finally:
|
|
199
|
-
if zf is not None:
|
|
200
|
-
zf.close()
|
|
201
|
-
manifest["counts"] = counts
|
|
202
|
-
return {"path": str(out), "size": out.stat().st_size,
|
|
203
|
-
"tasks": counts["tasks"], "runs": counts["runs"],
|
|
204
|
-
"data_files": counts["data_files"],
|
|
205
|
-
"workspace_files": counts["workspace_files"],
|
|
206
|
-
"external_workdirs": external, "warnings": warnings}
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
# ---------------------------------------------------------------- 预览
|
|
210
|
-
|
|
211
|
-
def _read_manifest(zf):
|
|
212
|
-
try:
|
|
213
|
-
raw = zf.read(MANIFEST_NAME)
|
|
214
|
-
except KeyError:
|
|
215
|
-
raise ValueError("不是 CodeBee 备份包(缺少 manifest.json)")
|
|
216
|
-
try:
|
|
217
|
-
m = json.loads(raw.decode("utf-8"))
|
|
218
|
-
except Exception:
|
|
219
|
-
raise ValueError("备份包 manifest.json 无法解析")
|
|
220
|
-
if not isinstance(m, dict) or m.get("app") != APP_ID:
|
|
221
|
-
raise ValueError("不是 CodeBee 备份包(manifest 不匹配)")
|
|
222
|
-
if int(m.get("schema") or 0) > SCHEMA:
|
|
223
|
-
raise ValueError("备份包版本较新(schema %s > %d),请先升级 CodeBee 再导入"
|
|
224
|
-
% (m.get("schema"), SCHEMA))
|
|
225
|
-
return m
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
def _safe_join(base: Path, name: str):
|
|
229
|
-
"""zip 条目名 → base 下安全路径(zip slip 四道闸,拒绝一切逃逸形态)。
|
|
230
|
-
|
|
231
|
-
①条目名含 .. 字面量即拒绝(任何形态的父目录逃逸,构造路径之前完成);
|
|
232
|
-
②按 / 与 \\ 切段后出现 .. 段、盘符(C:)同样拒绝;③只用纯段名 join 回
|
|
233
|
-
基线(base / 带盘符的 name 在 Windows 会整体替换掉 base,故绝不直接拼);
|
|
234
|
-
④normpath 后必须仍在 base 之内(双保险)。"""
|
|
235
|
-
if ".." in name:
|
|
236
|
-
raise ValueError("备份包含非法路径条目: %s" % name[:120])
|
|
237
|
-
parts = [p for p in _SEG_SPLIT.split(name) if p not in ("", ".")]
|
|
238
|
-
if (not parts or any(p == ".." for p in parts)
|
|
239
|
-
or _DRIVE_RE.match(parts[0])):
|
|
240
|
-
raise ValueError("备份包含非法路径条目: %s" % name[:120])
|
|
241
|
-
resolved = Path(os.path.normpath(os.path.join(str(base), *parts)))
|
|
242
|
-
try:
|
|
243
|
-
resolved.relative_to(base)
|
|
244
|
-
except ValueError:
|
|
245
|
-
raise ValueError("备份包含非法路径条目: %s" % name[:120])
|
|
246
|
-
return resolved
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
def _busy_runs():
|
|
250
|
-
from . import store
|
|
251
|
-
out = []
|
|
252
|
-
try:
|
|
253
|
-
for r in store.list_runs(limit=10 ** 6):
|
|
254
|
-
if r.get("status") in ("queued", "running"):
|
|
255
|
-
out.append(r.get("id"))
|
|
256
|
-
except Exception:
|
|
257
|
-
pass
|
|
258
|
-
return out
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
def inspect_backup(zip_path):
|
|
262
|
-
"""解析备份包,返回预览信息(不落任何盘)。包打不开/不是本家包抛 ValueError。"""
|
|
263
|
-
p = Path(str(zip_path or "").strip())
|
|
264
|
-
if not p.is_file():
|
|
265
|
-
raise ValueError("备份包不存在: %s" % p)
|
|
266
|
-
try:
|
|
267
|
-
zf = zipfile.ZipFile(str(p))
|
|
268
|
-
except Exception as e:
|
|
269
|
-
raise ValueError("备份包打不开:%s" % e)
|
|
270
|
-
with zf:
|
|
271
|
-
m = _read_manifest(zf)
|
|
272
|
-
names = zf.namelist()
|
|
273
|
-
data_n = sum(1 for n in names if n.startswith("data/"))
|
|
274
|
-
ws_n = sum(1 for n in names if n.startswith("workspace/"))
|
|
275
|
-
old_data = str(m.get("data_dir") or "")
|
|
276
|
-
old_wd = str(m.get("default_workdir") or "")
|
|
277
|
-
cur_data = str(Path(paths.DATA_DIR))
|
|
278
|
-
cur_wd = settings_mod.default_workdir()
|
|
279
|
-
return {
|
|
280
|
-
"exported_at": m.get("exported_at"), "version": m.get("version"),
|
|
281
|
-
"tasks": (m.get("counts") or {}).get("tasks"),
|
|
282
|
-
"runs": (m.get("counts") or {}).get("runs"),
|
|
283
|
-
"data_files": data_n, "workspace_files": ws_n,
|
|
284
|
-
"include_logs": bool(m.get("include_logs", True)),
|
|
285
|
-
"external_workdirs": m.get("external_workdirs") or [],
|
|
286
|
-
"old_data_dir": old_data, "old_workdir": old_wd,
|
|
287
|
-
"cur_data_dir": cur_data, "cur_workdir": cur_wd,
|
|
288
|
-
"remap_needed": (old_data.lower() != cur_data.lower()
|
|
289
|
-
or old_wd.lower() != cur_wd.lower()),
|
|
290
|
-
"busy_runs": _busy_runs(),
|
|
291
|
-
"same_machine": old_data.lower() == cur_data.lower(),
|
|
292
|
-
}
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
# ---------------------------------------------------------------- 导入
|
|
296
|
-
|
|
297
|
-
def _pre_import_backup(ts):
|
|
298
|
-
"""把当前小体积状态备份到 <data>/imports/pre-import-<ts>.zip(反悔用)。"""
|
|
299
|
-
data_dir = Path(paths.DATA_DIR)
|
|
300
|
-
bdir = data_dir / "imports"
|
|
301
|
-
bdir.mkdir(parents=True, exist_ok=True)
|
|
302
|
-
out = bdir / ("pre-import-" + ts + ".zip")
|
|
303
|
-
with zipfile.ZipFile(str(out), "w", zipfile.ZIP_DEFLATED) as zf:
|
|
304
|
-
for p in sorted(data_dir.glob("*.json")):
|
|
305
|
-
try:
|
|
306
|
-
zf.write(str(p), p.name)
|
|
307
|
-
except OSError:
|
|
308
|
-
pass
|
|
309
|
-
tdir = data_dir / "tasks"
|
|
310
|
-
if tdir.is_dir():
|
|
311
|
-
for p in sorted(tdir.glob("*.json")):
|
|
312
|
-
try:
|
|
313
|
-
zf.write(str(p), str(Path("tasks") / p.name))
|
|
314
|
-
except OSError:
|
|
315
|
-
pass
|
|
316
|
-
return out
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
def _remap_value(v, pairs):
|
|
320
|
-
"""字符串值按前缀对改写(长前缀优先);命中返回 (新串, True)。"""
|
|
321
|
-
if not isinstance(v, str) or not v:
|
|
322
|
-
return v, False
|
|
323
|
-
for old, new in pairs: # pairs 已按 len(old) 降序
|
|
324
|
-
if old.lower() == v.lower():
|
|
325
|
-
return new, True
|
|
326
|
-
lo = v.lower()
|
|
327
|
-
if lo.startswith(old.lower()):
|
|
328
|
-
tail = v[len(old):]
|
|
329
|
-
if tail and tail[0] not in "\\/":
|
|
330
|
-
continue # 只是碰巧同前缀(如 E:\GoOut2),不是旧路径
|
|
331
|
-
return new + tail, True
|
|
332
|
-
return v, False
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
def _remap_obj(o, pairs, stat):
|
|
336
|
-
if isinstance(o, dict):
|
|
337
|
-
return {k: _remap_obj(v, pairs, stat) for k, v in o.items()}
|
|
338
|
-
if isinstance(o, list):
|
|
339
|
-
return [_remap_obj(v, pairs, stat) for v in o]
|
|
340
|
-
nv, hit = _remap_value(o, pairs)
|
|
341
|
-
if hit:
|
|
342
|
-
stat[0] += 1
|
|
343
|
-
return nv
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
def _remap_all_json(pairs):
|
|
347
|
-
"""数据目录内所有 JSON 的字符串值做旧→新路径改写。返回 (文件数, 值数)。"""
|
|
348
|
-
data_dir = Path(paths.DATA_DIR)
|
|
349
|
-
files = hits = 0
|
|
350
|
-
for p in data_dir.rglob("*.json"):
|
|
351
|
-
rel = p.relative_to(data_dir)
|
|
352
|
-
if rel.parts[0] in ("imports", "exports"):
|
|
353
|
-
continue
|
|
354
|
-
try:
|
|
355
|
-
raw = p.read_text(encoding="utf-8")
|
|
356
|
-
obj = json.loads(raw)
|
|
357
|
-
if not isinstance(obj, (dict, list)):
|
|
358
|
-
continue
|
|
359
|
-
stat = [0]
|
|
360
|
-
new_obj = _remap_obj(obj, pairs, stat)
|
|
361
|
-
if stat[0]:
|
|
362
|
-
tmp = p.with_suffix(".tmp")
|
|
363
|
-
tmp.write_text(json.dumps(new_obj, ensure_ascii=False, indent=2),
|
|
364
|
-
encoding="utf-8")
|
|
365
|
-
os.replace(str(tmp), str(p))
|
|
366
|
-
files += 1
|
|
367
|
-
hits += stat[0]
|
|
368
|
-
except Exception:
|
|
369
|
-
continue # 单个文件坏/被锁不阻塞整体导入
|
|
370
|
-
return files, hits
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
def _extract(zf, names, base, prefix, skip=None):
|
|
374
|
-
n = 0
|
|
375
|
-
for name in names:
|
|
376
|
-
if not name.startswith(prefix) or name == prefix:
|
|
377
|
-
continue
|
|
378
|
-
rel = name[len(prefix):]
|
|
379
|
-
if not rel:
|
|
380
|
-
continue
|
|
381
|
-
if skip and skip(rel):
|
|
382
|
-
continue
|
|
383
|
-
target = _safe_join(base, rel)
|
|
384
|
-
if name.endswith("/"):
|
|
385
|
-
target.mkdir(parents=True, exist_ok=True)
|
|
386
|
-
continue
|
|
387
|
-
target.parent.mkdir(parents=True, exist_ok=True)
|
|
388
|
-
with zf.open(name) as src, open(str(target), "wb") as dst:
|
|
389
|
-
shutil.copyfileobj(src, dst)
|
|
390
|
-
n += 1
|
|
391
|
-
return n
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
def apply_import(zip_path, mode="merge", remap=True, include_workspace=True):
|
|
395
|
-
"""应用备份包。mode=merge 现有数据保留、同名覆盖;replace 先清空再落(不碰
|
|
396
|
-
imports/ 与 pet.lock)。返回信息 dict;校验失败/有运行中任务抛 ValueError。"""
|
|
397
|
-
p = Path(str(zip_path or "").strip())
|
|
398
|
-
if not p.is_file():
|
|
399
|
-
raise ValueError("备份包不存在: %s" % p)
|
|
400
|
-
mode = mode if mode in ("merge", "replace") else "merge"
|
|
401
|
-
busy = _busy_runs()
|
|
402
|
-
if busy:
|
|
403
|
-
raise ValueError("有 %d 个任务正在运行或排队(%s…),导入会踩到正在写的"
|
|
404
|
-
"文件;等它们跑完或取消后再试"
|
|
405
|
-
% (len(busy), "、".join(busy[:3])))
|
|
406
|
-
|
|
407
|
-
cur_wd = settings_mod.default_workdir()
|
|
408
|
-
data_dir = Path(paths.DATA_DIR)
|
|
409
|
-
zf = zipfile.ZipFile(str(p))
|
|
410
|
-
try:
|
|
411
|
-
m = _read_manifest(zf)
|
|
412
|
-
names = [n for n in zf.namelist() if not n.endswith("/")]
|
|
413
|
-
# 事前把全部条目做 zip-slip 校验,任何一个不合法就整体拒绝
|
|
414
|
-
for name in names:
|
|
415
|
-
if name.startswith("data/"):
|
|
416
|
-
_safe_join(data_dir, name[len("data/"):])
|
|
417
|
-
elif name.startswith("workspace/"):
|
|
418
|
-
_safe_join(Path(cur_wd), name[len("workspace/"):])
|
|
419
|
-
elif name != MANIFEST_NAME:
|
|
420
|
-
raise ValueError("备份包含无法识别的条目: %s" % name)
|
|
421
|
-
ts = time.strftime("%Y%m%d-%H%M%S")
|
|
422
|
-
backup_path = _pre_import_backup(ts)
|
|
423
|
-
if mode == "replace":
|
|
424
|
-
for child in data_dir.iterdir():
|
|
425
|
-
if child.name in ("imports", "pet.lock"):
|
|
426
|
-
continue
|
|
427
|
-
try:
|
|
428
|
-
if child.is_dir():
|
|
429
|
-
shutil.rmtree(str(child), ignore_errors=True)
|
|
430
|
-
else:
|
|
431
|
-
child.unlink()
|
|
432
|
-
except OSError:
|
|
433
|
-
pass
|
|
434
|
-
n_data = _extract(zf, names, data_dir, "data/")
|
|
435
|
-
n_ws = 0
|
|
436
|
-
if include_workspace:
|
|
437
|
-
Path(cur_wd).mkdir(parents=True, exist_ok=True)
|
|
438
|
-
n_ws = _extract(zf, names, Path(cur_wd), "workspace/")
|
|
439
|
-
finally:
|
|
440
|
-
zf.close()
|
|
441
|
-
|
|
442
|
-
# 路径重映射:长前缀优先(工作目录可能在数据目录之下,避免被短前缀截走)
|
|
443
|
-
old_data = str(m.get("data_dir") or "")
|
|
444
|
-
old_wd = str(m.get("default_workdir") or "")
|
|
445
|
-
pairs = []
|
|
446
|
-
if remap:
|
|
447
|
-
for old, new in ((old_wd, cur_wd), (old_data, str(data_dir))):
|
|
448
|
-
if old and old.lower() != new.lower():
|
|
449
|
-
pairs.append((old, new))
|
|
450
|
-
pairs.sort(key=lambda x: len(x[0]), reverse=True)
|
|
451
|
-
remapped_files = remapped_values = 0
|
|
452
|
-
if pairs:
|
|
453
|
-
remapped_files, remapped_values = _remap_all_json(pairs)
|
|
454
|
-
|
|
455
|
-
# 导入的 settings.json 里若带着旧机器默认路径,上面已重映射;把当前生效值对齐
|
|
456
|
-
try:
|
|
457
|
-
from . import store
|
|
458
|
-
with store.LOCK:
|
|
459
|
-
store._TASKS.clear()
|
|
460
|
-
store._RUNS.clear()
|
|
461
|
-
store.load_all()
|
|
462
|
-
store.bump_state()
|
|
463
|
-
except Exception:
|
|
464
|
-
pass
|
|
465
|
-
return {"mode": mode, "data_files": n_data, "workspace_files": n_ws,
|
|
466
|
-
"remap": {"applied": bool(pairs), "files": remapped_files,
|
|
467
|
-
"values": remapped_values, "pairs": [[o, n] for o, n in pairs]},
|
|
468
|
-
"backup_path": str(backup_path),
|
|
469
|
-
"external_workdirs": m.get("external_workdirs") or [],
|
|
470
|
-
"restart_required": True}
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""整机数据备份:导出 / 导入(换电脑、换路径安装时的数据搬家)。
|
|
3
|
+
|
|
4
|
+
备份包是一个 zip,布局:
|
|
5
|
+
manifest.json 清单(app/schema/旧路径/计数/未打包目录)
|
|
6
|
+
data/… 整个数据目录(见下方排除清单)
|
|
7
|
+
workspace/<相对默认保存路径的路径> 任务工作目录(手稿/成品文件——真正的劳动成果)
|
|
8
|
+
|
|
9
|
+
排除不打包(易失或机器相关,导入后也无意义):
|
|
10
|
+
- 发布浏览器 profile(data/publish/profiles 迁移残留 + 家目录 ~/.codebee/
|
|
11
|
+
publish_profiles 本来就在数据目录外)——GB 级且登录态 cookie 与机器绑定;
|
|
12
|
+
- 日志(data 根 *.log)、pet.lock、chat_cache、pending(运行中信箱)、
|
|
13
|
+
exports/imports(备份模块自己的输出/暂存区)。
|
|
14
|
+
运行过程日志(runs/*/steps/*.log)默认打包,include_logs=False 可去掉减小体积。
|
|
15
|
+
|
|
16
|
+
导入路径重映射:任务记录/设置里存的是旧机器的绝对路径。manifest 记下旧
|
|
17
|
+
data 目录与旧默认保存路径,导入时把所有 JSON 值里以旧路径开头的字符串改写
|
|
18
|
+
为新机器对应路径(长前缀优先,避免父子前缀互吞),工作目录文件按相对路径
|
|
19
|
+
落到新默认保存路径下。导入前自动把当前小体积状态(根 *.json + tasks/*.json)
|
|
20
|
+
备份到 <data>/imports/pre-import-<ts>.zip 供反悔。
|
|
21
|
+
|
|
22
|
+
导入要求没有运行中/排队中的任务(记录与文件正被使用);完成后必须重启
|
|
23
|
+
服务(各模块的内存态与 import 期绑定只认启动时的数据目录)。
|
|
24
|
+
"""
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import json
|
|
28
|
+
import os
|
|
29
|
+
import re
|
|
30
|
+
import shutil
|
|
31
|
+
import time
|
|
32
|
+
import zipfile
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
|
|
35
|
+
from . import paths, settings as settings_mod
|
|
36
|
+
|
|
37
|
+
_SEG_SPLIT = re.compile(r"[\\/]")
|
|
38
|
+
_DRIVE_RE = re.compile(r"^[A-Za-z]:")
|
|
39
|
+
|
|
40
|
+
MANIFEST_NAME = "manifest.json"
|
|
41
|
+
SCHEMA = 1
|
|
42
|
+
APP_ID = "codebee"
|
|
43
|
+
|
|
44
|
+
# 数据目录内不打包的一级目录 / 文件名(易失、缓存或备份模块自留地)
|
|
45
|
+
_EXCLUDE_TOP_DIRS = {"exports", "imports", "chat_cache", "pending"}
|
|
46
|
+
_EXCLUDE_TOP_FILES = {"pet.lock"}
|
|
47
|
+
# publish/profiles 是 GB 级浏览器 profile 迁移残留(登录态与机器绑定),不打包
|
|
48
|
+
_LOG_SUFFIXES = (".log", ".tmp")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _app_version():
|
|
52
|
+
try:
|
|
53
|
+
pkg = paths.ROOT / "package.json"
|
|
54
|
+
return str(json.loads(pkg.read_text(encoding="utf-8")).get("version") or "")
|
|
55
|
+
except Exception:
|
|
56
|
+
return ""
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _iter_files(base: Path):
|
|
60
|
+
"""yield (绝对路径, 相对 Path)。符号坏链等交给调用方逐文件兜底。"""
|
|
61
|
+
for cur, dirs, files in os.walk(base):
|
|
62
|
+
cur_p = Path(cur)
|
|
63
|
+
for f in files:
|
|
64
|
+
p = cur_p / f
|
|
65
|
+
try:
|
|
66
|
+
yield p, p.relative_to(base)
|
|
67
|
+
except ValueError:
|
|
68
|
+
continue
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _export_skip(rel: Path, include_logs=True):
|
|
72
|
+
"""导出时是否跳过该相对路径(rel 相对 DATA_DIR)。"""
|
|
73
|
+
parts = rel.parts
|
|
74
|
+
if not parts:
|
|
75
|
+
return True
|
|
76
|
+
if parts[0] in _EXCLUDE_TOP_DIRS:
|
|
77
|
+
return True
|
|
78
|
+
if parts[0] == "publish" and len(parts) > 1 and parts[1] == "profiles":
|
|
79
|
+
return True
|
|
80
|
+
if parts[0] in _EXCLUDE_TOP_FILES:
|
|
81
|
+
return True
|
|
82
|
+
if rel.suffix.lower() in _LOG_SUFFIXES:
|
|
83
|
+
# 运行过程日志按开关保留;数据根的服务日志一律不打包(易失)
|
|
84
|
+
if not (include_logs and parts[0] == "runs"):
|
|
85
|
+
return True
|
|
86
|
+
return False
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _collect_workspace_dirs():
|
|
90
|
+
"""收集该打包的任务工作目录。
|
|
91
|
+
|
|
92
|
+
只打包「默认保存路径」下的目录(当初由默认路径自动放置的 app 管辖区,
|
|
93
|
+
与 store.migrate_task_workdirs 同一口径);落在数据目录里的(如 demo)
|
|
94
|
+
随 data/ 打包;两者之外的是用户自己的目录(代码仓库等),不打包、只在
|
|
95
|
+
manifest.external_workdirs 里点名提醒。
|
|
96
|
+
返回 (根目录, [相对 Path…], [外部目录…])。
|
|
97
|
+
"""
|
|
98
|
+
root = Path(settings_mod.default_workdir())
|
|
99
|
+
from . import store
|
|
100
|
+
wds = set()
|
|
101
|
+
try:
|
|
102
|
+
for t in store.list_tasks(limit=10 ** 6):
|
|
103
|
+
wd = str(t.get("workdir") or "").strip()
|
|
104
|
+
if wd:
|
|
105
|
+
wds.add(Path(wd))
|
|
106
|
+
except Exception:
|
|
107
|
+
pass
|
|
108
|
+
try:
|
|
109
|
+
from . import automation
|
|
110
|
+
for at in automation.list_tasks():
|
|
111
|
+
wd = str(at.get("workdir") or "").strip()
|
|
112
|
+
if wd:
|
|
113
|
+
wds.add(Path(wd))
|
|
114
|
+
except Exception:
|
|
115
|
+
pass
|
|
116
|
+
inside, external = [], []
|
|
117
|
+
seen = set()
|
|
118
|
+
for wd in sorted(wds, key=lambda p: str(p).lower()):
|
|
119
|
+
s = str(wd)
|
|
120
|
+
if s.lower() in seen:
|
|
121
|
+
continue
|
|
122
|
+
seen.add(s.lower())
|
|
123
|
+
try:
|
|
124
|
+
rel = wd.resolve().relative_to(root.resolve())
|
|
125
|
+
except (ValueError, OSError):
|
|
126
|
+
try:
|
|
127
|
+
wd.resolve().relative_to(Path(paths.DATA_DIR).resolve())
|
|
128
|
+
continue # 数据目录内的随 data/ 打包,不重复收
|
|
129
|
+
except (ValueError, OSError):
|
|
130
|
+
external.append(s)
|
|
131
|
+
continue
|
|
132
|
+
if wd.is_dir():
|
|
133
|
+
inside.append(rel)
|
|
134
|
+
return root, inside, external
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
# ---------------------------------------------------------------- 导出
|
|
138
|
+
|
|
139
|
+
def export_data(target_dir="", include_logs=True, include_workspace=True):
|
|
140
|
+
"""打包当前数据目录 + 任务工作目录到 zip。返回信息 dict;参数非法抛 ValueError。"""
|
|
141
|
+
data_dir = Path(paths.DATA_DIR)
|
|
142
|
+
if not data_dir.is_dir():
|
|
143
|
+
raise ValueError("数据目录不存在: %s" % data_dir)
|
|
144
|
+
target = Path(target_dir).expanduser() if str(target_dir or "").strip() \
|
|
145
|
+
else (Path.home() / "Downloads")
|
|
146
|
+
if not target.is_absolute():
|
|
147
|
+
raise ValueError("导出位置必须是绝对路径")
|
|
148
|
+
try:
|
|
149
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
150
|
+
except OSError as e:
|
|
151
|
+
raise ValueError("导出位置不可创建: %s(%s)" % (target, e))
|
|
152
|
+
|
|
153
|
+
root, ws_rels, external = _collect_workspace_dirs()
|
|
154
|
+
manifest = {
|
|
155
|
+
"app": APP_ID, "schema": SCHEMA,
|
|
156
|
+
"exported_at": time.strftime("%Y-%m-%d %H:%M:%S"),
|
|
157
|
+
"version": _app_version(),
|
|
158
|
+
"data_dir": str(data_dir),
|
|
159
|
+
"default_workdir": str(root),
|
|
160
|
+
"include_logs": bool(include_logs),
|
|
161
|
+
"include_workspace": bool(include_workspace),
|
|
162
|
+
"external_workdirs": external,
|
|
163
|
+
"workspace_dirs": [str(r) for r in ws_rels],
|
|
164
|
+
}
|
|
165
|
+
out = target / ("codebee-backup-" + time.strftime("%Y%m%d-%H%M%S") + ".zip")
|
|
166
|
+
counts = {"data_files": 0, "workspace_files": 0, "tasks": 0, "runs": 0}
|
|
167
|
+
warnings = []
|
|
168
|
+
try:
|
|
169
|
+
counts["tasks"] = sum(1 for _ in (data_dir / "tasks").glob("t-*.json"))
|
|
170
|
+
rdir = data_dir / "runs"
|
|
171
|
+
counts["runs"] = sum(1 for _ in rdir.iterdir()) if rdir.is_dir() else 0
|
|
172
|
+
except OSError:
|
|
173
|
+
pass
|
|
174
|
+
manifest["counts"] = counts
|
|
175
|
+
|
|
176
|
+
zf = None
|
|
177
|
+
try:
|
|
178
|
+
zf = zipfile.ZipFile(str(out), "w", zipfile.ZIP_DEFLATED)
|
|
179
|
+
for p, rel in _iter_files(data_dir):
|
|
180
|
+
if _export_skip(rel, include_logs=include_logs) or p == out:
|
|
181
|
+
continue
|
|
182
|
+
try:
|
|
183
|
+
zf.write(str(p), str(Path("data") / rel))
|
|
184
|
+
counts["data_files"] += 1
|
|
185
|
+
except OSError as e:
|
|
186
|
+
warnings.append("跳过 %s(%s)" % (rel, e.strerror or e))
|
|
187
|
+
if include_workspace:
|
|
188
|
+
for rel in ws_rels:
|
|
189
|
+
base = root / rel
|
|
190
|
+
for p, frel in _iter_files(base):
|
|
191
|
+
try:
|
|
192
|
+
zf.write(str(p), str(Path("workspace") / rel / frel))
|
|
193
|
+
counts["workspace_files"] += 1
|
|
194
|
+
except OSError as e:
|
|
195
|
+
warnings.append("跳过 %s(%s)" % (rel / frel, e.strerror or e))
|
|
196
|
+
# manifest 最后写:此刻 data/workspace 计数才是终值
|
|
197
|
+
zf.writestr(MANIFEST_NAME, json.dumps(manifest, ensure_ascii=False, indent=2))
|
|
198
|
+
finally:
|
|
199
|
+
if zf is not None:
|
|
200
|
+
zf.close()
|
|
201
|
+
manifest["counts"] = counts
|
|
202
|
+
return {"path": str(out), "size": out.stat().st_size,
|
|
203
|
+
"tasks": counts["tasks"], "runs": counts["runs"],
|
|
204
|
+
"data_files": counts["data_files"],
|
|
205
|
+
"workspace_files": counts["workspace_files"],
|
|
206
|
+
"external_workdirs": external, "warnings": warnings}
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
# ---------------------------------------------------------------- 预览
|
|
210
|
+
|
|
211
|
+
def _read_manifest(zf):
|
|
212
|
+
try:
|
|
213
|
+
raw = zf.read(MANIFEST_NAME)
|
|
214
|
+
except KeyError:
|
|
215
|
+
raise ValueError("不是 CodeBee 备份包(缺少 manifest.json)")
|
|
216
|
+
try:
|
|
217
|
+
m = json.loads(raw.decode("utf-8"))
|
|
218
|
+
except Exception:
|
|
219
|
+
raise ValueError("备份包 manifest.json 无法解析")
|
|
220
|
+
if not isinstance(m, dict) or m.get("app") != APP_ID:
|
|
221
|
+
raise ValueError("不是 CodeBee 备份包(manifest 不匹配)")
|
|
222
|
+
if int(m.get("schema") or 0) > SCHEMA:
|
|
223
|
+
raise ValueError("备份包版本较新(schema %s > %d),请先升级 CodeBee 再导入"
|
|
224
|
+
% (m.get("schema"), SCHEMA))
|
|
225
|
+
return m
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _safe_join(base: Path, name: str):
|
|
229
|
+
"""zip 条目名 → base 下安全路径(zip slip 四道闸,拒绝一切逃逸形态)。
|
|
230
|
+
|
|
231
|
+
①条目名含 .. 字面量即拒绝(任何形态的父目录逃逸,构造路径之前完成);
|
|
232
|
+
②按 / 与 \\ 切段后出现 .. 段、盘符(C:)同样拒绝;③只用纯段名 join 回
|
|
233
|
+
基线(base / 带盘符的 name 在 Windows 会整体替换掉 base,故绝不直接拼);
|
|
234
|
+
④normpath 后必须仍在 base 之内(双保险)。"""
|
|
235
|
+
if ".." in name:
|
|
236
|
+
raise ValueError("备份包含非法路径条目: %s" % name[:120])
|
|
237
|
+
parts = [p for p in _SEG_SPLIT.split(name) if p not in ("", ".")]
|
|
238
|
+
if (not parts or any(p == ".." for p in parts)
|
|
239
|
+
or _DRIVE_RE.match(parts[0])):
|
|
240
|
+
raise ValueError("备份包含非法路径条目: %s" % name[:120])
|
|
241
|
+
resolved = Path(os.path.normpath(os.path.join(str(base), *parts)))
|
|
242
|
+
try:
|
|
243
|
+
resolved.relative_to(base)
|
|
244
|
+
except ValueError:
|
|
245
|
+
raise ValueError("备份包含非法路径条目: %s" % name[:120])
|
|
246
|
+
return resolved
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _busy_runs():
|
|
250
|
+
from . import store
|
|
251
|
+
out = []
|
|
252
|
+
try:
|
|
253
|
+
for r in store.list_runs(limit=10 ** 6):
|
|
254
|
+
if r.get("status") in ("queued", "running"):
|
|
255
|
+
out.append(r.get("id"))
|
|
256
|
+
except Exception:
|
|
257
|
+
pass
|
|
258
|
+
return out
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def inspect_backup(zip_path):
|
|
262
|
+
"""解析备份包,返回预览信息(不落任何盘)。包打不开/不是本家包抛 ValueError。"""
|
|
263
|
+
p = Path(str(zip_path or "").strip())
|
|
264
|
+
if not p.is_file():
|
|
265
|
+
raise ValueError("备份包不存在: %s" % p)
|
|
266
|
+
try:
|
|
267
|
+
zf = zipfile.ZipFile(str(p))
|
|
268
|
+
except Exception as e:
|
|
269
|
+
raise ValueError("备份包打不开:%s" % e)
|
|
270
|
+
with zf:
|
|
271
|
+
m = _read_manifest(zf)
|
|
272
|
+
names = zf.namelist()
|
|
273
|
+
data_n = sum(1 for n in names if n.startswith("data/"))
|
|
274
|
+
ws_n = sum(1 for n in names if n.startswith("workspace/"))
|
|
275
|
+
old_data = str(m.get("data_dir") or "")
|
|
276
|
+
old_wd = str(m.get("default_workdir") or "")
|
|
277
|
+
cur_data = str(Path(paths.DATA_DIR))
|
|
278
|
+
cur_wd = settings_mod.default_workdir()
|
|
279
|
+
return {
|
|
280
|
+
"exported_at": m.get("exported_at"), "version": m.get("version"),
|
|
281
|
+
"tasks": (m.get("counts") or {}).get("tasks"),
|
|
282
|
+
"runs": (m.get("counts") or {}).get("runs"),
|
|
283
|
+
"data_files": data_n, "workspace_files": ws_n,
|
|
284
|
+
"include_logs": bool(m.get("include_logs", True)),
|
|
285
|
+
"external_workdirs": m.get("external_workdirs") or [],
|
|
286
|
+
"old_data_dir": old_data, "old_workdir": old_wd,
|
|
287
|
+
"cur_data_dir": cur_data, "cur_workdir": cur_wd,
|
|
288
|
+
"remap_needed": (old_data.lower() != cur_data.lower()
|
|
289
|
+
or old_wd.lower() != cur_wd.lower()),
|
|
290
|
+
"busy_runs": _busy_runs(),
|
|
291
|
+
"same_machine": old_data.lower() == cur_data.lower(),
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
# ---------------------------------------------------------------- 导入
|
|
296
|
+
|
|
297
|
+
def _pre_import_backup(ts):
|
|
298
|
+
"""把当前小体积状态备份到 <data>/imports/pre-import-<ts>.zip(反悔用)。"""
|
|
299
|
+
data_dir = Path(paths.DATA_DIR)
|
|
300
|
+
bdir = data_dir / "imports"
|
|
301
|
+
bdir.mkdir(parents=True, exist_ok=True)
|
|
302
|
+
out = bdir / ("pre-import-" + ts + ".zip")
|
|
303
|
+
with zipfile.ZipFile(str(out), "w", zipfile.ZIP_DEFLATED) as zf:
|
|
304
|
+
for p in sorted(data_dir.glob("*.json")):
|
|
305
|
+
try:
|
|
306
|
+
zf.write(str(p), p.name)
|
|
307
|
+
except OSError:
|
|
308
|
+
pass
|
|
309
|
+
tdir = data_dir / "tasks"
|
|
310
|
+
if tdir.is_dir():
|
|
311
|
+
for p in sorted(tdir.glob("*.json")):
|
|
312
|
+
try:
|
|
313
|
+
zf.write(str(p), str(Path("tasks") / p.name))
|
|
314
|
+
except OSError:
|
|
315
|
+
pass
|
|
316
|
+
return out
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _remap_value(v, pairs):
|
|
320
|
+
"""字符串值按前缀对改写(长前缀优先);命中返回 (新串, True)。"""
|
|
321
|
+
if not isinstance(v, str) or not v:
|
|
322
|
+
return v, False
|
|
323
|
+
for old, new in pairs: # pairs 已按 len(old) 降序
|
|
324
|
+
if old.lower() == v.lower():
|
|
325
|
+
return new, True
|
|
326
|
+
lo = v.lower()
|
|
327
|
+
if lo.startswith(old.lower()):
|
|
328
|
+
tail = v[len(old):]
|
|
329
|
+
if tail and tail[0] not in "\\/":
|
|
330
|
+
continue # 只是碰巧同前缀(如 E:\GoOut2),不是旧路径
|
|
331
|
+
return new + tail, True
|
|
332
|
+
return v, False
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _remap_obj(o, pairs, stat):
|
|
336
|
+
if isinstance(o, dict):
|
|
337
|
+
return {k: _remap_obj(v, pairs, stat) for k, v in o.items()}
|
|
338
|
+
if isinstance(o, list):
|
|
339
|
+
return [_remap_obj(v, pairs, stat) for v in o]
|
|
340
|
+
nv, hit = _remap_value(o, pairs)
|
|
341
|
+
if hit:
|
|
342
|
+
stat[0] += 1
|
|
343
|
+
return nv
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _remap_all_json(pairs):
|
|
347
|
+
"""数据目录内所有 JSON 的字符串值做旧→新路径改写。返回 (文件数, 值数)。"""
|
|
348
|
+
data_dir = Path(paths.DATA_DIR)
|
|
349
|
+
files = hits = 0
|
|
350
|
+
for p in data_dir.rglob("*.json"):
|
|
351
|
+
rel = p.relative_to(data_dir)
|
|
352
|
+
if rel.parts[0] in ("imports", "exports"):
|
|
353
|
+
continue
|
|
354
|
+
try:
|
|
355
|
+
raw = p.read_text(encoding="utf-8")
|
|
356
|
+
obj = json.loads(raw)
|
|
357
|
+
if not isinstance(obj, (dict, list)):
|
|
358
|
+
continue
|
|
359
|
+
stat = [0]
|
|
360
|
+
new_obj = _remap_obj(obj, pairs, stat)
|
|
361
|
+
if stat[0]:
|
|
362
|
+
tmp = p.with_suffix(".tmp")
|
|
363
|
+
tmp.write_text(json.dumps(new_obj, ensure_ascii=False, indent=2),
|
|
364
|
+
encoding="utf-8")
|
|
365
|
+
os.replace(str(tmp), str(p))
|
|
366
|
+
files += 1
|
|
367
|
+
hits += stat[0]
|
|
368
|
+
except Exception:
|
|
369
|
+
continue # 单个文件坏/被锁不阻塞整体导入
|
|
370
|
+
return files, hits
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def _extract(zf, names, base, prefix, skip=None):
|
|
374
|
+
n = 0
|
|
375
|
+
for name in names:
|
|
376
|
+
if not name.startswith(prefix) or name == prefix:
|
|
377
|
+
continue
|
|
378
|
+
rel = name[len(prefix):]
|
|
379
|
+
if not rel:
|
|
380
|
+
continue
|
|
381
|
+
if skip and skip(rel):
|
|
382
|
+
continue
|
|
383
|
+
target = _safe_join(base, rel)
|
|
384
|
+
if name.endswith("/"):
|
|
385
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
386
|
+
continue
|
|
387
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
388
|
+
with zf.open(name) as src, open(str(target), "wb") as dst:
|
|
389
|
+
shutil.copyfileobj(src, dst)
|
|
390
|
+
n += 1
|
|
391
|
+
return n
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def apply_import(zip_path, mode="merge", remap=True, include_workspace=True):
|
|
395
|
+
"""应用备份包。mode=merge 现有数据保留、同名覆盖;replace 先清空再落(不碰
|
|
396
|
+
imports/ 与 pet.lock)。返回信息 dict;校验失败/有运行中任务抛 ValueError。"""
|
|
397
|
+
p = Path(str(zip_path or "").strip())
|
|
398
|
+
if not p.is_file():
|
|
399
|
+
raise ValueError("备份包不存在: %s" % p)
|
|
400
|
+
mode = mode if mode in ("merge", "replace") else "merge"
|
|
401
|
+
busy = _busy_runs()
|
|
402
|
+
if busy:
|
|
403
|
+
raise ValueError("有 %d 个任务正在运行或排队(%s…),导入会踩到正在写的"
|
|
404
|
+
"文件;等它们跑完或取消后再试"
|
|
405
|
+
% (len(busy), "、".join(busy[:3])))
|
|
406
|
+
|
|
407
|
+
cur_wd = settings_mod.default_workdir()
|
|
408
|
+
data_dir = Path(paths.DATA_DIR)
|
|
409
|
+
zf = zipfile.ZipFile(str(p))
|
|
410
|
+
try:
|
|
411
|
+
m = _read_manifest(zf)
|
|
412
|
+
names = [n for n in zf.namelist() if not n.endswith("/")]
|
|
413
|
+
# 事前把全部条目做 zip-slip 校验,任何一个不合法就整体拒绝
|
|
414
|
+
for name in names:
|
|
415
|
+
if name.startswith("data/"):
|
|
416
|
+
_safe_join(data_dir, name[len("data/"):])
|
|
417
|
+
elif name.startswith("workspace/"):
|
|
418
|
+
_safe_join(Path(cur_wd), name[len("workspace/"):])
|
|
419
|
+
elif name != MANIFEST_NAME:
|
|
420
|
+
raise ValueError("备份包含无法识别的条目: %s" % name)
|
|
421
|
+
ts = time.strftime("%Y%m%d-%H%M%S")
|
|
422
|
+
backup_path = _pre_import_backup(ts)
|
|
423
|
+
if mode == "replace":
|
|
424
|
+
for child in data_dir.iterdir():
|
|
425
|
+
if child.name in ("imports", "pet.lock"):
|
|
426
|
+
continue
|
|
427
|
+
try:
|
|
428
|
+
if child.is_dir():
|
|
429
|
+
shutil.rmtree(str(child), ignore_errors=True)
|
|
430
|
+
else:
|
|
431
|
+
child.unlink()
|
|
432
|
+
except OSError:
|
|
433
|
+
pass
|
|
434
|
+
n_data = _extract(zf, names, data_dir, "data/")
|
|
435
|
+
n_ws = 0
|
|
436
|
+
if include_workspace:
|
|
437
|
+
Path(cur_wd).mkdir(parents=True, exist_ok=True)
|
|
438
|
+
n_ws = _extract(zf, names, Path(cur_wd), "workspace/")
|
|
439
|
+
finally:
|
|
440
|
+
zf.close()
|
|
441
|
+
|
|
442
|
+
# 路径重映射:长前缀优先(工作目录可能在数据目录之下,避免被短前缀截走)
|
|
443
|
+
old_data = str(m.get("data_dir") or "")
|
|
444
|
+
old_wd = str(m.get("default_workdir") or "")
|
|
445
|
+
pairs = []
|
|
446
|
+
if remap:
|
|
447
|
+
for old, new in ((old_wd, cur_wd), (old_data, str(data_dir))):
|
|
448
|
+
if old and old.lower() != new.lower():
|
|
449
|
+
pairs.append((old, new))
|
|
450
|
+
pairs.sort(key=lambda x: len(x[0]), reverse=True)
|
|
451
|
+
remapped_files = remapped_values = 0
|
|
452
|
+
if pairs:
|
|
453
|
+
remapped_files, remapped_values = _remap_all_json(pairs)
|
|
454
|
+
|
|
455
|
+
# 导入的 settings.json 里若带着旧机器默认路径,上面已重映射;把当前生效值对齐
|
|
456
|
+
try:
|
|
457
|
+
from . import store
|
|
458
|
+
with store.LOCK:
|
|
459
|
+
store._TASKS.clear()
|
|
460
|
+
store._RUNS.clear()
|
|
461
|
+
store.load_all()
|
|
462
|
+
store.bump_state()
|
|
463
|
+
except Exception:
|
|
464
|
+
pass
|
|
465
|
+
return {"mode": mode, "data_files": n_data, "workspace_files": n_ws,
|
|
466
|
+
"remap": {"applied": bool(pairs), "files": remapped_files,
|
|
467
|
+
"values": remapped_values, "pairs": [[o, n] for o, n in pairs]},
|
|
468
|
+
"backup_path": str(backup_path),
|
|
469
|
+
"external_workdirs": m.get("external_workdirs") or [],
|
|
470
|
+
"restart_required": True}
|