codebee 0.1.68 → 0.1.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +580 -543
- package/LICENSE +21 -21
- package/README.md +11 -0
- package/app/browser-final-1440.png +0 -0
- package/app/browser-final-390.png +0 -0
- package/app/browser-google-fallback.png +0 -0
- package/app/browser-google-test.png +0 -0
- package/app/browser-workbench-1440.png +0 -0
- package/app/browser-workbench-390.png +0 -0
- package/app/browser-workspace-google.png +0 -0
- package/app/browser-workspace-smoke.png +0 -0
- package/app/browser-workspace.png +0 -0
- package/app/core/acceptance.py +100 -100
- package/app/core/aiflavor.py +197 -197
- package/app/core/artifacts.py +148 -148
- package/app/core/assets.py +110 -110
- package/app/core/attachments.py +623 -623
- package/app/core/audit.py +30 -0
- package/app/core/automation.py +853 -758
- package/app/core/backup.py +470 -470
- package/app/core/beekeeper.py +183 -183
- package/app/core/benchstore.py +95 -0
- package/app/core/board.py +209 -209
- package/app/core/bookmeta.py +935 -930
- package/app/core/bookmeta_catalog.py +180 -180
- package/app/core/branching.py +137 -137
- package/app/core/builtin_agent.py +1673 -1234
- package/app/core/capability.py +144 -144
- package/app/core/catalog.py +7 -1
- package/app/core/chaptersafe.py +32 -32
- package/app/core/cleanup.py +303 -303
- package/app/core/compaction.py +201 -201
- package/app/core/covergen.py +356 -356
- package/app/core/defectretro.py +43 -43
- package/app/core/diagnostics.py +114 -114
- package/app/core/dispatch.py +190 -173
- package/app/core/dispatch_log.py +128 -128
- package/app/core/env_scrub.py +85 -85
- package/app/core/error_codes.py +33 -24
- package/app/core/errorlog.py +137 -137
- package/app/core/evalbench.py +720 -0
- package/app/core/evaluation.py +174 -174
- package/app/core/failure_notify.py +290 -290
- package/app/core/flows.py +228 -3
- package/app/core/goal_service.py +167 -167
- package/app/core/health.py +374 -374
- package/app/core/history.py +38 -38
- package/app/core/hooks.py +162 -162
- package/app/core/jobs.py +51 -30
- package/app/core/manager.py +2143 -2122
- package/app/core/market.py +362 -362
- package/app/core/market_remote.py +10 -0
- package/app/core/mcp_client.py +287 -0
- package/app/core/mocks.py +64 -64
- package/app/core/modelhub.py +193 -10
- package/app/core/notify.py +201 -106
- package/app/core/novel_quality.py +189 -0
- package/app/core/operations.py +228 -228
- package/app/core/paihang.py +138 -138
- package/app/core/paths.py +6 -2
- package/app/core/pipeline.py +426 -28
- package/app/core/planner.py +243 -22
- package/app/core/plugins.py +525 -0
- package/app/core/portguard.py +111 -111
- package/app/core/preview.py +293 -293
- package/app/core/publish/__init__.py +7 -7
- package/app/core/publish/auto.py +367 -367
- package/app/core/publish/browser.py +655 -539
- package/app/core/publish/fanqie.py +165 -159
- package/app/core/publish/flow.py +589 -440
- package/app/core/publish/flows-fanqie-calibrated.json +336 -332
- package/app/core/publish/ledger.py +229 -203
- package/app/core/publish/manager.py +824 -692
- package/app/core/publish/qimao.py +146 -141
- package/app/core/publish/ws.py +139 -139
- package/app/core/redact.py +38 -38
- package/app/core/remotesync.py +121 -0
- package/app/core/repeat_guard.py +129 -129
- package/app/core/router.py +398 -398
- package/app/core/runner.py +2142 -2067
- package/app/core/selfupdate.py +384 -384
- package/app/core/session_log.py +161 -161
- package/app/core/sessions.py +312 -312
- package/app/core/settings.py +186 -149
- package/app/core/settings_schema.py +270 -262
- package/app/core/share_page.py +114 -114
- package/app/core/skill_scan.py +122 -122
- package/app/core/skillpacks/fanqie-novel.md +80 -80
- package/app/core/skillpacks/market/character-bible.md +66 -66
- package/app/core/skillpacks/market/code-risk-checklist.md +58 -58
- package/app/core/skillpacks/market/git-workflow.md +57 -57
- package/app/core/skillpacks/market/release-notes.md +72 -72
- package/app/core/skillpacks/market/weekly-report.md +71 -71
- package/app/core/skillpacks/market/worldview-consistency.md +70 -70
- package/app/core/step_runner.py +132 -132
- package/app/core/store.py +104 -82
- package/app/core/task_compile.py +199 -199
- package/app/core/telemetry.py +292 -292
- package/app/core/tlsctx.py +64 -64
- package/app/core/token_meter.py +147 -147
- package/app/core/tracing.py +144 -144
- package/app/core/upstream.py +41 -41
- package/app/core/usage.py +1035 -862
- package/app/core/volumes.py +389 -389
- package/app/core/wxdigest.py +858 -858
- package/app/core/zentao.py +2614 -2465
- package/app/current-browser.png +0 -0
- package/app/main.py +3245 -2981
- package/app/mcp_server.py +175 -0
- package/app/pet.py +82 -21
- package/app/pick_dialog.py +110 -110
- package/app/plugins/code-review-helper/.codex-plugin/plugin.json +18 -0
- package/app/plugins/code-review-helper/skills/review.md +16 -0
- package/app/ui/app.js +15706 -14399
- package/app/ui/artdiff.js +74 -74
- package/app/ui/board.html +162 -162
- package/app/ui/board.js +305 -305
- package/app/ui/i18n.js +3012 -2770
- package/app/ui/icons/bee.svg +79 -79
- package/app/ui/index.html +277 -13
- package/app/ui/manifest.json +16 -16
- package/app/ui/qrcode.js +2297 -2297
- package/app/ui/style.css +802 -16
- package/bin/tutti.js +147 -147
- package/package.json +47 -46
- package/tools/wx_reader.py +261 -261
- package/app/.playwright-cli/console-2026-09-23T08-26-03-635Z.log +0 -88
- package/app/.playwright-cli/page-2026-09-23T08-26-03-845Z.yml +0 -126
- package/app/.playwright-cli/page-2026-09-23T08-29-44-203Z.png +0 -0
- package/app/.playwright-cli/page-2026-09-23T08-35-42-046Z.yml +0 -210
- package/app/.playwright-cli/page-2026-09-23T08-36-52-858Z.png +0 -0
- package/app/.playwright-cli/page-2026-09-23T08-37-33-938Z.png +0 -0
- package/app/.playwright-cli/run-detail-after.png +0 -0
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Deterministic quality checks for signing-oriented Chinese web fiction.
|
|
3
|
+
|
|
4
|
+
The LLM remains responsible for literary judgment. This module supplies
|
|
5
|
+
stable rubric defaults and small text signals so reviewers can cite evidence
|
|
6
|
+
for the common rejection reasons instead of relying on vague impressions.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
SIGNING_RUBRIC = [
|
|
14
|
+
"开篇吸引力",
|
|
15
|
+
"情节推进",
|
|
16
|
+
"文风统一",
|
|
17
|
+
"情感细腻度",
|
|
18
|
+
"节奏控制",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
OPENING_DIM = "开篇吸引力"
|
|
22
|
+
SIGNING_CHECKPOINT_WORDS = 20000 # 平台口径:前 2 万字用于签约评估
|
|
23
|
+
|
|
24
|
+
_SENTENCE_RE = re.compile(r"[^。!?!?;;\n]+[。!?!?;;]?")
|
|
25
|
+
_DIALOGUE_RE = re.compile(r"[“‘\"「『](.*?)[”’\"」』]", re.S)
|
|
26
|
+
_EMOTION_RE = re.compile(r"哭|泪|笑|怒|怕|颤|抖|咬|攥|握紧|心跳|呼吸|喉咙|指甲|沉默")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def rubric_for(task):
|
|
30
|
+
"""Return the configured rubric, or the signing rubric for serial novels."""
|
|
31
|
+
task = task if isinstance(task, dict) else {}
|
|
32
|
+
raw = task.get("rubric")
|
|
33
|
+
if isinstance(raw, str):
|
|
34
|
+
raw = [x.strip() for x in raw.replace(",", ",").split(",") if x.strip()]
|
|
35
|
+
if isinstance(raw, (list, tuple)):
|
|
36
|
+
values = [str(x).strip() for x in raw if str(x).strip()]
|
|
37
|
+
if values:
|
|
38
|
+
return values[:8]
|
|
39
|
+
if str(task.get("type") or "").lower() == "serial_novel":
|
|
40
|
+
return list(SIGNING_RUBRIC)
|
|
41
|
+
return []
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def opening_requirements(chapter, target_words=None):
|
|
45
|
+
"""Return concise, chapter-aware acceptance requirements for prompts."""
|
|
46
|
+
try:
|
|
47
|
+
chapter = int(chapter)
|
|
48
|
+
except (TypeError, ValueError):
|
|
49
|
+
chapter = 1
|
|
50
|
+
if chapter == 1:
|
|
51
|
+
try:
|
|
52
|
+
short = int(target_words or 0) < 1200
|
|
53
|
+
except (TypeError, ValueError):
|
|
54
|
+
short = False
|
|
55
|
+
core = ("前 1200 字" if short else "前 2000 字")
|
|
56
|
+
return ("黄金一章门禁:前 100 字让读者知道主角是谁;前 200 字出现异常、欲望或冲突;"
|
|
57
|
+
"前 600 字把困境具体化;%s 展示核心卖点;结尾必须留下可兑现的钩子。" % core
|
|
58
|
+
+ "禁止用世界观、环境或人物关系长铺垫开场。")
|
|
59
|
+
return "本章门禁:开头尽快承接上一章并制造新变化;结尾必须留下下一步行动或可兑现的章末钩子。"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def signal_summary(text, chapter=None):
|
|
63
|
+
"""Extract explainable text signals; no claim is made that they are scores."""
|
|
64
|
+
text = str(text or "").replace("\r\n", "\n").strip()
|
|
65
|
+
paragraphs = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
|
|
66
|
+
sentences = [s.strip() for s in _SENTENCE_RE.findall(text) if s.strip()]
|
|
67
|
+
sentence_lengths = [len(re.sub(r"\s+", "", s)) for s in sentences]
|
|
68
|
+
dialogue_chars = sum(len(x) for x in _DIALOGUE_RE.findall(text))
|
|
69
|
+
emotion_hits = len(_EMOTION_RE.findall(text))
|
|
70
|
+
chars = len(re.sub(r"\s+", "", text))
|
|
71
|
+
avg = round(sum(sentence_lengths) / len(sentence_lengths), 1) if sentence_lengths else 0.0
|
|
72
|
+
long_sentences = sum(1 for n in sentence_lengths if n >= 55)
|
|
73
|
+
hints = []
|
|
74
|
+
if chapter == 1 and chars and chars < 600:
|
|
75
|
+
hints.append("首章正文不足 600 字,无法证明前 600 字已进入具体困境")
|
|
76
|
+
if avg >= 35 or long_sentences:
|
|
77
|
+
hints.append("存在偏长句,复查长定语、重复说明与动作拆分")
|
|
78
|
+
if len(paragraphs) >= 4 and chars / len(paragraphs) > 220:
|
|
79
|
+
hints.append("段落偏长,复查手机阅读下的节奏与信息密度")
|
|
80
|
+
if text and dialogue_chars == 0:
|
|
81
|
+
hints.append("缺少对白证据,复查是否用场景/行动推进冲突")
|
|
82
|
+
if text and emotion_hits == 0:
|
|
83
|
+
hints.append("缺少可见的情绪动作或生理反应,复查复杂情感是否写得过于概括")
|
|
84
|
+
return {
|
|
85
|
+
"chapter": chapter,
|
|
86
|
+
"chars": chars,
|
|
87
|
+
"paragraphs": len(paragraphs),
|
|
88
|
+
"sentences": len(sentences),
|
|
89
|
+
"avg_sentence_chars": avg,
|
|
90
|
+
"long_sentences": long_sentences,
|
|
91
|
+
"dialogue_chars": dialogue_chars,
|
|
92
|
+
"emotion_action_hits": emotion_hits,
|
|
93
|
+
"hints": hints[:6],
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def format_signal_summary(summary):
|
|
98
|
+
"""Render signal_summary output as a compact prompt block."""
|
|
99
|
+
if not isinstance(summary, dict):
|
|
100
|
+
return ""
|
|
101
|
+
parts = [
|
|
102
|
+
"文本信号(仅作证据,不替代文学判断):约 %s 字;%s 段;%s 句;平均句长 %s 字;长句 %s;对白 %s 字;情绪动作线索 %s 处。"
|
|
103
|
+
% (summary.get("chars", 0), summary.get("paragraphs", 0),
|
|
104
|
+
summary.get("sentences", 0), summary.get("avg_sentence_chars", 0),
|
|
105
|
+
summary.get("long_sentences", 0), summary.get("dialogue_chars", 0),
|
|
106
|
+
summary.get("emotion_action_hits", 0))]
|
|
107
|
+
hints = summary.get("hints") or []
|
|
108
|
+
if hints:
|
|
109
|
+
parts.append("确定性复查提示:" + ";".join(str(x) for x in hints))
|
|
110
|
+
return "\n".join(parts)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def style_anchor_block(anchor):
|
|
114
|
+
"""Render the book-level style anchor as a prompt block; empty when absent."""
|
|
115
|
+
anchor = str(anchor or "").strip()
|
|
116
|
+
if not anchor:
|
|
117
|
+
return ""
|
|
118
|
+
return ("## 全书文风锚(大纲裁定,所有章节起草/修订/评审必须遵守;"
|
|
119
|
+
"与其冲突处以本锚为准)\n%s\n" % anchor[:400])
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def fingerprint(summary):
|
|
123
|
+
"""Normalize a signal_summary into per-1000-char style metrics."""
|
|
124
|
+
if not isinstance(summary, dict):
|
|
125
|
+
return {}
|
|
126
|
+
chars = max(1, int(summary.get("chars") or 0))
|
|
127
|
+
return {
|
|
128
|
+
"avg_sentence_chars": float(summary.get("avg_sentence_chars") or 0.0),
|
|
129
|
+
"dialogue_per_k": round(1000.0 * float(summary.get("dialogue_chars") or 0) / chars, 1),
|
|
130
|
+
"emotion_per_k": round(1000.0 * float(summary.get("emotion_action_hits") or 0) / chars, 1),
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _median(values):
|
|
135
|
+
xs = sorted(float(v) for v in values if isinstance(v, (int, float)))
|
|
136
|
+
if not xs:
|
|
137
|
+
return 0.0
|
|
138
|
+
mid = len(xs) // 2
|
|
139
|
+
return xs[mid] if len(xs) % 2 else round((xs[mid - 1] + xs[mid]) / 2.0, 1)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def style_deviation_notes(baselines, summary, min_baseline=2):
|
|
143
|
+
"""Cross-chapter style drift warnings from deterministic fingerprints.
|
|
144
|
+
|
|
145
|
+
baselines: signal_summary dicts of the book's previous chapters. Needs at
|
|
146
|
+
least ``min_baseline`` chapters before speaking (开篇几章没有稳定基线);
|
|
147
|
+
all notes are evidence hints for reviewers/revisers, never scores.
|
|
148
|
+
"""
|
|
149
|
+
notes = []
|
|
150
|
+
base = [fingerprint(b) for b in (baselines or []) if isinstance(b, dict)]
|
|
151
|
+
base = [b for b in base if b]
|
|
152
|
+
if len(base) < min_baseline:
|
|
153
|
+
return notes
|
|
154
|
+
cur = fingerprint(summary)
|
|
155
|
+
if not cur:
|
|
156
|
+
return notes
|
|
157
|
+
avg_base = _median([b["avg_sentence_chars"] for b in base])
|
|
158
|
+
if avg_base >= 12 and cur["avg_sentence_chars"] > avg_base * 1.45:
|
|
159
|
+
notes.append("本章平均句长 %s 字,此前各章中位 %s 字——句式明显变长,"
|
|
160
|
+
"复查长定语、说明性插叙与节奏拖沓"
|
|
161
|
+
% (cur["avg_sentence_chars"], avg_base))
|
|
162
|
+
elif avg_base >= 12 and cur["avg_sentence_chars"] < avg_base * 0.62:
|
|
163
|
+
notes.append("本章平均句长 %s 字,此前各章中位 %s 字——句式明显变碎,"
|
|
164
|
+
"复查是否为凑字数拆散动作与因果"
|
|
165
|
+
% (cur["avg_sentence_chars"], avg_base))
|
|
166
|
+
dial_base = _median([b["dialogue_per_k"] for b in base])
|
|
167
|
+
if dial_base >= 40 and cur["dialogue_per_k"] < dial_base * 0.45:
|
|
168
|
+
notes.append("对白占比骤降(本章 %s/千字,此前中位 %s/千字)——"
|
|
169
|
+
"复查是否换成了大段叙述,文风与推进方式是否漂移"
|
|
170
|
+
% (cur["dialogue_per_k"], dial_base))
|
|
171
|
+
emo_base = _median([b["emotion_per_k"] for b in base])
|
|
172
|
+
if emo_base >= 1.5 and cur["emotion_per_k"] < emo_base * 0.4:
|
|
173
|
+
notes.append("情绪动作密度骤降(本章 %s/千字,此前中位 %s/千字)——"
|
|
174
|
+
"复查复杂情感是否退回「很伤心/很愤怒」式概括"
|
|
175
|
+
% (cur["emotion_per_k"], emo_base))
|
|
176
|
+
return notes
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def opening_below(means, threshold):
|
|
180
|
+
"""True when the signing opening dimension is scored below the line."""
|
|
181
|
+
try:
|
|
182
|
+
value = float((means or {}).get(OPENING_DIM))
|
|
183
|
+
except (TypeError, ValueError):
|
|
184
|
+
return False
|
|
185
|
+
try:
|
|
186
|
+
line = float(threshold)
|
|
187
|
+
except (TypeError, ValueError):
|
|
188
|
+
return False
|
|
189
|
+
return value < line
|
package/app/core/operations.py
CHANGED
|
@@ -1,228 +1,228 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
"""Append-only external operation ledger.
|
|
3
|
-
|
|
4
|
-
External writes cannot promise exactly-once delivery. Each write therefore
|
|
5
|
-
has a stable operation id and an explicit state: pending, confirmed, failed or
|
|
6
|
-
unknown. Unknown means the caller must reconcile with the remote system before
|
|
7
|
-
retrying; it is never treated as success.
|
|
8
|
-
"""
|
|
9
|
-
from __future__ import annotations
|
|
10
|
-
|
|
11
|
-
import hashlib
|
|
12
|
-
import json
|
|
13
|
-
import secrets
|
|
14
|
-
import threading
|
|
15
|
-
import time
|
|
16
|
-
import urllib.error
|
|
17
|
-
from pathlib import Path
|
|
18
|
-
|
|
19
|
-
from . import paths
|
|
20
|
-
|
|
21
|
-
LOCK = threading.RLock()
|
|
22
|
-
STATUSES = ("pending", "confirmed", "failed", "unknown")
|
|
23
|
-
PENDING_TTL_SECONDS = 15 * 60
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
def _file(day=None) -> Path:
|
|
27
|
-
day = day or time.strftime("%Y-%m-%d")
|
|
28
|
-
return paths.DATA_DIR / "operations" / ("operations-%s.jsonl" % day[:7].replace("-", ""))
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
def _read_all():
|
|
32
|
-
rows = []
|
|
33
|
-
root = paths.DATA_DIR / "operations"
|
|
34
|
-
try:
|
|
35
|
-
files = sorted(root.glob("operations-*.jsonl"))
|
|
36
|
-
except OSError:
|
|
37
|
-
files = []
|
|
38
|
-
for fp in files:
|
|
39
|
-
try:
|
|
40
|
-
for line in fp.read_text(encoding="utf-8").splitlines():
|
|
41
|
-
try:
|
|
42
|
-
row = json.loads(line)
|
|
43
|
-
except (TypeError, ValueError):
|
|
44
|
-
continue
|
|
45
|
-
if isinstance(row, dict) and row.get("operation_id"):
|
|
46
|
-
rows.append(row)
|
|
47
|
-
except OSError:
|
|
48
|
-
continue
|
|
49
|
-
return rows
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
def _write_all(rows):
|
|
53
|
-
"""Test/support migration helper; production writes use append-only _append."""
|
|
54
|
-
root = paths.DATA_DIR / "operations"
|
|
55
|
-
root.mkdir(parents=True, exist_ok=True)
|
|
56
|
-
fp = _file()
|
|
57
|
-
fp.write_text("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in rows),
|
|
58
|
-
encoding="utf-8")
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
def _append(row):
|
|
62
|
-
try:
|
|
63
|
-
fp = _file(row.get("submitted_at") and
|
|
64
|
-
time.strftime("%Y-%m-%d", time.localtime(float(row["submitted_at"]))))
|
|
65
|
-
fp.parent.mkdir(parents=True, exist_ok=True)
|
|
66
|
-
with fp.open("a", encoding="utf-8") as handle:
|
|
67
|
-
handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")
|
|
68
|
-
handle.flush()
|
|
69
|
-
except Exception:
|
|
70
|
-
# Observability must not turn a remote write into a local failure.
|
|
71
|
-
pass
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
def _request_hash(request) -> str:
|
|
75
|
-
raw = json.dumps(request or {}, ensure_ascii=False, sort_keys=True,
|
|
76
|
-
separators=(",", ":"), default=str)
|
|
77
|
-
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
def _operation_id(target: str) -> str:
|
|
81
|
-
stamp = time.strftime("%Y%m%d-%H%M%S")
|
|
82
|
-
return "op-%s-%s-%s" % (stamp, secrets.token_hex(4),
|
|
83
|
-
hashlib.sha256(str(target).encode()).hexdigest()[:6])
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
def _safe_meta(metadata):
|
|
87
|
-
if not isinstance(metadata, dict):
|
|
88
|
-
return {}
|
|
89
|
-
out = {}
|
|
90
|
-
for key, value in metadata.items():
|
|
91
|
-
name = str(key)[:40]
|
|
92
|
-
if any(token in name.lower() for token in ("key", "token", "password", "secret")):
|
|
93
|
-
continue
|
|
94
|
-
if isinstance(value, (str, int, float, bool)) or value is None:
|
|
95
|
-
out[name] = str(value)[:300] if isinstance(value, str) else value
|
|
96
|
-
return out
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
def begin(target, request=None, *, task_id="", run_id="", metadata=None):
|
|
100
|
-
operation_id = _operation_id(target)
|
|
101
|
-
now = time.time()
|
|
102
|
-
row = {
|
|
103
|
-
"operation_id": operation_id,
|
|
104
|
-
"target": str(target or "")[:160],
|
|
105
|
-
"request_hash": _request_hash(request),
|
|
106
|
-
"submitted_at": now,
|
|
107
|
-
"status": "pending",
|
|
108
|
-
"remote_receipt": "",
|
|
109
|
-
"error": "",
|
|
110
|
-
"last_reconciled_at": None,
|
|
111
|
-
"task_id": str(task_id or "")[:80],
|
|
112
|
-
"run_id": str(run_id or "")[:80],
|
|
113
|
-
"metadata": _safe_meta(metadata),
|
|
114
|
-
}
|
|
115
|
-
with LOCK:
|
|
116
|
-
_append(row)
|
|
117
|
-
return operation_id
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
def update(operation_id, status, *, remote_receipt="", error="",
|
|
121
|
-
metadata=None, reconciled=False):
|
|
122
|
-
operation_id = str(operation_id or "").strip()
|
|
123
|
-
status = str(status or "").strip().lower()
|
|
124
|
-
if not operation_id or status not in STATUSES:
|
|
125
|
-
return False
|
|
126
|
-
with LOCK:
|
|
127
|
-
# Read and transition-check under the same lock as append. Two
|
|
128
|
-
# concurrent callbacks must not race a pending operation into
|
|
129
|
-
# conflicting terminal states.
|
|
130
|
-
current = get(operation_id)
|
|
131
|
-
if current is None:
|
|
132
|
-
return False
|
|
133
|
-
current_status = str(current.get("status") or "pending")
|
|
134
|
-
# External effects are monotonic: a confirmed write must never be
|
|
135
|
-
# downgraded by cleanup/error handling. Unknown/failed can only be
|
|
136
|
-
# promoted by an explicit remote reconciliation.
|
|
137
|
-
if current_status == "confirmed" and status != "confirmed":
|
|
138
|
-
return False
|
|
139
|
-
if current_status in ("unknown", "failed") and status == "confirmed" and not reconciled:
|
|
140
|
-
return False
|
|
141
|
-
if current_status in ("unknown", "failed") and status != current_status and not reconciled:
|
|
142
|
-
return False
|
|
143
|
-
if status == "confirmed" and not remote_receipt:
|
|
144
|
-
remote_receipt = current.get("remote_receipt") or ""
|
|
145
|
-
now = time.time()
|
|
146
|
-
row = {
|
|
147
|
-
"operation_id": operation_id,
|
|
148
|
-
"status": status,
|
|
149
|
-
"remote_receipt": str(remote_receipt or "")[:300],
|
|
150
|
-
"error": str(error or "")[:500],
|
|
151
|
-
"last_reconciled_at": now if reconciled else None,
|
|
152
|
-
"updated_at": now,
|
|
153
|
-
}
|
|
154
|
-
if metadata:
|
|
155
|
-
row["metadata"] = _safe_meta(metadata)
|
|
156
|
-
_append(row)
|
|
157
|
-
return True
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
def confirm(operation_id, remote_receipt="", *, metadata=None):
|
|
161
|
-
return update(operation_id, "confirmed", remote_receipt=remote_receipt,
|
|
162
|
-
metadata=metadata)
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
def fail(operation_id, error="", *, metadata=None):
|
|
166
|
-
return update(operation_id, "failed", error=error, metadata=metadata)
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
def mark_unknown(operation_id, error="", *, metadata=None):
|
|
170
|
-
return update(operation_id, "unknown", error=error, metadata=metadata)
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
def reconcile(operation_id, status, remote_receipt="", error="", metadata=None):
|
|
174
|
-
"""Record a remote lookup result for an unknown operation."""
|
|
175
|
-
return update(operation_id, status, remote_receipt=remote_receipt, error=error,
|
|
176
|
-
metadata=metadata, reconciled=True)
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
def get(operation_id):
|
|
180
|
-
operation_id = str(operation_id or "")
|
|
181
|
-
latest = None
|
|
182
|
-
with LOCK:
|
|
183
|
-
for row in _read_all():
|
|
184
|
-
if row.get("operation_id") == operation_id:
|
|
185
|
-
latest = dict(latest or {}, **row)
|
|
186
|
-
return latest
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
def recent(*, target="", status="", limit=100):
|
|
190
|
-
latest = {}
|
|
191
|
-
with LOCK:
|
|
192
|
-
for row in _read_all():
|
|
193
|
-
if target and str(row.get("target") or "") != str(target):
|
|
194
|
-
continue
|
|
195
|
-
latest[row["operation_id"]] = dict(latest.get(row["operation_id"]) or {}, **row)
|
|
196
|
-
rows = list(latest.values())
|
|
197
|
-
if status:
|
|
198
|
-
rows = [r for r in rows if r.get("status") == status]
|
|
199
|
-
rows.sort(key=lambda r: float(r.get("updated_at") or r.get("submitted_at") or 0), reverse=True)
|
|
200
|
-
return rows[:max(1, min(1000, int(limit or 100)))]
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
def recover_pending(*, now=None, ttl_seconds=PENDING_TTL_SECONDS):
|
|
204
|
-
"""Turn stale pending writes into unknown so restart never replays blindly."""
|
|
205
|
-
now = time.time() if now is None else float(now)
|
|
206
|
-
changed = 0
|
|
207
|
-
for row in recent(status="pending", limit=1000):
|
|
208
|
-
submitted = float(row.get("submitted_at") or 0)
|
|
209
|
-
if submitted and now - submitted > max(1, float(ttl_seconds)):
|
|
210
|
-
if mark_unknown(row["operation_id"], "进程重启后未收到远端回执,请先对账"):
|
|
211
|
-
changed += 1
|
|
212
|
-
return changed
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
def status_for_exception(error) -> str:
|
|
216
|
-
text = str(error or "").lower()
|
|
217
|
-
if isinstance(error, (TimeoutError, ConnectionError, urllib.error.URLError)) or any(x in text for x in
|
|
218
|
-
("timeout", "timed out", "超时", "connection", "connect", "connection reset",
|
|
219
|
-
"断开", "断连", "连不上", "网络", "network", "unreachable", "disconnect")):
|
|
220
|
-
return "unknown"
|
|
221
|
-
return "failed"
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
def finish_exception(operation_id, error):
|
|
225
|
-
status = status_for_exception(error)
|
|
226
|
-
if status == "unknown":
|
|
227
|
-
return mark_unknown(operation_id, str(error))
|
|
228
|
-
return fail(operation_id, str(error))
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Append-only external operation ledger.
|
|
3
|
+
|
|
4
|
+
External writes cannot promise exactly-once delivery. Each write therefore
|
|
5
|
+
has a stable operation id and an explicit state: pending, confirmed, failed or
|
|
6
|
+
unknown. Unknown means the caller must reconcile with the remote system before
|
|
7
|
+
retrying; it is never treated as success.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import json
|
|
13
|
+
import secrets
|
|
14
|
+
import threading
|
|
15
|
+
import time
|
|
16
|
+
import urllib.error
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
from . import paths
|
|
20
|
+
|
|
21
|
+
LOCK = threading.RLock()
|
|
22
|
+
STATUSES = ("pending", "confirmed", "failed", "unknown")
|
|
23
|
+
PENDING_TTL_SECONDS = 15 * 60
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _file(day=None) -> Path:
|
|
27
|
+
day = day or time.strftime("%Y-%m-%d")
|
|
28
|
+
return paths.DATA_DIR / "operations" / ("operations-%s.jsonl" % day[:7].replace("-", ""))
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _read_all():
|
|
32
|
+
rows = []
|
|
33
|
+
root = paths.DATA_DIR / "operations"
|
|
34
|
+
try:
|
|
35
|
+
files = sorted(root.glob("operations-*.jsonl"))
|
|
36
|
+
except OSError:
|
|
37
|
+
files = []
|
|
38
|
+
for fp in files:
|
|
39
|
+
try:
|
|
40
|
+
for line in fp.read_text(encoding="utf-8").splitlines():
|
|
41
|
+
try:
|
|
42
|
+
row = json.loads(line)
|
|
43
|
+
except (TypeError, ValueError):
|
|
44
|
+
continue
|
|
45
|
+
if isinstance(row, dict) and row.get("operation_id"):
|
|
46
|
+
rows.append(row)
|
|
47
|
+
except OSError:
|
|
48
|
+
continue
|
|
49
|
+
return rows
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _write_all(rows):
|
|
53
|
+
"""Test/support migration helper; production writes use append-only _append."""
|
|
54
|
+
root = paths.DATA_DIR / "operations"
|
|
55
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
56
|
+
fp = _file()
|
|
57
|
+
fp.write_text("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in rows),
|
|
58
|
+
encoding="utf-8")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _append(row):
|
|
62
|
+
try:
|
|
63
|
+
fp = _file(row.get("submitted_at") and
|
|
64
|
+
time.strftime("%Y-%m-%d", time.localtime(float(row["submitted_at"]))))
|
|
65
|
+
fp.parent.mkdir(parents=True, exist_ok=True)
|
|
66
|
+
with fp.open("a", encoding="utf-8") as handle:
|
|
67
|
+
handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")
|
|
68
|
+
handle.flush()
|
|
69
|
+
except Exception:
|
|
70
|
+
# Observability must not turn a remote write into a local failure.
|
|
71
|
+
pass
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _request_hash(request) -> str:
|
|
75
|
+
raw = json.dumps(request or {}, ensure_ascii=False, sort_keys=True,
|
|
76
|
+
separators=(",", ":"), default=str)
|
|
77
|
+
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _operation_id(target: str) -> str:
|
|
81
|
+
stamp = time.strftime("%Y%m%d-%H%M%S")
|
|
82
|
+
return "op-%s-%s-%s" % (stamp, secrets.token_hex(4),
|
|
83
|
+
hashlib.sha256(str(target).encode()).hexdigest()[:6])
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _safe_meta(metadata):
|
|
87
|
+
if not isinstance(metadata, dict):
|
|
88
|
+
return {}
|
|
89
|
+
out = {}
|
|
90
|
+
for key, value in metadata.items():
|
|
91
|
+
name = str(key)[:40]
|
|
92
|
+
if any(token in name.lower() for token in ("key", "token", "password", "secret")):
|
|
93
|
+
continue
|
|
94
|
+
if isinstance(value, (str, int, float, bool)) or value is None:
|
|
95
|
+
out[name] = str(value)[:300] if isinstance(value, str) else value
|
|
96
|
+
return out
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def begin(target, request=None, *, task_id="", run_id="", metadata=None):
|
|
100
|
+
operation_id = _operation_id(target)
|
|
101
|
+
now = time.time()
|
|
102
|
+
row = {
|
|
103
|
+
"operation_id": operation_id,
|
|
104
|
+
"target": str(target or "")[:160],
|
|
105
|
+
"request_hash": _request_hash(request),
|
|
106
|
+
"submitted_at": now,
|
|
107
|
+
"status": "pending",
|
|
108
|
+
"remote_receipt": "",
|
|
109
|
+
"error": "",
|
|
110
|
+
"last_reconciled_at": None,
|
|
111
|
+
"task_id": str(task_id or "")[:80],
|
|
112
|
+
"run_id": str(run_id or "")[:80],
|
|
113
|
+
"metadata": _safe_meta(metadata),
|
|
114
|
+
}
|
|
115
|
+
with LOCK:
|
|
116
|
+
_append(row)
|
|
117
|
+
return operation_id
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def update(operation_id, status, *, remote_receipt="", error="",
|
|
121
|
+
metadata=None, reconciled=False):
|
|
122
|
+
operation_id = str(operation_id or "").strip()
|
|
123
|
+
status = str(status or "").strip().lower()
|
|
124
|
+
if not operation_id or status not in STATUSES:
|
|
125
|
+
return False
|
|
126
|
+
with LOCK:
|
|
127
|
+
# Read and transition-check under the same lock as append. Two
|
|
128
|
+
# concurrent callbacks must not race a pending operation into
|
|
129
|
+
# conflicting terminal states.
|
|
130
|
+
current = get(operation_id)
|
|
131
|
+
if current is None:
|
|
132
|
+
return False
|
|
133
|
+
current_status = str(current.get("status") or "pending")
|
|
134
|
+
# External effects are monotonic: a confirmed write must never be
|
|
135
|
+
# downgraded by cleanup/error handling. Unknown/failed can only be
|
|
136
|
+
# promoted by an explicit remote reconciliation.
|
|
137
|
+
if current_status == "confirmed" and status != "confirmed":
|
|
138
|
+
return False
|
|
139
|
+
if current_status in ("unknown", "failed") and status == "confirmed" and not reconciled:
|
|
140
|
+
return False
|
|
141
|
+
if current_status in ("unknown", "failed") and status != current_status and not reconciled:
|
|
142
|
+
return False
|
|
143
|
+
if status == "confirmed" and not remote_receipt:
|
|
144
|
+
remote_receipt = current.get("remote_receipt") or ""
|
|
145
|
+
now = time.time()
|
|
146
|
+
row = {
|
|
147
|
+
"operation_id": operation_id,
|
|
148
|
+
"status": status,
|
|
149
|
+
"remote_receipt": str(remote_receipt or "")[:300],
|
|
150
|
+
"error": str(error or "")[:500],
|
|
151
|
+
"last_reconciled_at": now if reconciled else None,
|
|
152
|
+
"updated_at": now,
|
|
153
|
+
}
|
|
154
|
+
if metadata:
|
|
155
|
+
row["metadata"] = _safe_meta(metadata)
|
|
156
|
+
_append(row)
|
|
157
|
+
return True
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def confirm(operation_id, remote_receipt="", *, metadata=None):
|
|
161
|
+
return update(operation_id, "confirmed", remote_receipt=remote_receipt,
|
|
162
|
+
metadata=metadata)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def fail(operation_id, error="", *, metadata=None):
|
|
166
|
+
return update(operation_id, "failed", error=error, metadata=metadata)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def mark_unknown(operation_id, error="", *, metadata=None):
|
|
170
|
+
return update(operation_id, "unknown", error=error, metadata=metadata)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def reconcile(operation_id, status, remote_receipt="", error="", metadata=None):
|
|
174
|
+
"""Record a remote lookup result for an unknown operation."""
|
|
175
|
+
return update(operation_id, status, remote_receipt=remote_receipt, error=error,
|
|
176
|
+
metadata=metadata, reconciled=True)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def get(operation_id):
|
|
180
|
+
operation_id = str(operation_id or "")
|
|
181
|
+
latest = None
|
|
182
|
+
with LOCK:
|
|
183
|
+
for row in _read_all():
|
|
184
|
+
if row.get("operation_id") == operation_id:
|
|
185
|
+
latest = dict(latest or {}, **row)
|
|
186
|
+
return latest
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def recent(*, target="", status="", limit=100):
|
|
190
|
+
latest = {}
|
|
191
|
+
with LOCK:
|
|
192
|
+
for row in _read_all():
|
|
193
|
+
if target and str(row.get("target") or "") != str(target):
|
|
194
|
+
continue
|
|
195
|
+
latest[row["operation_id"]] = dict(latest.get(row["operation_id"]) or {}, **row)
|
|
196
|
+
rows = list(latest.values())
|
|
197
|
+
if status:
|
|
198
|
+
rows = [r for r in rows if r.get("status") == status]
|
|
199
|
+
rows.sort(key=lambda r: float(r.get("updated_at") or r.get("submitted_at") or 0), reverse=True)
|
|
200
|
+
return rows[:max(1, min(1000, int(limit or 100)))]
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def recover_pending(*, now=None, ttl_seconds=PENDING_TTL_SECONDS):
|
|
204
|
+
"""Turn stale pending writes into unknown so restart never replays blindly."""
|
|
205
|
+
now = time.time() if now is None else float(now)
|
|
206
|
+
changed = 0
|
|
207
|
+
for row in recent(status="pending", limit=1000):
|
|
208
|
+
submitted = float(row.get("submitted_at") or 0)
|
|
209
|
+
if submitted and now - submitted > max(1, float(ttl_seconds)):
|
|
210
|
+
if mark_unknown(row["operation_id"], "进程重启后未收到远端回执,请先对账"):
|
|
211
|
+
changed += 1
|
|
212
|
+
return changed
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def status_for_exception(error) -> str:
|
|
216
|
+
text = str(error or "").lower()
|
|
217
|
+
if isinstance(error, (TimeoutError, ConnectionError, urllib.error.URLError)) or any(x in text for x in
|
|
218
|
+
("timeout", "timed out", "超时", "connection", "connect", "connection reset",
|
|
219
|
+
"断开", "断连", "连不上", "网络", "network", "unreachable", "disconnect")):
|
|
220
|
+
return "unknown"
|
|
221
|
+
return "failed"
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def finish_exception(operation_id, error):
|
|
225
|
+
status = status_for_exception(error)
|
|
226
|
+
if status == "unknown":
|
|
227
|
+
return mark_unknown(operation_id, str(error))
|
|
228
|
+
return fail(operation_id, str(error))
|