codebee 0.1.101 → 0.1.103
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/LICENSE +21 -21
- package/README.md +3 -6
- package/app/.playwright-cli/console-2026-09-23T08-26-03-635Z.log +88 -0
- package/app/.playwright-cli/page-2026-09-23T08-26-03-845Z.yml +126 -0
- package/app/.playwright-cli/page-2026-09-23T08-29-44-203Z.png +0 -0
- package/app/.playwright-cli/page-2026-09-23T08-35-42-046Z.yml +210 -0
- package/app/.playwright-cli/page-2026-09-23T08-36-52-858Z.png +0 -0
- package/app/.playwright-cli/page-2026-09-23T08-37-33-938Z.png +0 -0
- package/app/.playwright-cli/run-detail-after.png +0 -0
- package/app/core/acceptance.py +100 -100
- package/app/core/acp_client.py +457 -426
- package/app/core/aiflavor.py +197 -197
- package/app/core/analytics.py +38 -38
- package/app/core/artifacts.py +148 -148
- package/app/core/assets.py +110 -110
- package/app/core/attachments.py +623 -623
- package/app/core/audit.py +30 -30
- package/app/core/backend_profiles.py +188 -188
- package/app/core/backup.py +470 -470
- package/app/core/beekeeper.py +183 -183
- package/app/core/benchstore.py +102 -102
- package/app/core/board.py +21 -17
- package/app/core/bookmeta.py +1036 -1036
- package/app/core/bookmeta_catalog.py +180 -180
- package/app/core/branching.py +137 -137
- package/app/core/builtin_agent.py +13 -4
- package/app/core/capability.py +144 -144
- package/app/core/catalog.py +28 -6
- package/app/core/chaptersafe.py +32 -32
- package/app/core/checkpoints.py +117 -31
- package/app/core/cleanup.py +303 -303
- package/app/core/command_auth.py +302 -302
- package/app/core/compaction.py +201 -201
- package/app/core/contracts.py +255 -255
- package/app/core/covergen.py +356 -356
- package/app/core/defectretro.py +43 -43
- package/app/core/diagnostics.py +114 -114
- package/app/core/dispatch.py +1 -1
- package/app/core/dispatch_log.py +128 -128
- package/app/core/env_scrub.py +85 -85
- package/app/core/errorlog.py +137 -137
- package/app/core/eval_matrix.py +91 -82
- package/app/core/evaluation.py +174 -174
- package/app/core/failure_notify.py +290 -290
- package/app/core/flow_graph.py +33 -33
- package/app/core/goal_service.py +167 -167
- package/app/core/health.py +389 -389
- package/app/core/history.py +38 -38
- package/app/core/hooks.py +162 -162
- package/app/core/knowledge_pipeline.py +86 -86
- package/app/core/manager.py +2385 -2385
- package/app/core/mcp_client.py +7 -2
- package/app/core/mocks.py +64 -64
- package/app/core/notify.py +201 -201
- package/app/core/novel_quality.py +181 -181
- package/app/core/openapi.py +217 -209
- package/app/core/operations.py +228 -228
- package/app/core/paihang.py +145 -145
- package/app/core/pipeline.py +6 -3
- package/app/core/policy.py +9 -0
- package/app/core/portguard.py +111 -111
- package/app/core/presence.py +77 -77
- package/app/core/preview.py +293 -293
- package/app/core/project_memory.py +206 -206
- package/app/core/prompt_guard.py +78 -78
- package/app/core/publish/__init__.py +7 -7
- package/app/core/publish/auto.py +659 -562
- package/app/core/publish/chaprows.py +138 -138
- package/app/core/publish/fanqie.py +16 -10
- package/app/core/publish/flow.py +29 -1
- package/app/core/publish/flows-fanqie-calibrated.json +442 -442
- package/app/core/publish/manager.py +1185 -1177
- package/app/core/publish/qimao.py +193 -193
- package/app/core/publish/volumes.py +108 -108
- package/app/core/publish/ws.py +139 -139
- package/app/core/quality_gate.py +201 -201
- package/app/core/redact.py +38 -38
- package/app/core/repeat_guard.py +129 -129
- package/app/core/retrieval.py +137 -137
- package/app/core/sandbox_runtime.py +95 -0
- package/app/core/selfupdate.py +384 -384
- package/app/core/session_log.py +161 -161
- package/app/core/sessions.py +312 -312
- package/app/core/settings.py +215 -215
- package/app/core/settings_schema.py +282 -282
- package/app/core/share_page.py +114 -114
- package/app/core/skill_scan.py +139 -139
- package/app/core/skillpacks/market/character-bible.md +66 -66
- package/app/core/skillpacks/market/code-risk-checklist.md +58 -58
- package/app/core/skillpacks/market/git-workflow.md +57 -57
- package/app/core/skillpacks/market/release-notes.md +72 -72
- package/app/core/skillpacks/market/weekly-report.md +71 -71
- package/app/core/skillpacks/market/worldview-consistency.md +70 -70
- package/app/core/step_runner.py +132 -132
- package/app/core/store.py +13 -3
- package/app/core/story_tracking.py +275 -275
- package/app/core/task_compile.py +199 -199
- package/app/core/telemetry.py +292 -292
- package/app/core/tlsctx.py +64 -64
- package/app/core/token_meter.py +147 -147
- package/app/core/tool_outputs.py +107 -107
- package/app/core/tracing.py +207 -207
- package/app/core/upstream.py +41 -41
- package/app/core/usage.py +1038 -1038
- package/app/core/volumes.py +389 -389
- package/app/core/webhooks.py +89 -89
- package/app/core/wxdigest.py +865 -865
- package/app/core/zentao.py +2692 -2692
- package/app/main.py +1 -0
- package/app/mcp_server.py +244 -244
- package/app/output/_createbook_body.json +1 -0
- package/app/output/_probe_fq_click.py +50 -0
- package/app/output/_probe_fq_form.py +42 -0
- package/app/output/_probe_fq_modal.py +55 -0
- package/app/output/_probe_fq_points.py +71 -0
- package/app/output/_probe_fq_submit.py +34 -0
- package/app/output/_probe_fq_watch.py +62 -0
- package/app/output/workbench-clean-preview.jpg +0 -0
- package/app/output/workbench-reference-render.png +0 -0
- package/app/output/workbench-shell.png +0 -0
- package/app/pick_dialog.py +110 -110
- package/app/ui/app.js +165 -29
- package/app/ui/artdiff.js +74 -74
- package/app/ui/hive3d.js +491 -223
- package/app/ui/i18n.js +18 -1
- package/app/ui/icons/bee.svg +79 -79
- package/app/ui/index.html +24 -4
- package/app/ui/manifest.json +16 -16
- package/app/ui/qrcode.js +2297 -2297
- package/app/ui/style.css +548 -18
- package/bin/tutti.js +147 -147
- package/package.json +1 -1
- package/tools/wx_reader.py +261 -261
package/app/core/repeat_guard.py
CHANGED
|
@@ -1,129 +1,129 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
"""重复 CLI 调用检测:防止编排者用同一份 prompt 反复调同一 CLI 死循环。
|
|
3
|
-
|
|
4
|
-
设计稿:docs/migration/01-defense-patterns.md §5C。
|
|
5
|
-
参考 dsh packages/guard/repeat-tool-reminder/src/index.ts:
|
|
6
|
-
监听 tools/post-execute,按 (tool, canonical_args) 跟踪,3/5/8 阈值注入 user-message 提醒。
|
|
7
|
-
|
|
8
|
-
Tutti 的 "tool" 对应到 step role + 执行者身份,"canonical_args" 对应到 prompt
|
|
9
|
-
模板指纹。不同 CLI 间的换将是有效的回退,不能共享同一个重复预算。步骤级调用
|
|
10
|
-
代价远高于普通工具调用,因此第 3 次提醒、第 5 次强制停止。
|
|
11
|
-
"""
|
|
12
|
-
from __future__ import annotations
|
|
13
|
-
|
|
14
|
-
import hashlib
|
|
15
|
-
import logging
|
|
16
|
-
import threading
|
|
17
|
-
import time
|
|
18
|
-
|
|
19
|
-
log = logging.getLogger(__name__)
|
|
20
|
-
|
|
21
|
-
# 渐进阈值:第 N 次命中时给提醒;最后一次及以上时强制停止。
|
|
22
|
-
DEFAULT_THRESHOLDS = (3, 5)
|
|
23
|
-
|
|
24
|
-
REMINDER_TEMPLATE = (
|
|
25
|
-
"⚠️ 提醒:上一步使用了几乎相同的输入(已连续 {count} 次调用 {role})。"
|
|
26
|
-
"请换一种方法:读不同的文件、用更具体的指令、或明确询问用户澄清。"
|
|
27
|
-
)
|
|
28
|
-
|
|
29
|
-
STOP_TEMPLATE = (
|
|
30
|
-
"已连续 {count} 次使用相同输入调用 {role},超过 {last_thr} 次阈值,"
|
|
31
|
-
"强制停止本 step 以防止无限循环。请用户检查输入或调整 prompt 模板。"
|
|
32
|
-
)
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
def _prompt_fingerprint(prompt: str, identity: str = "") -> str:
|
|
36
|
-
"""prompt 指纹:用前 200 字符 + 总长度(CLI prompt 头几行通常是模板部分)。
|
|
37
|
-
|
|
38
|
-
不做严格 canonical args 比对(CLI prompt 通常很长,且 dsh 也用 JSON.stringify);
|
|
39
|
-
指纹命中说明是同一个模板路径走过多次,已经够分辨。
|
|
40
|
-
|
|
41
|
-
用 SHA-256 取前 12 字符(96 bit):48 亿分之一碰撞概率对去重足够;不追求密码学强度。
|
|
42
|
-
"""
|
|
43
|
-
head = (prompt or "")[:200]
|
|
44
|
-
return hashlib.sha256(
|
|
45
|
-
f"{identity}\0{head}|{len(prompt or '')}".encode("utf-8")).hexdigest()[:12]
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
class RepeatGuard:
|
|
49
|
-
"""每 (run_id, step_role) 的最近执行链独立计数;指纹包含执行者身份。
|
|
50
|
-
|
|
51
|
-
计数仅统计相同 key 连续出现;新 prompt 会重置。
|
|
52
|
-
"""
|
|
53
|
-
|
|
54
|
-
def __init__(self, thresholds=DEFAULT_THRESHOLDS, *, max_history=50):
|
|
55
|
-
self.thresholds = tuple(sorted(set(thresholds)))
|
|
56
|
-
self.max_history = max_history
|
|
57
|
-
self._lock = threading.Lock()
|
|
58
|
-
# chain[(run_id, role)] = [(ts, fingerprint)]
|
|
59
|
-
self._chains: dict = {}
|
|
60
|
-
|
|
61
|
-
def check(self, run_id: str, role: str, prompt: str, *, identity: str = "") -> dict:
|
|
62
|
-
"""检查并更新 (run_id, role) 的连续计数。
|
|
63
|
-
|
|
64
|
-
identity 标识实际执行 CLI/agent。相同 prompt 换到其他执行者属于回退,
|
|
65
|
-
会从新的连续链计数;同一执行者重复相同调用仍按阈值停止。
|
|
66
|
-
|
|
67
|
-
Returns:
|
|
68
|
-
{"count": int, "reminder": str | None, "should_stop": bool}
|
|
69
|
-
"""
|
|
70
|
-
fp = _prompt_fingerprint(prompt, str(identity or ""))
|
|
71
|
-
chain_key = (run_id, role)
|
|
72
|
-
with self._lock:
|
|
73
|
-
chain = self._chains.setdefault(chain_key, [])
|
|
74
|
-
chain.append((time.time(), fp))
|
|
75
|
-
# 仅统计相同 fp 连续出现
|
|
76
|
-
count = 1
|
|
77
|
-
for _, prev_fp in reversed(chain[:-1]):
|
|
78
|
-
if prev_fp == fp:
|
|
79
|
-
count += 1
|
|
80
|
-
else:
|
|
81
|
-
break
|
|
82
|
-
# 截尾避免内存膨胀
|
|
83
|
-
if len(chain) > self.max_history:
|
|
84
|
-
self._chains[chain_key] = chain[-self.max_history:]
|
|
85
|
-
|
|
86
|
-
reminder = None
|
|
87
|
-
stop = False
|
|
88
|
-
last_thr = self.thresholds[-1]
|
|
89
|
-
if count >= last_thr:
|
|
90
|
-
stop = True
|
|
91
|
-
reminder = STOP_TEMPLATE.format(count=count, role=role, last_thr=last_thr)
|
|
92
|
-
elif count in self.thresholds:
|
|
93
|
-
reminder = REMINDER_TEMPLATE.format(count=count, role=role)
|
|
94
|
-
|
|
95
|
-
if reminder:
|
|
96
|
-
log.warning("repeat-guard %s/%s count=%d stop=%s reminder=%s",
|
|
97
|
-
run_id, role, count, stop, reminder[:80])
|
|
98
|
-
return {"count": count, "reminder": reminder, "should_stop": stop}
|
|
99
|
-
|
|
100
|
-
def reset(self, run_id: str) -> int:
|
|
101
|
-
"""重置某 run_id 的所有 (run_id, *) 链。返回清除的链条数。"""
|
|
102
|
-
with self._lock:
|
|
103
|
-
cleared = 0
|
|
104
|
-
for k in list(self._chains.keys()):
|
|
105
|
-
if k[0] == run_id:
|
|
106
|
-
del self._chains[k]
|
|
107
|
-
cleared += 1
|
|
108
|
-
return cleared
|
|
109
|
-
|
|
110
|
-
def stats(self, run_id: str, role: str) -> dict:
|
|
111
|
-
"""读取 (run_id, role) 当前状态(不更新)。用于 UI 调试。"""
|
|
112
|
-
chain_key = (run_id, role)
|
|
113
|
-
with self._lock:
|
|
114
|
-
chain = self._chains.get(chain_key, [])
|
|
115
|
-
if not chain:
|
|
116
|
-
return {"chain_len": 0, "last_fp": ""}
|
|
117
|
-
# 当前连续计数
|
|
118
|
-
last_fp = chain[-1][1]
|
|
119
|
-
count = 1
|
|
120
|
-
for _, fp in reversed(chain[:-1]):
|
|
121
|
-
if fp == last_fp:
|
|
122
|
-
count += 1
|
|
123
|
-
else:
|
|
124
|
-
break
|
|
125
|
-
return {"chain_len": len(chain), "last_fp": last_fp, "count": count}
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
# 模块级默认实例(pipeline 接线用;run_id 入键天然隔离多任务,测试无需替换)
|
|
129
|
-
guard = RepeatGuard()
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""重复 CLI 调用检测:防止编排者用同一份 prompt 反复调同一 CLI 死循环。
|
|
3
|
+
|
|
4
|
+
设计稿:docs/migration/01-defense-patterns.md §5C。
|
|
5
|
+
参考 dsh packages/guard/repeat-tool-reminder/src/index.ts:
|
|
6
|
+
监听 tools/post-execute,按 (tool, canonical_args) 跟踪,3/5/8 阈值注入 user-message 提醒。
|
|
7
|
+
|
|
8
|
+
Tutti 的 "tool" 对应到 step role + 执行者身份,"canonical_args" 对应到 prompt
|
|
9
|
+
模板指纹。不同 CLI 间的换将是有效的回退,不能共享同一个重复预算。步骤级调用
|
|
10
|
+
代价远高于普通工具调用,因此第 3 次提醒、第 5 次强制停止。
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import hashlib
|
|
15
|
+
import logging
|
|
16
|
+
import threading
|
|
17
|
+
import time
|
|
18
|
+
|
|
19
|
+
log = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
# 渐进阈值:第 N 次命中时给提醒;最后一次及以上时强制停止。
|
|
22
|
+
DEFAULT_THRESHOLDS = (3, 5)
|
|
23
|
+
|
|
24
|
+
REMINDER_TEMPLATE = (
|
|
25
|
+
"⚠️ 提醒:上一步使用了几乎相同的输入(已连续 {count} 次调用 {role})。"
|
|
26
|
+
"请换一种方法:读不同的文件、用更具体的指令、或明确询问用户澄清。"
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
STOP_TEMPLATE = (
|
|
30
|
+
"已连续 {count} 次使用相同输入调用 {role},超过 {last_thr} 次阈值,"
|
|
31
|
+
"强制停止本 step 以防止无限循环。请用户检查输入或调整 prompt 模板。"
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _prompt_fingerprint(prompt: str, identity: str = "") -> str:
|
|
36
|
+
"""prompt 指纹:用前 200 字符 + 总长度(CLI prompt 头几行通常是模板部分)。
|
|
37
|
+
|
|
38
|
+
不做严格 canonical args 比对(CLI prompt 通常很长,且 dsh 也用 JSON.stringify);
|
|
39
|
+
指纹命中说明是同一个模板路径走过多次,已经够分辨。
|
|
40
|
+
|
|
41
|
+
用 SHA-256 取前 12 字符(96 bit):48 亿分之一碰撞概率对去重足够;不追求密码学强度。
|
|
42
|
+
"""
|
|
43
|
+
head = (prompt or "")[:200]
|
|
44
|
+
return hashlib.sha256(
|
|
45
|
+
f"{identity}\0{head}|{len(prompt or '')}".encode("utf-8")).hexdigest()[:12]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class RepeatGuard:
|
|
49
|
+
"""每 (run_id, step_role) 的最近执行链独立计数;指纹包含执行者身份。
|
|
50
|
+
|
|
51
|
+
计数仅统计相同 key 连续出现;新 prompt 会重置。
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
def __init__(self, thresholds=DEFAULT_THRESHOLDS, *, max_history=50):
|
|
55
|
+
self.thresholds = tuple(sorted(set(thresholds)))
|
|
56
|
+
self.max_history = max_history
|
|
57
|
+
self._lock = threading.Lock()
|
|
58
|
+
# chain[(run_id, role)] = [(ts, fingerprint)]
|
|
59
|
+
self._chains: dict = {}
|
|
60
|
+
|
|
61
|
+
def check(self, run_id: str, role: str, prompt: str, *, identity: str = "") -> dict:
|
|
62
|
+
"""检查并更新 (run_id, role) 的连续计数。
|
|
63
|
+
|
|
64
|
+
identity 标识实际执行 CLI/agent。相同 prompt 换到其他执行者属于回退,
|
|
65
|
+
会从新的连续链计数;同一执行者重复相同调用仍按阈值停止。
|
|
66
|
+
|
|
67
|
+
Returns:
|
|
68
|
+
{"count": int, "reminder": str | None, "should_stop": bool}
|
|
69
|
+
"""
|
|
70
|
+
fp = _prompt_fingerprint(prompt, str(identity or ""))
|
|
71
|
+
chain_key = (run_id, role)
|
|
72
|
+
with self._lock:
|
|
73
|
+
chain = self._chains.setdefault(chain_key, [])
|
|
74
|
+
chain.append((time.time(), fp))
|
|
75
|
+
# 仅统计相同 fp 连续出现
|
|
76
|
+
count = 1
|
|
77
|
+
for _, prev_fp in reversed(chain[:-1]):
|
|
78
|
+
if prev_fp == fp:
|
|
79
|
+
count += 1
|
|
80
|
+
else:
|
|
81
|
+
break
|
|
82
|
+
# 截尾避免内存膨胀
|
|
83
|
+
if len(chain) > self.max_history:
|
|
84
|
+
self._chains[chain_key] = chain[-self.max_history:]
|
|
85
|
+
|
|
86
|
+
reminder = None
|
|
87
|
+
stop = False
|
|
88
|
+
last_thr = self.thresholds[-1]
|
|
89
|
+
if count >= last_thr:
|
|
90
|
+
stop = True
|
|
91
|
+
reminder = STOP_TEMPLATE.format(count=count, role=role, last_thr=last_thr)
|
|
92
|
+
elif count in self.thresholds:
|
|
93
|
+
reminder = REMINDER_TEMPLATE.format(count=count, role=role)
|
|
94
|
+
|
|
95
|
+
if reminder:
|
|
96
|
+
log.warning("repeat-guard %s/%s count=%d stop=%s reminder=%s",
|
|
97
|
+
run_id, role, count, stop, reminder[:80])
|
|
98
|
+
return {"count": count, "reminder": reminder, "should_stop": stop}
|
|
99
|
+
|
|
100
|
+
def reset(self, run_id: str) -> int:
|
|
101
|
+
"""重置某 run_id 的所有 (run_id, *) 链。返回清除的链条数。"""
|
|
102
|
+
with self._lock:
|
|
103
|
+
cleared = 0
|
|
104
|
+
for k in list(self._chains.keys()):
|
|
105
|
+
if k[0] == run_id:
|
|
106
|
+
del self._chains[k]
|
|
107
|
+
cleared += 1
|
|
108
|
+
return cleared
|
|
109
|
+
|
|
110
|
+
def stats(self, run_id: str, role: str) -> dict:
|
|
111
|
+
"""读取 (run_id, role) 当前状态(不更新)。用于 UI 调试。"""
|
|
112
|
+
chain_key = (run_id, role)
|
|
113
|
+
with self._lock:
|
|
114
|
+
chain = self._chains.get(chain_key, [])
|
|
115
|
+
if not chain:
|
|
116
|
+
return {"chain_len": 0, "last_fp": ""}
|
|
117
|
+
# 当前连续计数
|
|
118
|
+
last_fp = chain[-1][1]
|
|
119
|
+
count = 1
|
|
120
|
+
for _, fp in reversed(chain[:-1]):
|
|
121
|
+
if fp == last_fp:
|
|
122
|
+
count += 1
|
|
123
|
+
else:
|
|
124
|
+
break
|
|
125
|
+
return {"chain_len": len(chain), "last_fp": last_fp, "count": count}
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
# 模块级默认实例(pipeline 接线用;run_id 入键天然隔离多任务,测试无需替换)
|
|
129
|
+
guard = RepeatGuard()
|
package/app/core/retrieval.py
CHANGED
|
@@ -1,137 +1,137 @@
|
|
|
1
|
-
"""Dependency-free retrieval index with traceable keyword scoring.
|
|
2
|
-
|
|
3
|
-
This is intentionally a RAG-lite layer: SQLite/vector backends can be added
|
|
4
|
-
later without changing the API contract. Every hit carries its source and the
|
|
5
|
-
terms that caused the match.
|
|
6
|
-
"""
|
|
7
|
-
from __future__ import annotations
|
|
8
|
-
|
|
9
|
-
import hashlib
|
|
10
|
-
import json
|
|
11
|
-
import re
|
|
12
|
-
import threading
|
|
13
|
-
from pathlib import Path
|
|
14
|
-
|
|
15
|
-
from . import paths
|
|
16
|
-
|
|
17
|
-
_LOCK = threading.RLock()
|
|
18
|
-
_FILE = paths.DATA_DIR / "retrieval_index.json"
|
|
19
|
-
_WORD_RE = re.compile(r"[A-Za-z0-9_]+|[\u4e00-\u9fff]")
|
|
20
|
-
_MAX_FILES = 500
|
|
21
|
-
_MAX_BYTES = 20 * 1024 * 1024
|
|
22
|
-
_SKIP_DIRS = {".git", "node_modules", "__pycache__", ".venv", "venv", ".idea", ".vscode"}
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
def _terms(text):
|
|
26
|
-
return [x.lower() for x in _WORD_RE.findall(str(text or ""))]
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
def _load():
|
|
30
|
-
try:
|
|
31
|
-
value = json.loads(_FILE.read_text(encoding="utf-8"))
|
|
32
|
-
return value if isinstance(value, dict) else {"documents": {}}
|
|
33
|
-
except Exception:
|
|
34
|
-
return {"version": 1, "documents": {}}
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
def _save(value):
|
|
38
|
-
_FILE.parent.mkdir(parents=True, exist_ok=True)
|
|
39
|
-
tmp = _FILE.with_suffix(".tmp")
|
|
40
|
-
tmp.write_text(json.dumps(value, ensure_ascii=False, indent=1), encoding="utf-8")
|
|
41
|
-
tmp.replace(_FILE)
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def upsert(source, text, metadata=None, doc_id=""):
|
|
45
|
-
source = str(source or "").strip()[:500]
|
|
46
|
-
text = str(text or "").strip()
|
|
47
|
-
if not source or not text:
|
|
48
|
-
raise ValueError("source 和 text 不能为空")
|
|
49
|
-
root_hint = ""
|
|
50
|
-
if isinstance(metadata, dict):
|
|
51
|
-
root_hint = str(metadata.get("root") or "")[:500]
|
|
52
|
-
did = str(doc_id or hashlib.sha256((root_hint + "\0" + source).encode("utf-8")).hexdigest()[:20])[:100]
|
|
53
|
-
meta = metadata if isinstance(metadata, dict) else {}
|
|
54
|
-
meta = {str(k)[:80]: str(v)[:1000] for k, v in list(meta.items())[:30]}
|
|
55
|
-
row = {"id": did, "source": source, "text": text[:200000],
|
|
56
|
-
"terms": sorted(set(_terms(text))),
|
|
57
|
-
"metadata": meta}
|
|
58
|
-
with _LOCK:
|
|
59
|
-
data = _load()
|
|
60
|
-
data.setdefault("version", 1)
|
|
61
|
-
documents = data.setdefault("documents", {})
|
|
62
|
-
# Explicit IDs are client supplied; avoid silently replacing an
|
|
63
|
-
# unrelated document when two sources reuse the same short ID.
|
|
64
|
-
if did in documents and documents[did].get("source") != source:
|
|
65
|
-
suffix = hashlib.sha256((source + "\0" + text).encode("utf-8")).hexdigest()[:12]
|
|
66
|
-
did = (did[:87] + "-" + suffix)[:100]
|
|
67
|
-
row["id"] = did
|
|
68
|
-
documents[did] = row
|
|
69
|
-
_save(data)
|
|
70
|
-
return row
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
def index_directory(root, extensions=(".md", ".txt", ".json"), limit=500,
|
|
74
|
-
allowed_roots=None, max_bytes=_MAX_BYTES):
|
|
75
|
-
root = Path(root).expanduser().resolve()
|
|
76
|
-
if not root.is_dir():
|
|
77
|
-
raise ValueError("目录不存在")
|
|
78
|
-
if allowed_roots is not None:
|
|
79
|
-
roots = [Path(p).expanduser().resolve() for p in allowed_roots if p]
|
|
80
|
-
if not any(root == p or p in root.parents for p in roots):
|
|
81
|
-
raise ValueError("只能索引任务工作目录范围内的文件")
|
|
82
|
-
try:
|
|
83
|
-
file_limit = max(1, min(_MAX_FILES, int(limit)))
|
|
84
|
-
except (TypeError, ValueError):
|
|
85
|
-
raise ValueError("limit 必须是整数")
|
|
86
|
-
try:
|
|
87
|
-
byte_limit = max(1, min(_MAX_BYTES, int(max_bytes)))
|
|
88
|
-
except (TypeError, ValueError):
|
|
89
|
-
raise ValueError("max_bytes 必须是整数")
|
|
90
|
-
count = 0
|
|
91
|
-
total_bytes = 0
|
|
92
|
-
for path in root.rglob("*"):
|
|
93
|
-
if count >= file_limit or total_bytes >= byte_limit:
|
|
94
|
-
break
|
|
95
|
-
# Never follow a symlink while indexing user-controlled trees.
|
|
96
|
-
if path.is_symlink() or any(part in _SKIP_DIRS for part in path.relative_to(root).parts):
|
|
97
|
-
continue
|
|
98
|
-
if path.is_file() and path.suffix.lower() in extensions:
|
|
99
|
-
try:
|
|
100
|
-
resolved = path.resolve()
|
|
101
|
-
if resolved != root and root not in resolved.parents:
|
|
102
|
-
continue
|
|
103
|
-
size = path.stat().st_size
|
|
104
|
-
if size > byte_limit - total_bytes:
|
|
105
|
-
continue
|
|
106
|
-
upsert(str(path.relative_to(root)), path.read_text(encoding="utf-8", errors="replace"),
|
|
107
|
-
{"root": str(root)})
|
|
108
|
-
count += 1
|
|
109
|
-
total_bytes += size
|
|
110
|
-
except OSError:
|
|
111
|
-
pass
|
|
112
|
-
return {"indexed": count, "bytes": total_bytes,
|
|
113
|
-
"total": len(_load().get("documents") or {})}
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
def search(query, limit=10):
|
|
117
|
-
try:
|
|
118
|
-
n = max(1, min(50, int(limit or 10)))
|
|
119
|
-
except (TypeError, ValueError):
|
|
120
|
-
raise ValueError("limit 必须是整数")
|
|
121
|
-
terms = set(_terms(str(query or "")[:500]))
|
|
122
|
-
if not terms:
|
|
123
|
-
return []
|
|
124
|
-
with _LOCK:
|
|
125
|
-
docs = list((_load().get("documents") or {}).values())
|
|
126
|
-
rows = []
|
|
127
|
-
for doc in docs:
|
|
128
|
-
matched = sorted(terms.intersection(doc.get("terms") or []))
|
|
129
|
-
if not matched:
|
|
130
|
-
continue
|
|
131
|
-
score = len(matched) / max(1, len(terms))
|
|
132
|
-
rows.append({"id": doc.get("id"), "source": doc.get("source"),
|
|
133
|
-
"score": round(score, 4), "matched_terms": matched,
|
|
134
|
-
"snippet": str(doc.get("text") or "")[:600],
|
|
135
|
-
"metadata": doc.get("metadata") or {}})
|
|
136
|
-
rows.sort(key=lambda x: (-x["score"], x["source"] or ""))
|
|
137
|
-
return rows[:n]
|
|
1
|
+
"""Dependency-free retrieval index with traceable keyword scoring.
|
|
2
|
+
|
|
3
|
+
This is intentionally a RAG-lite layer: SQLite/vector backends can be added
|
|
4
|
+
later without changing the API contract. Every hit carries its source and the
|
|
5
|
+
terms that caused the match.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
import re
|
|
12
|
+
import threading
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from . import paths
|
|
16
|
+
|
|
17
|
+
_LOCK = threading.RLock()
|
|
18
|
+
_FILE = paths.DATA_DIR / "retrieval_index.json"
|
|
19
|
+
_WORD_RE = re.compile(r"[A-Za-z0-9_]+|[\u4e00-\u9fff]")
|
|
20
|
+
_MAX_FILES = 500
|
|
21
|
+
_MAX_BYTES = 20 * 1024 * 1024
|
|
22
|
+
_SKIP_DIRS = {".git", "node_modules", "__pycache__", ".venv", "venv", ".idea", ".vscode"}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _terms(text):
|
|
26
|
+
return [x.lower() for x in _WORD_RE.findall(str(text or ""))]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _load():
|
|
30
|
+
try:
|
|
31
|
+
value = json.loads(_FILE.read_text(encoding="utf-8"))
|
|
32
|
+
return value if isinstance(value, dict) else {"documents": {}}
|
|
33
|
+
except Exception:
|
|
34
|
+
return {"version": 1, "documents": {}}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _save(value):
|
|
38
|
+
_FILE.parent.mkdir(parents=True, exist_ok=True)
|
|
39
|
+
tmp = _FILE.with_suffix(".tmp")
|
|
40
|
+
tmp.write_text(json.dumps(value, ensure_ascii=False, indent=1), encoding="utf-8")
|
|
41
|
+
tmp.replace(_FILE)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def upsert(source, text, metadata=None, doc_id=""):
|
|
45
|
+
source = str(source or "").strip()[:500]
|
|
46
|
+
text = str(text or "").strip()
|
|
47
|
+
if not source or not text:
|
|
48
|
+
raise ValueError("source 和 text 不能为空")
|
|
49
|
+
root_hint = ""
|
|
50
|
+
if isinstance(metadata, dict):
|
|
51
|
+
root_hint = str(metadata.get("root") or "")[:500]
|
|
52
|
+
did = str(doc_id or hashlib.sha256((root_hint + "\0" + source).encode("utf-8")).hexdigest()[:20])[:100]
|
|
53
|
+
meta = metadata if isinstance(metadata, dict) else {}
|
|
54
|
+
meta = {str(k)[:80]: str(v)[:1000] for k, v in list(meta.items())[:30]}
|
|
55
|
+
row = {"id": did, "source": source, "text": text[:200000],
|
|
56
|
+
"terms": sorted(set(_terms(text))),
|
|
57
|
+
"metadata": meta}
|
|
58
|
+
with _LOCK:
|
|
59
|
+
data = _load()
|
|
60
|
+
data.setdefault("version", 1)
|
|
61
|
+
documents = data.setdefault("documents", {})
|
|
62
|
+
# Explicit IDs are client supplied; avoid silently replacing an
|
|
63
|
+
# unrelated document when two sources reuse the same short ID.
|
|
64
|
+
if did in documents and documents[did].get("source") != source:
|
|
65
|
+
suffix = hashlib.sha256((source + "\0" + text).encode("utf-8")).hexdigest()[:12]
|
|
66
|
+
did = (did[:87] + "-" + suffix)[:100]
|
|
67
|
+
row["id"] = did
|
|
68
|
+
documents[did] = row
|
|
69
|
+
_save(data)
|
|
70
|
+
return row
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def index_directory(root, extensions=(".md", ".txt", ".json"), limit=500,
|
|
74
|
+
allowed_roots=None, max_bytes=_MAX_BYTES):
|
|
75
|
+
root = Path(root).expanduser().resolve()
|
|
76
|
+
if not root.is_dir():
|
|
77
|
+
raise ValueError("目录不存在")
|
|
78
|
+
if allowed_roots is not None:
|
|
79
|
+
roots = [Path(p).expanduser().resolve() for p in allowed_roots if p]
|
|
80
|
+
if not any(root == p or p in root.parents for p in roots):
|
|
81
|
+
raise ValueError("只能索引任务工作目录范围内的文件")
|
|
82
|
+
try:
|
|
83
|
+
file_limit = max(1, min(_MAX_FILES, int(limit)))
|
|
84
|
+
except (TypeError, ValueError):
|
|
85
|
+
raise ValueError("limit 必须是整数")
|
|
86
|
+
try:
|
|
87
|
+
byte_limit = max(1, min(_MAX_BYTES, int(max_bytes)))
|
|
88
|
+
except (TypeError, ValueError):
|
|
89
|
+
raise ValueError("max_bytes 必须是整数")
|
|
90
|
+
count = 0
|
|
91
|
+
total_bytes = 0
|
|
92
|
+
for path in root.rglob("*"):
|
|
93
|
+
if count >= file_limit or total_bytes >= byte_limit:
|
|
94
|
+
break
|
|
95
|
+
# Never follow a symlink while indexing user-controlled trees.
|
|
96
|
+
if path.is_symlink() or any(part in _SKIP_DIRS for part in path.relative_to(root).parts):
|
|
97
|
+
continue
|
|
98
|
+
if path.is_file() and path.suffix.lower() in extensions:
|
|
99
|
+
try:
|
|
100
|
+
resolved = path.resolve()
|
|
101
|
+
if resolved != root and root not in resolved.parents:
|
|
102
|
+
continue
|
|
103
|
+
size = path.stat().st_size
|
|
104
|
+
if size > byte_limit - total_bytes:
|
|
105
|
+
continue
|
|
106
|
+
upsert(str(path.relative_to(root)), path.read_text(encoding="utf-8", errors="replace"),
|
|
107
|
+
{"root": str(root)})
|
|
108
|
+
count += 1
|
|
109
|
+
total_bytes += size
|
|
110
|
+
except OSError:
|
|
111
|
+
pass
|
|
112
|
+
return {"indexed": count, "bytes": total_bytes,
|
|
113
|
+
"total": len(_load().get("documents") or {})}
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def search(query, limit=10):
|
|
117
|
+
try:
|
|
118
|
+
n = max(1, min(50, int(limit or 10)))
|
|
119
|
+
except (TypeError, ValueError):
|
|
120
|
+
raise ValueError("limit 必须是整数")
|
|
121
|
+
terms = set(_terms(str(query or "")[:500]))
|
|
122
|
+
if not terms:
|
|
123
|
+
return []
|
|
124
|
+
with _LOCK:
|
|
125
|
+
docs = list((_load().get("documents") or {}).values())
|
|
126
|
+
rows = []
|
|
127
|
+
for doc in docs:
|
|
128
|
+
matched = sorted(terms.intersection(doc.get("terms") or []))
|
|
129
|
+
if not matched:
|
|
130
|
+
continue
|
|
131
|
+
score = len(matched) / max(1, len(terms))
|
|
132
|
+
rows.append({"id": doc.get("id"), "source": doc.get("source"),
|
|
133
|
+
"score": round(score, 4), "matched_terms": matched,
|
|
134
|
+
"snippet": str(doc.get("text") or "")[:600],
|
|
135
|
+
"metadata": doc.get("metadata") or {}})
|
|
136
|
+
rows.sort(key=lambda x: (-x["score"], x["source"] or ""))
|
|
137
|
+
return rows[:n]
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""Optional process sandbox launchers.
|
|
2
|
+
|
|
3
|
+
The default CodeBee install has no container dependency. Docker is an explicit
|
|
4
|
+
opt-in backend selected by a task policy and a configured image; this module
|
|
5
|
+
only builds argv and never silently falls back to a weaker backend.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
import re
|
|
11
|
+
import shutil
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class SandboxUnavailable(RuntimeError):
|
|
16
|
+
pass
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def requested_backend(sandbox):
|
|
20
|
+
value = str((sandbox or {}).get("backend") or "native").strip().lower()
|
|
21
|
+
return value or "native"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def docker_argv(command, *, sandbox, workdir, image=None, extra_args=(),
|
|
25
|
+
container_workdir=None,
|
|
26
|
+
container_command=("/bin/sh", "-lc")):
|
|
27
|
+
"""Build a restricted ``docker run`` argv for a command or stdio server."""
|
|
28
|
+
if requested_backend(sandbox) != "docker":
|
|
29
|
+
raise SandboxUnavailable("Docker 后端未被任务策略启用")
|
|
30
|
+
docker = shutil.which("docker")
|
|
31
|
+
if not docker:
|
|
32
|
+
raise SandboxUnavailable("当前系统没有可用的 Docker CLI")
|
|
33
|
+
root = Path(workdir or ".").expanduser().resolve(strict=True)
|
|
34
|
+
image = str(image or (sandbox or {}).get("docker_image") or "").strip()
|
|
35
|
+
if not image or len(image) > 240 or not re.match(r"^[A-Za-z0-9][A-Za-z0-9._/:@-]*$", image):
|
|
36
|
+
raise SandboxUnavailable("Docker 后端必须配置可信镜像")
|
|
37
|
+
roots = []
|
|
38
|
+
for value in (sandbox or {}).get("allowed_roots") or [str(root)]:
|
|
39
|
+
candidate = Path(value).expanduser().resolve(strict=True)
|
|
40
|
+
# Docker currently mounts exactly one root. Require the task workspace
|
|
41
|
+
# itself; narrower roots need a separately resolved container cwd and
|
|
42
|
+
# must never accidentally broaden into an ancestor mount.
|
|
43
|
+
if candidate != root:
|
|
44
|
+
raise SandboxUnavailable("Docker 目前只支持挂载任务工作目录")
|
|
45
|
+
if candidate not in roots:
|
|
46
|
+
roots.append(candidate)
|
|
47
|
+
if not roots:
|
|
48
|
+
raise SandboxUnavailable("Docker 后端没有允许目录")
|
|
49
|
+
selected_root = next((candidate for candidate in roots
|
|
50
|
+
if candidate == root or candidate in root.parents
|
|
51
|
+
or root in candidate.parents), None)
|
|
52
|
+
if selected_root is None:
|
|
53
|
+
raise SandboxUnavailable("当前工作目录不在 Docker 允许目录内")
|
|
54
|
+
if len(roots) != 1:
|
|
55
|
+
raise SandboxUnavailable("Docker 暂只支持一个明确的允许目录")
|
|
56
|
+
argv = [docker, "run", "--rm", "--init", "--read-only",
|
|
57
|
+
"--security-opt", "no-new-privileges", "--cap-drop=ALL",
|
|
58
|
+
"--ipc=none", "--pids-limit", "256", "--memory", "2g", "--cpus", "2",
|
|
59
|
+
"--tmpfs", "/tmp:rw,noexec,nosuid,size=64m"]
|
|
60
|
+
if hasattr(os, "getuid") and hasattr(os, "getgid"):
|
|
61
|
+
argv += ["--user", "%d:%d" % (os.getuid(), os.getgid())]
|
|
62
|
+
else:
|
|
63
|
+
argv += ["--user", "65534:65534"]
|
|
64
|
+
argv += ["--network", "none" if not bool((sandbox or {}).get("network", True)) else "bridge"]
|
|
65
|
+
for name in (sandbox or {}).get("env_allowlist") or []:
|
|
66
|
+
name = str(name)
|
|
67
|
+
if name and name in os.environ:
|
|
68
|
+
# Value is supplied through the filtered Docker client environment;
|
|
69
|
+
# never place credentials in argv, where process/log inspection sees them.
|
|
70
|
+
argv += ["--env", name]
|
|
71
|
+
for candidate in roots:
|
|
72
|
+
if "," in str(candidate):
|
|
73
|
+
raise SandboxUnavailable("Docker 允许目录包含不支持的逗号")
|
|
74
|
+
argv += ["--mount", "type=bind,src=%s,dst=/workspace" % str(candidate)]
|
|
75
|
+
if container_workdir is None:
|
|
76
|
+
rel_workdir = root.relative_to(selected_root).as_posix()
|
|
77
|
+
container_workdir = "/workspace" + ("/" + rel_workdir if rel_workdir != "." else "")
|
|
78
|
+
if not str(container_workdir).startswith("/") or ".." in Path(container_workdir).parts:
|
|
79
|
+
raise SandboxUnavailable("Docker 容器工作目录无效")
|
|
80
|
+
argv += ["--workdir", str(container_workdir), *list(extra_args), image]
|
|
81
|
+
if isinstance(command, (list, tuple)):
|
|
82
|
+
argv += [str(x) for x in command]
|
|
83
|
+
else:
|
|
84
|
+
argv += list(container_command) + [str(command)]
|
|
85
|
+
return argv
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def docker_env(sandbox):
|
|
89
|
+
"""Docker receives only explicitly allowed host environment values."""
|
|
90
|
+
out = {"PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
|
|
91
|
+
"HOME": "/tmp", "TMPDIR": "/tmp"}
|
|
92
|
+
for name in (sandbox or {}).get("env_allowlist") or []:
|
|
93
|
+
if name in os.environ:
|
|
94
|
+
out[str(name)] = os.environ[str(name)]
|
|
95
|
+
return out
|