@sakanano-moe/agenthelm 6.5.3 → 6.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/agenthelm.js +1 -1
- package/package.json +2 -2
- package/runtime/index.json +80 -0
- package/runtime/plugins/agent-pack/.codex-plugin/plugin.json +19 -0
- package/runtime/plugins/agent-pack/.mcp.json +12 -0
- package/runtime/plugins/agent-pack/hooks/hooks.json +16 -0
- package/runtime/plugins/agent-pack/mcp.json +14 -0
- package/runtime/plugins/agent-pack/plugin.json +20 -0
- package/runtime/plugins/agent-pack/references/capability-config.md +124 -0
- package/runtime/{agenthelm.py → plugins/agent-pack/runtime/agenthelm.py} +42 -1
- package/runtime/plugins/agent-pack/runtime/agenthelm_runtime.py +388 -0
- package/runtime/plugins/agent-pack/skills/agent-pack/SKILL.md +46 -0
- package/runtime/plugins/agents-init/.codex-plugin/plugin.json +22 -0
- package/runtime/plugins/agents-init/content.json +31 -0
- package/runtime/plugins/agents-init/plugin.json +19 -0
- package/runtime/plugins/agents-init/skills/agents-init/AGENT.template.md +376 -0
- package/runtime/plugins/agents-init/skills/agents-init/SKILL.md +119 -0
- package/runtime/plugins/agents-init/skills/agents-init/index.html +357 -0
- package/runtime/plugins/agents-init/skills/agents-init/launcher.sh +93 -0
- package/runtime/plugins/agents-init/skills/agents-init/project.schema.json +171 -0
- package/runtime/plugins/agents-init/skills/agents-init/scripts/init_project.py +1190 -0
- package/runtime/plugins/agents-init/skills/agents-init/scripts/test_init_project.py +678 -0
- package/runtime/plugins/agents-init/skills/agents-init/skill.json +122 -0
- package/runtime/plugins/opinion-manager/.codex-plugin/plugin.json +26 -0
- package/runtime/plugins/opinion-manager/catalog/README.md +15 -0
- package/runtime/plugins/opinion-manager/catalog/rules/.gitkeep +0 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.align-requirements/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.assumptions-authority/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.batch-questions/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.bilingual-terms/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.calculations/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.chinese-response/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.clean-deliverable/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.complete-options/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.conclusion-first/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.concrete-titles/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.consistent-items/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.device-readable/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.editable-diagrams/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.evidence-inference/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.maintenance-chinese/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.maintenance-project-language/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.markdown-structure/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.portable-files/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.report-context/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.report-formats/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.report-review/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.response-brief/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.response-dense/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.rules-tools/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.semantic-wrap/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.solutions-proportional/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.solutions-three-level/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.tables-compact/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.tables-list-first/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.test-evidence/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/rules/om.versioned-files/versions/1.0.0.json +11 -0
- package/runtime/plugins/opinion-manager/catalog/templates/.gitkeep +0 -0
- package/runtime/plugins/opinion-manager/catalog/templates/delivery-standard/compact-tables/versions/1.0.0.json +21 -0
- package/runtime/plugins/opinion-manager/catalog/templates/delivery-standard/list-first/versions/1.0.0.json +21 -0
- package/runtime/plugins/opinion-manager/catalog/templates/development-standard/chinese-maintenance/versions/1.0.0.json +15 -0
- package/runtime/plugins/opinion-manager/catalog/templates/development-standard/project-language/versions/1.0.0.json +15 -0
- package/runtime/plugins/opinion-manager/catalog/templates/response-style/brief-chinese/versions/1.0.0.json +19 -0
- package/runtime/plugins/opinion-manager/catalog/templates/response-style/dense-chinese/versions/1.0.0.json +19 -0
- package/runtime/plugins/opinion-manager/catalog/templates/scenario/concise-project/versions/1.0.0.json +39 -0
- package/runtime/plugins/opinion-manager/catalog/templates/scenario/reviewed-project/versions/1.0.0.json +39 -0
- package/runtime/plugins/opinion-manager/catalog/templates/work-method/proportional/versions/1.0.0.json +20 -0
- package/runtime/plugins/opinion-manager/catalog/templates/work-method/three-level/versions/1.0.0.json +20 -0
- package/runtime/plugins/opinion-manager/content.json +63 -0
- package/runtime/plugins/opinion-manager/plugin.json +18 -0
- package/runtime/plugins/opinion-manager/references/lock.schema.json +14 -0
- package/runtime/plugins/opinion-manager/references/onboarding.md +57 -0
- package/runtime/plugins/opinion-manager/references/profile.schema.json +56 -0
- package/runtime/plugins/opinion-manager/references/rule.schema.json +16 -0
- package/runtime/plugins/opinion-manager/references/template.schema.json +71 -0
- package/runtime/plugins/opinion-manager/references/versioned-template.schema.json +19 -0
- package/runtime/plugins/opinion-manager/references/versioning.md +263 -0
- package/runtime/plugins/opinion-manager/runtime/constants.py +7 -0
- package/runtime/plugins/opinion-manager/runtime/context.py +180 -0
- package/runtime/plugins/opinion-manager/runtime/lifecycle.py +395 -0
- package/runtime/plugins/opinion-manager/runtime/opinion_manager.py +419 -0
- package/runtime/plugins/opinion-manager/runtime/versioned.py +705 -0
- package/runtime/plugins/opinion-manager/skills/opinion-manager/SKILL.md +62 -0
- package/runtime/plugins/opinion-manager/skills/opinion-manager/agents/openai.yaml +4 -0
- package/runtime/plugins/opinion-manager/templates/.gitkeep +1 -0
- package/runtime/plugins/project-orchestrator/.codex-plugin/plugin.json +18 -0
- package/runtime/plugins/project-orchestrator/content.json +37 -0
- package/runtime/plugins/project-orchestrator/plugin.json +19 -0
- package/runtime/plugins/project-orchestrator/references/asset-synonyms.example.json +8 -0
- package/runtime/plugins/project-orchestrator/references/case-workspaces.md +159 -0
- package/runtime/plugins/project-orchestrator/references/case.schema.json +31 -0
- package/runtime/plugins/project-orchestrator/runtime/assets.py +1288 -0
- package/runtime/plugins/project-orchestrator/runtime/case.py +557 -0
- package/runtime/plugins/project-orchestrator/runtime/case_workspace.py +350 -0
- package/runtime/plugins/project-orchestrator/runtime/memory.py +363 -0
- package/runtime/plugins/project-orchestrator/runtime/project_base.py +107 -0
- package/runtime/plugins/project-orchestrator/runtime/project_cli.py +599 -0
- package/runtime/plugins/project-orchestrator/runtime/project_demo.py +504 -0
- package/runtime/plugins/project-orchestrator/runtime/project_maintain.py +470 -0
- package/runtime/plugins/project-orchestrator/runtime/project_menu.py +394 -0
- package/runtime/plugins/project-orchestrator/runtime/project_real.py +522 -0
- package/runtime/plugins/project-orchestrator/runtime/project_write.py +329 -0
- package/runtime/plugins/project-orchestrator/runtime/wf.py +439 -0
- package/runtime/plugins/project-orchestrator/skills/project-orchestrator/SKILL.md +230 -0
- package/runtime/plugins/project-orchestrator/skills/project-orchestrator/agents/openai.yaml +4 -0
- package/runtime/plugins/project-orchestrator/templates/CASE.md +39 -0
- /package/runtime/{agent_pack_config.py → plugins/agent-pack/runtime/agent_pack_config.py} +0 -0
- /package/runtime/{mcp_env_launcher.py → plugins/agent-pack/runtime/mcp_env_launcher.py} +0 -0
- /package/runtime/{mcp_server.py → plugins/agent-pack/runtime/mcp_server.py} +0 -0
|
@@ -0,0 +1,1288 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""交付物台账:登记 Case 与额外扫描根下的文件,按标题、日期、同义词查找,追踪版本与引用。
|
|
3
|
+
|
|
4
|
+
这是 agenthelm 的官方样例实现,只读:台账是一份可随时重建的 SQLite 投影,放在用户
|
|
5
|
+
缓存目录(``$XDG_CACHE_HOME/agenthelm/assets/``,缺省 ``~/.cache``),项目里的文件与
|
|
6
|
+
各 CASE.md 仍是权威,本工具不写回任何项目文件。
|
|
7
|
+
|
|
8
|
+
扫描范围来自 Project Profile:Case 根取 ``work.case.root``(缺省 ``cases``),额外扫描根取
|
|
9
|
+
``work.assets.roots``(缺省无)。同义词来自项目可选文件
|
|
10
|
+
``.agents/moe.sakanano.agent-pack/asset-synonyms.json``(缺省为空)。凭据、密钥与证书
|
|
11
|
+
无论是否入库都不登记;嵌套的 SVN/Git 外部检出、纯缓存目录与非 shared 的 Case 不展开。
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import bisect
|
|
17
|
+
import collections
|
|
18
|
+
import hashlib
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
import re
|
|
22
|
+
import sqlite3
|
|
23
|
+
import subprocess
|
|
24
|
+
import sys
|
|
25
|
+
import time
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import Any, Iterator
|
|
28
|
+
from urllib.parse import unquote
|
|
29
|
+
|
|
30
|
+
try:
|
|
31
|
+
import fcntl
|
|
32
|
+
except ImportError: # Windows
|
|
33
|
+
fcntl = None # type: ignore[assignment]
|
|
34
|
+
|
|
35
|
+
PROFILE = Path(".agents/moe.sakanano.agent-pack/project.json")
|
|
36
|
+
SYNONYMS = Path(".agents/moe.sakanano.agent-pack/asset-synonyms.json")
|
|
37
|
+
ASSET_SCHEMA_VERSION = 1
|
|
38
|
+
DB_NAME = "assets.sqlite3"
|
|
39
|
+
LOCK_NAME = "assets.lock"
|
|
40
|
+
LAST_NAME = "wf-last.json"
|
|
41
|
+
# 生成的派生视图(Dashboard 等)在开头带此标记,不作为引用来源。
|
|
42
|
+
DERIVED_MARKER = "<!-- derived:"
|
|
43
|
+
|
|
44
|
+
SKIP_NAMES = {".DS_Store", ".gitkeep", ".localized", "Thumbs.db", "desktop.ini"}
|
|
45
|
+
SKIP_DIRS = {"__pycache__", "node_modules", ".git", ".svn", ".hg", ".venv", ".obsidian", ".idea", ".vscode"}
|
|
46
|
+
# 目录里出现这些标记时视为嵌套的外部检出,整体不登记。
|
|
47
|
+
EXTERNAL_MARKERS = (".svn", ".git")
|
|
48
|
+
CACHE_DIR_RE = re.compile(r"^(\.cache|cache|.*[-_]cache(?:[-_].*)?)$", re.IGNORECASE)
|
|
49
|
+
# 被 .gitignore 排除的文件只在属于业务文档、表格或压缩包时登记(仅元数据)。
|
|
50
|
+
BUSINESS_EXT = {".xlsx", ".xls", ".csv", ".zip", ".7z", ".rar", ".tar", ".gz", ".tgz",
|
|
51
|
+
".docx", ".doc", ".pptx", ".ppt", ".pdf", ".numbers", ".pages"}
|
|
52
|
+
# 凭据、密钥、证书:无论是否入库都不登记。
|
|
53
|
+
DENY_NAME_RE = re.compile(r"(?i)(credential|password|passwd|secret|token|cookie|\.env|id_rsa|id_ed25519|"
|
|
54
|
+
r"private[-_]?key)")
|
|
55
|
+
DENY_EXT = {".pem", ".key", ".p12", ".pfx", ".cer", ".crt", ".mobileprovision", ".keystore", ".jks"}
|
|
56
|
+
# macOS iCloud 未下载文件(st_flags 的 SF_DATALESS):不读内容、不算哈希,避免触发下载。
|
|
57
|
+
SF_DATALESS = 0x40000000
|
|
58
|
+
HASH_LIMIT = 200 * 1024 * 1024
|
|
59
|
+
TEXT_EXT = {".md", ".csv", ".txt", ".json", ".jsonl", ".html", ".htm", ".py", ".yaml", ".yml", ".sql"}
|
|
60
|
+
CONTENT_LIMIT = 1024 * 1024
|
|
61
|
+
DECLARED_KEYS = ("deliverables", "evidence")
|
|
62
|
+
STATUS_ORDER = {"active": 0, "waiting": 1, "captured": 2, "suspended": 3, "done": 4, "abandoned": 5}
|
|
63
|
+
ROLE_SCORE = {"deliverable": 2, "draft": 1, "source": -1, "process": -1, "image": -2, "case-record": -1}
|
|
64
|
+
|
|
65
|
+
MD_LINK = re.compile(r"\]\((<[^>]+>|[^)\n]+)\)")
|
|
66
|
+
WIKI = re.compile(r"\[\[([^\]\n|#]+)(?:#[^\]\n|]*)?(?:\|[^\]\n]*)?\]\]")
|
|
67
|
+
TICK = re.compile(r"`([^`\n]{3,300})`")
|
|
68
|
+
# 只匹配正文里的 Case 编号;路径片段(前后带 /)由路径解析处理。
|
|
69
|
+
CASE_ID = re.compile(r"(?<![/\w-])CASE-\d{8}-\d{3}(?:-[a-z0-9]+)*(?![/\w])")
|
|
70
|
+
LEGACY_TASK = re.compile(r"\btasks/(\d{8}-[\w.\-]+)")
|
|
71
|
+
|
|
72
|
+
CREATE_SQL = """
|
|
73
|
+
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
74
|
+
CREATE TABLE containers (
|
|
75
|
+
id TEXT PRIMARY KEY, kind TEXT NOT NULL, title TEXT NOT NULL, status TEXT NOT NULL,
|
|
76
|
+
sensitivity TEXT NOT NULL, goal TEXT NOT NULL, tags TEXT NOT NULL, aliases TEXT NOT NULL,
|
|
77
|
+
updated_at TEXT NOT NULL
|
|
78
|
+
);
|
|
79
|
+
CREATE TABLE files (
|
|
80
|
+
path TEXT PRIMARY KEY, name TEXT NOT NULL, ext TEXT NOT NULL, container TEXT NOT NULL,
|
|
81
|
+
container_kind TEXT NOT NULL, role TEXT NOT NULL, size INTEGER NOT NULL,
|
|
82
|
+
mtime_ns INTEGER NOT NULL, sha256 TEXT, git_state TEXT NOT NULL, git_dirty INTEGER NOT NULL,
|
|
83
|
+
last_commit TEXT, dataless INTEGER NOT NULL, title TEXT NOT NULL, family TEXT NOT NULL, dates TEXT NOT NULL
|
|
84
|
+
);
|
|
85
|
+
CREATE TABLE edges (src TEXT NOT NULL, dst TEXT NOT NULL, type TEXT NOT NULL, raw TEXT NOT NULL,
|
|
86
|
+
PRIMARY KEY (src, dst));
|
|
87
|
+
CREATE TABLE broken (src TEXT NOT NULL, raw TEXT NOT NULL, type TEXT NOT NULL, class TEXT NOT NULL,
|
|
88
|
+
PRIMARY KEY (src, raw));
|
|
89
|
+
CREATE INDEX files_family ON files(family);
|
|
90
|
+
CREATE INDEX files_container ON files(container);
|
|
91
|
+
CREATE INDEX edges_dst ON edges(dst);
|
|
92
|
+
"""
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class AssetError(Exception):
|
|
96
|
+
def __init__(self, message: str, code: str = "invalid-request") -> None:
|
|
97
|
+
super().__init__(message)
|
|
98
|
+
self.code = code
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _iso(seconds: float | None) -> str | None:
|
|
102
|
+
return time.strftime("%Y-%m-%dT%H:%M:%S%z", time.localtime(seconds)) if seconds else None
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _now() -> str:
|
|
106
|
+
return _iso(time.time()) or ""
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
# ---------------------------------------------------------------- project
|
|
110
|
+
|
|
111
|
+
class Project:
|
|
112
|
+
"""Scan layout from the Project Profile; every field is optional."""
|
|
113
|
+
|
|
114
|
+
def __init__(self, root: Path) -> None:
|
|
115
|
+
self.root = root.expanduser().resolve()
|
|
116
|
+
if not self.root.is_dir():
|
|
117
|
+
raise AssetError(f"项目目录不存在:{root}", "project-missing")
|
|
118
|
+
profile_path = self.root / PROFILE
|
|
119
|
+
self.profile_found = profile_path.is_file()
|
|
120
|
+
profile: dict = {}
|
|
121
|
+
if self.profile_found:
|
|
122
|
+
try:
|
|
123
|
+
profile = json.loads(profile_path.read_text(encoding="utf-8"))
|
|
124
|
+
except (OSError, ValueError) as exc:
|
|
125
|
+
raise AssetError(f"Project Profile 无法解析:{exc}", "profile-invalid") from exc
|
|
126
|
+
work = profile.get("work") if isinstance(profile, dict) else None
|
|
127
|
+
work = work if isinstance(work, dict) else {}
|
|
128
|
+
case = work.get("case") if isinstance(work.get("case"), dict) else {}
|
|
129
|
+
assets = work.get("assets") if isinstance(work.get("assets"), dict) else {}
|
|
130
|
+
self.case_root = self._relative(case.get("root") or "cases", "work.case.root")
|
|
131
|
+
extra = assets.get("roots") or []
|
|
132
|
+
if not isinstance(extra, list) or not all(isinstance(item, str) for item in extra):
|
|
133
|
+
raise AssetError("work.assets.roots 必须是字符串列表", "profile-invalid")
|
|
134
|
+
roots = [self.case_root]
|
|
135
|
+
for value in extra:
|
|
136
|
+
relative = self._relative(value, "work.assets.roots")
|
|
137
|
+
if relative not in roots:
|
|
138
|
+
roots.append(relative)
|
|
139
|
+
for first in roots:
|
|
140
|
+
for second in roots:
|
|
141
|
+
if first != second and second.startswith(first + "/"):
|
|
142
|
+
raise AssetError(f"扫描根互相嵌套:{first} 与 {second}", "profile-invalid")
|
|
143
|
+
self.roots = roots
|
|
144
|
+
|
|
145
|
+
@classmethod
|
|
146
|
+
def discover(cls, start: Path | None, explicit: bool) -> "Project":
|
|
147
|
+
"""``--project`` is taken as given; otherwise walk up from cwd to the nearest Profile."""
|
|
148
|
+
base = (start or Path.cwd()).expanduser().resolve()
|
|
149
|
+
if not explicit:
|
|
150
|
+
for candidate in (base, *base.parents):
|
|
151
|
+
if (candidate / PROFILE).is_file():
|
|
152
|
+
return cls(candidate)
|
|
153
|
+
return cls(base)
|
|
154
|
+
|
|
155
|
+
def _relative(self, value: Any, label: str) -> str:
|
|
156
|
+
if not isinstance(value, str):
|
|
157
|
+
raise AssetError(f"{label} 必须是项目内相对路径", "profile-invalid")
|
|
158
|
+
relative = Path(value)
|
|
159
|
+
if relative.is_absolute() or ".." in relative.parts or not relative.parts:
|
|
160
|
+
raise AssetError(f"{label} 必须是项目内相对路径", "profile-invalid")
|
|
161
|
+
current = self.root
|
|
162
|
+
for part in relative.parts:
|
|
163
|
+
current = current / part
|
|
164
|
+
if current.is_symlink():
|
|
165
|
+
raise AssetError(f"{label} 不能经过符号链接", "symlink")
|
|
166
|
+
return relative.as_posix()
|
|
167
|
+
|
|
168
|
+
def state_dir(self, create: bool = False) -> Path:
|
|
169
|
+
base = os.environ.get("XDG_CACHE_HOME")
|
|
170
|
+
cache = (Path(base) if base else Path.home() / ".cache") / "agenthelm" / "assets"
|
|
171
|
+
key = hashlib.sha256(str(self.root).encode("utf-8")).hexdigest()[:16]
|
|
172
|
+
directory = cache / key
|
|
173
|
+
resolved = directory.resolve()
|
|
174
|
+
if resolved == self.root or self.root in resolved.parents:
|
|
175
|
+
raise AssetError("缓存目录不能位于项目内(检查 XDG_CACHE_HOME)", "state-inside-project")
|
|
176
|
+
if create:
|
|
177
|
+
directory.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
178
|
+
try:
|
|
179
|
+
os.chmod(directory, 0o700)
|
|
180
|
+
except OSError:
|
|
181
|
+
pass
|
|
182
|
+
for path in (directory, directory / DB_NAME, directory / LOCK_NAME, directory / LAST_NAME):
|
|
183
|
+
if path.is_symlink():
|
|
184
|
+
raise AssetError(f"缓存路径不能是符号链接:{path.name}", "symlink")
|
|
185
|
+
return directory
|
|
186
|
+
|
|
187
|
+
def synonyms(self) -> list[list[str]]:
|
|
188
|
+
path = self.root / SYNONYMS
|
|
189
|
+
if not path.is_file() or path.is_symlink():
|
|
190
|
+
return []
|
|
191
|
+
try:
|
|
192
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
193
|
+
except (OSError, ValueError) as exc:
|
|
194
|
+
raise AssetError(f"同义词文件无法解析:{exc}", "synonyms-invalid") from exc
|
|
195
|
+
groups = data.get("groups") if isinstance(data, dict) else None
|
|
196
|
+
if not isinstance(groups, list):
|
|
197
|
+
raise AssetError("同义词文件需要 groups 列表", "synonyms-invalid")
|
|
198
|
+
return [[str(word).lower() for word in group if str(word).strip()]
|
|
199
|
+
for group in groups if isinstance(group, list) and len(group) > 1]
|
|
200
|
+
|
|
201
|
+
def container_of(self, rel: str) -> tuple[str, str, str]:
|
|
202
|
+
"""(container id, kind, role) for a project-relative path."""
|
|
203
|
+
for root in self.roots:
|
|
204
|
+
if not rel.startswith(root + "/"):
|
|
205
|
+
continue
|
|
206
|
+
inner = rel[len(root) + 1:].split("/")
|
|
207
|
+
if len(inner) < 2:
|
|
208
|
+
return root, "other", "other"
|
|
209
|
+
if root == self.case_root:
|
|
210
|
+
return inner[0], "case", _role_of(inner[1:])
|
|
211
|
+
return f"{root}/{inner[0]}", "project", "project-file"
|
|
212
|
+
return rel.split("/", 1)[0], "other", "other"
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _role_of(parts: list[str]) -> str:
|
|
216
|
+
"""Role inside a Case, following the case-workspace layout."""
|
|
217
|
+
if parts == ["CASE.md"]:
|
|
218
|
+
return "case-record"
|
|
219
|
+
top = parts[0]
|
|
220
|
+
if top == "docs":
|
|
221
|
+
sub = parts[1] if len(parts) > 2 else ""
|
|
222
|
+
return {"drafts": "draft", "refs": "source", "images": "image"}.get(sub, "deliverable")
|
|
223
|
+
return {"refs": "source", "data": "data", "images": "image", "scripts": "script",
|
|
224
|
+
".agent-doc": "process", "work-items": "work-item"}.get(top, "other")
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def parse_frontmatter(text: str) -> dict:
|
|
228
|
+
"""Tolerant top-level reader (scalars and one-level lists); project Schemas vary."""
|
|
229
|
+
if not text.startswith("---"):
|
|
230
|
+
return {}
|
|
231
|
+
end = text.find("\n---", 3)
|
|
232
|
+
if end < 0:
|
|
233
|
+
return {}
|
|
234
|
+
data: dict[str, Any] = {}
|
|
235
|
+
key = None
|
|
236
|
+
for line in text[3:end].splitlines():
|
|
237
|
+
match = re.match(r"^([A-Za-z_]\w*):\s*(.*)$", line)
|
|
238
|
+
if match:
|
|
239
|
+
key, value = match.group(1), match.group(2).strip()
|
|
240
|
+
data[key] = [] if value in ("", "[]") else value.strip('"')
|
|
241
|
+
continue
|
|
242
|
+
match = re.match(r"^\s+-\s+(.*)$", line)
|
|
243
|
+
if match and key is not None:
|
|
244
|
+
if not isinstance(data.get(key), list):
|
|
245
|
+
data[key] = []
|
|
246
|
+
data[key].append(match.group(1).strip().strip('"'))
|
|
247
|
+
return data
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
# ---------------------------------------------------------------- titles, dates, versions
|
|
251
|
+
|
|
252
|
+
_TITLE_H1 = re.compile(r"(?m)^#\s+(.+?)\s*$")
|
|
253
|
+
_TITLE_HTML = re.compile(r"(?is)<title[^>]*>(.*?)</title>")
|
|
254
|
+
_DATE_FULL = (
|
|
255
|
+
re.compile(r"(?<!\d)(20\d{2})(\d{2})(\d{2})(?!\d)"),
|
|
256
|
+
re.compile(r"(?<!\d)(20\d{2})[-_./](\d{1,2})[-_./](\d{1,2})(?!\d)"),
|
|
257
|
+
re.compile(r"(?<!\d)(20\d{2})\s*年\s*(\d{1,2})\s*月\s*(\d{1,2})\s*[日号]?"),
|
|
258
|
+
)
|
|
259
|
+
_DATE_MD = (
|
|
260
|
+
re.compile(r"(?<!\d)(\d{1,2})\s*月\s*(\d{1,2})\s*[日号]?"),
|
|
261
|
+
re.compile(r"(?<![\d.])(\d{1,2})[./](\d{1,2})(?![\d.])"),
|
|
262
|
+
re.compile(r"(?<!\d)(0[1-9]|1[0-2])(0[1-9]|[12]\d|3[01])(?!\d)"),
|
|
263
|
+
)
|
|
264
|
+
_VERSION_TOKEN = re.compile(
|
|
265
|
+
r"(?i)[-_ ]?(v\d+(?:\.\d+)*|final|draft|web|test\d*|copy|终稿|定稿|草稿|评审稿|讨论稿|副本|\(\d+\))"
|
|
266
|
+
)
|
|
267
|
+
_TRAILING_DATE = re.compile(r"[-_ ]*\d{8,10}$")
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def read_title(path: Path, ext: str) -> str:
|
|
271
|
+
"""Best-effort document title; cheap, bounded read, never raises."""
|
|
272
|
+
if ext not in (".md", ".html", ".htm"):
|
|
273
|
+
return ""
|
|
274
|
+
try:
|
|
275
|
+
with open(path, "rb") as handle:
|
|
276
|
+
head = handle.read(16384).decode("utf-8", "replace")
|
|
277
|
+
except OSError:
|
|
278
|
+
return ""
|
|
279
|
+
if ext == ".md":
|
|
280
|
+
front = parse_frontmatter(head)
|
|
281
|
+
if isinstance(front.get("title"), str) and front["title"]:
|
|
282
|
+
return front["title"][:200]
|
|
283
|
+
body = head
|
|
284
|
+
if head.startswith("---"):
|
|
285
|
+
end = head.find("\n---", 3)
|
|
286
|
+
body = head[end + 4:] if end > 0 else head
|
|
287
|
+
match = _TITLE_H1.search(body)
|
|
288
|
+
return match.group(1)[:200] if match else ""
|
|
289
|
+
match = _TITLE_HTML.search(head)
|
|
290
|
+
return re.sub(r"\s+", " ", match.group(1)).strip()[:200] if match else ""
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def extract_dates(*texts: str) -> str:
|
|
294
|
+
"""Normalized date tokens: ``yyyymmdd`` when the year is known, plus ``mmdd``."""
|
|
295
|
+
found = set()
|
|
296
|
+
for text in texts:
|
|
297
|
+
if not text:
|
|
298
|
+
continue
|
|
299
|
+
for pattern in _DATE_FULL:
|
|
300
|
+
for match in pattern.finditer(text):
|
|
301
|
+
month, day = int(match.group(2)), int(match.group(3))
|
|
302
|
+
if 1 <= month <= 12 and 1 <= day <= 31:
|
|
303
|
+
found.add(f"{match.group(1)}{month:02d}{day:02d}")
|
|
304
|
+
found.add(f"{month:02d}{day:02d}")
|
|
305
|
+
for pattern in _DATE_MD:
|
|
306
|
+
for match in pattern.finditer(text):
|
|
307
|
+
month, day = int(match.group(1)), int(match.group(2))
|
|
308
|
+
if 1 <= month <= 12 and 1 <= day <= 31:
|
|
309
|
+
found.add(f"{month:02d}{day:02d}")
|
|
310
|
+
return " ".join(sorted(found))
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def family_key(container: str, name: str) -> str:
|
|
314
|
+
"""Files that differ only by version, draft marker or trailing date share a family."""
|
|
315
|
+
stem, ext = os.path.splitext(name)
|
|
316
|
+
stem = _VERSION_TOKEN.sub("", stem)
|
|
317
|
+
stem = _TRAILING_DATE.sub("", stem).strip(" -_") or stem
|
|
318
|
+
return f"{container}|{stem.lower()}{ext.lower()}"
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def version_rank(name: str) -> tuple:
|
|
322
|
+
"""Explicit vN and a trailing date outrank mtime, which sync clients rewrite."""
|
|
323
|
+
stem = os.path.splitext(name)[0]
|
|
324
|
+
match = re.search(r"(?i)v(\d+(?:\.\d+)*)", stem)
|
|
325
|
+
numbers = tuple(int(part) for part in match.group(1).split(".")) if match else ()
|
|
326
|
+
date = re.search(r"(\d{8,10})$", stem)
|
|
327
|
+
return numbers, int(date.group(1)) if date else 0
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def parse_query(terms: list[str]) -> tuple[list[str], list[tuple[str | None, str]]]:
|
|
331
|
+
"""Split free text into (text terms, [(year or None, 'mmdd')])."""
|
|
332
|
+
text = " ".join(terms)
|
|
333
|
+
dates: list[tuple[str | None, str]] = []
|
|
334
|
+
for pattern in _DATE_FULL:
|
|
335
|
+
for match in pattern.finditer(text):
|
|
336
|
+
month, day = int(match.group(2)), int(match.group(3))
|
|
337
|
+
if 1 <= month <= 12 and 1 <= day <= 31:
|
|
338
|
+
dates.append((match.group(1), f"{month:02d}{day:02d}"))
|
|
339
|
+
text = pattern.sub(" ", text)
|
|
340
|
+
for pattern in _DATE_MD:
|
|
341
|
+
for match in pattern.finditer(text):
|
|
342
|
+
month, day = int(match.group(1)), int(match.group(2))
|
|
343
|
+
if 1 <= month <= 12 and 1 <= day <= 31:
|
|
344
|
+
dates.append((None, f"{month:02d}{day:02d}"))
|
|
345
|
+
text = pattern.sub(" ", text)
|
|
346
|
+
return [term for term in text.split() if term], dates
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def expand_terms(terms: list[str], groups: list[list[str]]) -> list[list[str]]:
|
|
350
|
+
"""Each term splits into known synonym words plus leftovers; each part gains its alternatives."""
|
|
351
|
+
keys = sorted({word for group in groups for word in group}, key=len, reverse=True)
|
|
352
|
+
out = []
|
|
353
|
+
for term in terms:
|
|
354
|
+
lowered = term.lower()
|
|
355
|
+
parts: list[str] = []
|
|
356
|
+
rest = lowered
|
|
357
|
+
while rest:
|
|
358
|
+
for key in keys:
|
|
359
|
+
if rest.startswith(key):
|
|
360
|
+
parts.append(key)
|
|
361
|
+
rest = rest[len(key):]
|
|
362
|
+
break
|
|
363
|
+
else:
|
|
364
|
+
# Unknown characters glue together into one sub-term.
|
|
365
|
+
if parts and parts[-1] not in keys:
|
|
366
|
+
parts[-1] += rest[0]
|
|
367
|
+
else:
|
|
368
|
+
parts.append(rest[0])
|
|
369
|
+
rest = rest[1:]
|
|
370
|
+
if not any(part in keys for part in parts):
|
|
371
|
+
parts = [lowered]
|
|
372
|
+
for part in parts:
|
|
373
|
+
alternatives = {part}
|
|
374
|
+
for group in groups:
|
|
375
|
+
if part in group:
|
|
376
|
+
alternatives.update(group)
|
|
377
|
+
out.append(sorted(alternatives))
|
|
378
|
+
return out
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
# ---------------------------------------------------------------- scanning
|
|
382
|
+
|
|
383
|
+
def _walk(project: Project, skipped: list | None = None,
|
|
384
|
+
checkouts: list | None = None) -> Iterator[tuple[str, os.stat_result]]:
|
|
385
|
+
"""Yield (relative path, stat) for regular files under the scan roots; symlinks are never followed."""
|
|
386
|
+
root = project.root
|
|
387
|
+
stack = [root / base for base in project.roots if (root / base).is_dir()]
|
|
388
|
+
while stack:
|
|
389
|
+
directory = stack.pop()
|
|
390
|
+
try:
|
|
391
|
+
entries = list(os.scandir(directory))
|
|
392
|
+
except OSError:
|
|
393
|
+
continue
|
|
394
|
+
if checkouts is not None and any(
|
|
395
|
+
entry.name in EXTERNAL_MARKERS and entry.is_dir(follow_symlinks=False) for entry in entries):
|
|
396
|
+
checkouts.append(directory.relative_to(root).as_posix())
|
|
397
|
+
for entry in entries:
|
|
398
|
+
try:
|
|
399
|
+
if entry.is_symlink():
|
|
400
|
+
continue
|
|
401
|
+
if entry.is_dir(follow_symlinks=False):
|
|
402
|
+
if entry.name in SKIP_DIRS:
|
|
403
|
+
continue
|
|
404
|
+
if CACHE_DIR_RE.match(entry.name):
|
|
405
|
+
if skipped is not None:
|
|
406
|
+
skipped.append({"reason": "cache", "path": Path(entry.path).relative_to(root).as_posix()})
|
|
407
|
+
continue
|
|
408
|
+
stack.append(Path(entry.path))
|
|
409
|
+
elif entry.is_file(follow_symlinks=False):
|
|
410
|
+
if entry.name in SKIP_NAMES or entry.name.endswith(".pyc"):
|
|
411
|
+
continue
|
|
412
|
+
yield Path(entry.path).relative_to(root).as_posix(), entry.stat(follow_symlinks=False)
|
|
413
|
+
except OSError:
|
|
414
|
+
continue
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def listing_fingerprint(project: Project) -> tuple[str, int]:
|
|
418
|
+
digest = hashlib.sha256()
|
|
419
|
+
for root in project.roots:
|
|
420
|
+
digest.update(f"root\0{root}\n".encode("utf-8"))
|
|
421
|
+
count = 0
|
|
422
|
+
for rel, stat in sorted(_walk(project), key=lambda item: item[0]):
|
|
423
|
+
digest.update(f"{rel}\0{stat.st_size}\0{stat.st_mtime_ns}\n".encode("utf-8"))
|
|
424
|
+
count += 1
|
|
425
|
+
return digest.hexdigest(), count
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def _git(root: Path, *args: str, input_bytes: bytes | None = None) -> bytes | None:
|
|
429
|
+
"""git output, or None when git is missing or the project is not a work tree."""
|
|
430
|
+
try:
|
|
431
|
+
result = subprocess.run(["git", "-c", "core.quotepath=false", *args], cwd=root, input=input_bytes,
|
|
432
|
+
capture_output=True, check=False, timeout=120)
|
|
433
|
+
except (OSError, subprocess.SubprocessError):
|
|
434
|
+
return None
|
|
435
|
+
return result.stdout if result.returncode in (0, 1) else None
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def _git_state(project: Project, paths: list[str]) -> tuple[bool, set, dict, set, set]:
|
|
439
|
+
root = project.root
|
|
440
|
+
inside = _git(root, "rev-parse", "--is-inside-work-tree")
|
|
441
|
+
if not inside or inside.strip() != b"true":
|
|
442
|
+
return False, set(), {}, set(), set()
|
|
443
|
+
roots = list(project.roots)
|
|
444
|
+
# log and status print paths from the repository top; the project may be a subdirectory.
|
|
445
|
+
prefix = (_git(root, "rev-parse", "--show-prefix") or b"").decode("utf-8", "replace").strip()
|
|
446
|
+
tracked = {item.decode("utf-8", "replace")
|
|
447
|
+
for item in (_git(root, "ls-files", "-z", "--", *roots) or b"").split(b"\0") if item}
|
|
448
|
+
last: dict[str, int] = {}
|
|
449
|
+
stamp = None
|
|
450
|
+
log = (_git(root, "log", "--format=@%ct", "--name-only", "--", *roots) or b"").decode("utf-8", "replace")
|
|
451
|
+
for line in log.splitlines():
|
|
452
|
+
if line.startswith("@"):
|
|
453
|
+
stamp = int(line[1:])
|
|
454
|
+
elif line.startswith(prefix) and line[len(prefix):] not in last and stamp is not None:
|
|
455
|
+
last[line[len(prefix):]] = stamp
|
|
456
|
+
dirty = set()
|
|
457
|
+
for record in (_git(root, "status", "--porcelain", "-z", "--", *roots) or b"").split(b"\0"):
|
|
458
|
+
path = record[3:].decode("utf-8", "replace")
|
|
459
|
+
if len(record) > 3 and path.startswith(prefix):
|
|
460
|
+
dirty.add(path[len(prefix):])
|
|
461
|
+
candidates = [path for path in paths if path not in tracked]
|
|
462
|
+
ignored: set = set()
|
|
463
|
+
if candidates:
|
|
464
|
+
output = _git(root, "check-ignore", "--stdin", "-z", input_bytes="\0".join(candidates).encode("utf-8"))
|
|
465
|
+
ignored = {item for item in (output or b"").decode("utf-8", "replace").split("\0") if item}
|
|
466
|
+
return True, tracked, last, dirty, ignored
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def _ignored_dir(root: Path, relative: str) -> bool:
|
|
470
|
+
try:
|
|
471
|
+
return subprocess.run(["git", "check-ignore", "-q", "--", relative + "/"], cwd=root,
|
|
472
|
+
capture_output=True, timeout=30).returncode == 0
|
|
473
|
+
except (OSError, subprocess.SubprocessError):
|
|
474
|
+
return False
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def _checkout_roots(project: Project, checkouts: list[str]) -> set[str]:
|
|
478
|
+
"""Top-most git-ignored ancestor of each nested checkout, never above its scan root."""
|
|
479
|
+
found = set()
|
|
480
|
+
for directory in checkouts:
|
|
481
|
+
current = directory
|
|
482
|
+
while True:
|
|
483
|
+
parent = os.path.dirname(current)
|
|
484
|
+
if not parent or parent in project.roots or not _ignored_dir(project.root, parent):
|
|
485
|
+
break
|
|
486
|
+
current = parent
|
|
487
|
+
found.add(current)
|
|
488
|
+
return found
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _is_dataless(stat: os.stat_result) -> bool:
|
|
492
|
+
return sys.platform == "darwin" and bool(getattr(stat, "st_flags", 0) & SF_DATALESS)
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def _sha256_file(path: Path) -> str:
|
|
496
|
+
digest = hashlib.sha256()
|
|
497
|
+
with open(path, "rb") as handle:
|
|
498
|
+
for chunk in iter(lambda: handle.read(1 << 20), b""):
|
|
499
|
+
digest.update(chunk)
|
|
500
|
+
return digest.hexdigest()
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def _load_containers(project: Project) -> dict[str, dict]:
|
|
504
|
+
containers = {}
|
|
505
|
+
case_root = project.root / project.case_root
|
|
506
|
+
if not case_root.is_dir():
|
|
507
|
+
return containers
|
|
508
|
+
for directory in sorted(case_root.iterdir()):
|
|
509
|
+
record = directory / "CASE.md"
|
|
510
|
+
if directory.is_symlink() or not directory.is_dir() or record.is_symlink() or not record.is_file():
|
|
511
|
+
continue
|
|
512
|
+
try:
|
|
513
|
+
front = parse_frontmatter(record.read_text(encoding="utf-8", errors="replace"))
|
|
514
|
+
except OSError:
|
|
515
|
+
continue
|
|
516
|
+
|
|
517
|
+
def text(key: str, default: str = "") -> str:
|
|
518
|
+
value = front.get(key)
|
|
519
|
+
return value if isinstance(value, str) and value else default
|
|
520
|
+
|
|
521
|
+
def as_list(key: str) -> list:
|
|
522
|
+
value = front.get(key)
|
|
523
|
+
return value if isinstance(value, list) else []
|
|
524
|
+
|
|
525
|
+
containers[directory.name] = {
|
|
526
|
+
"id": directory.name, "kind": "case", "title": text("title"), "status": text("status"),
|
|
527
|
+
"sensitivity": text("sensitivity", "shared"), "goal": text("goal"), "tags": as_list("tags"),
|
|
528
|
+
"aliases": as_list("aliases"), "legacy_ids": as_list("legacy_ids"), "updated_at": text("updated_at"),
|
|
529
|
+
}
|
|
530
|
+
return containers
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def scan(project: Project, previous: dict | None = None) -> dict:
|
|
534
|
+
"""Scan the project into an in-memory registry; nothing is written."""
|
|
535
|
+
previous = previous or {}
|
|
536
|
+
root = project.root
|
|
537
|
+
containers = _load_containers(project)
|
|
538
|
+
non_shared = {cid for cid, item in containers.items() if item["sensitivity"] != "shared"}
|
|
539
|
+
skipped: list[dict] = []
|
|
540
|
+
checkouts: list[str] = []
|
|
541
|
+
raw = []
|
|
542
|
+
for rel, stat in _walk(project, skipped, checkouts):
|
|
543
|
+
container, kind, _ = project.container_of(rel)
|
|
544
|
+
if kind == "case" and container in non_shared:
|
|
545
|
+
continue
|
|
546
|
+
raw.append((rel, stat))
|
|
547
|
+
external = _checkout_roots(project, checkouts)
|
|
548
|
+
for directory in sorted(external):
|
|
549
|
+
skipped.append({"reason": "external-checkout", "path": directory})
|
|
550
|
+
if external:
|
|
551
|
+
raw = [(rel, stat) for rel, stat in raw if not any(rel.startswith(item + "/") for item in external)]
|
|
552
|
+
has_git, tracked, last_commit, dirty, ignored = _git_state(project, [rel for rel, _ in raw])
|
|
553
|
+
|
|
554
|
+
files = {}
|
|
555
|
+
denied = 0
|
|
556
|
+
for rel, stat in raw:
|
|
557
|
+
name = rel.rsplit("/", 1)[-1]
|
|
558
|
+
ext = Path(name).suffix.lower()
|
|
559
|
+
if DENY_NAME_RE.search(name) or ext in DENY_EXT:
|
|
560
|
+
denied += 1
|
|
561
|
+
continue
|
|
562
|
+
if not has_git:
|
|
563
|
+
git_state = "none"
|
|
564
|
+
elif rel in ignored:
|
|
565
|
+
git_state = "ignored"
|
|
566
|
+
if ext not in BUSINESS_EXT:
|
|
567
|
+
continue
|
|
568
|
+
elif rel in tracked:
|
|
569
|
+
git_state = "tracked"
|
|
570
|
+
else:
|
|
571
|
+
git_state = "untracked"
|
|
572
|
+
container, kind, role = project.container_of(rel)
|
|
573
|
+
dataless = _is_dataless(stat)
|
|
574
|
+
digest = None
|
|
575
|
+
if not dataless and stat.st_size <= HASH_LIMIT:
|
|
576
|
+
cached = previous.get(rel)
|
|
577
|
+
if cached and cached[0] == stat.st_size and cached[1] == stat.st_mtime_ns:
|
|
578
|
+
digest = cached[2]
|
|
579
|
+
else:
|
|
580
|
+
try:
|
|
581
|
+
digest = _sha256_file(root / rel)
|
|
582
|
+
except OSError:
|
|
583
|
+
digest = None
|
|
584
|
+
title = "" if dataless else read_title(root / rel, ext)
|
|
585
|
+
files[rel] = {
|
|
586
|
+
"path": rel, "name": name, "ext": ext, "container": container, "container_kind": kind, "role": role,
|
|
587
|
+
"size": stat.st_size, "mtime_ns": stat.st_mtime_ns, "sha256": digest, "git_state": git_state,
|
|
588
|
+
"git_dirty": 1 if rel in dirty else 0, "last_commit": _iso(last_commit.get(rel)),
|
|
589
|
+
"dataless": 1 if dataless else 0, "title": title, "family": family_key(container, name),
|
|
590
|
+
"dates": extract_dates(name, title),
|
|
591
|
+
}
|
|
592
|
+
for item in files.values():
|
|
593
|
+
if item["container_kind"] == "project" and item["container"] not in containers:
|
|
594
|
+
containers[item["container"]] = {
|
|
595
|
+
"id": item["container"], "kind": "project", "title": item["container"].split("/")[-1],
|
|
596
|
+
"status": "", "sensitivity": "shared", "goal": "", "tags": [], "aliases": [],
|
|
597
|
+
"legacy_ids": [], "updated_at": "",
|
|
598
|
+
}
|
|
599
|
+
edges, broken = _extract_edges(project, files, containers)
|
|
600
|
+
return {"files": files, "containers": containers, "edges": edges, "broken": broken,
|
|
601
|
+
"non_shared": sorted(non_shared), "skipped": skipped, "denied": denied, "git": has_git}
|
|
602
|
+
|
|
603
|
+
|
|
604
|
+
def _prefixes(project: Project) -> tuple[str, ...]:
|
|
605
|
+
"""Bare path mentions are recognised when they start at a top-level project directory."""
|
|
606
|
+
names = set(project.roots)
|
|
607
|
+
try:
|
|
608
|
+
names.update(entry.name for entry in os.scandir(project.root)
|
|
609
|
+
if entry.is_dir(follow_symlinks=False) and not entry.name.startswith("."))
|
|
610
|
+
except OSError:
|
|
611
|
+
pass
|
|
612
|
+
return tuple(sorted(name + "/" for name in names))
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
def _extract_edges(project: Project, files: dict, containers: dict) -> tuple[dict, dict]:
|
|
616
|
+
root, case_root = project.root, project.case_root
|
|
617
|
+
prefixes = _prefixes(project)
|
|
618
|
+
bare = re.compile(r"(?<![\w/.])((?:" + "|".join(re.escape(prefix) for prefix in prefixes)
|
|
619
|
+
+ r")[^\s`\"'<>)\]|,。;、]+)") if prefixes else None
|
|
620
|
+
directories = set()
|
|
621
|
+
for rel in files:
|
|
622
|
+
parent = os.path.dirname(rel)
|
|
623
|
+
while parent:
|
|
624
|
+
directories.add(parent)
|
|
625
|
+
parent = os.path.dirname(parent)
|
|
626
|
+
by_base: dict[str, list[str]] = collections.defaultdict(list)
|
|
627
|
+
for rel in files:
|
|
628
|
+
by_base[os.path.basename(rel)].append(rel)
|
|
629
|
+
if rel.endswith(".md"):
|
|
630
|
+
by_base[os.path.basename(rel)[:-3]].append(rel)
|
|
631
|
+
sorted_paths = sorted(files)
|
|
632
|
+
legacy = {str(legacy_id): f"{case_root}/{cid}/CASE.md"
|
|
633
|
+
for cid, item in containers.items() for legacy_id in item.get("legacy_ids", [])}
|
|
634
|
+
case_ids = sorted(cid for cid, item in containers.items() if item["kind"] == "case")
|
|
635
|
+
|
|
636
|
+
def exists_any(rel: str) -> str | None:
|
|
637
|
+
if rel in files:
|
|
638
|
+
return rel
|
|
639
|
+
if rel in directories:
|
|
640
|
+
# A bare top-level directory carries no locating value.
|
|
641
|
+
return rel + "/" if "/" in rel else None
|
|
642
|
+
if rel != "." and not rel.startswith("../") and (root / rel).exists():
|
|
643
|
+
return rel
|
|
644
|
+
return None
|
|
645
|
+
|
|
646
|
+
def prefix_unique(candidate: str) -> str | None:
|
|
647
|
+
if len(candidate) < 12:
|
|
648
|
+
return None
|
|
649
|
+
index = bisect.bisect_left(sorted_paths, candidate)
|
|
650
|
+
hits = []
|
|
651
|
+
while index < len(sorted_paths) and sorted_paths[index].startswith(candidate):
|
|
652
|
+
hits.append(sorted_paths[index])
|
|
653
|
+
if len(hits) > 1:
|
|
654
|
+
return None
|
|
655
|
+
index += 1
|
|
656
|
+
return hits[0] if hits else None
|
|
657
|
+
|
|
658
|
+
def resolve(raw: str, src: str, mode: str) -> tuple[str | None, bool]:
|
|
659
|
+
raw = unquote(raw.strip().strip("<>")).strip().split("#", 1)[0].rstrip(".,:;)")
|
|
660
|
+
if not raw or re.match(r"^[a-zA-Z][a-zA-Z0-9+.-]*:", raw) or raw.startswith(("/", "~")) or len(raw) > 260:
|
|
661
|
+
return None, False
|
|
662
|
+
candidates = [os.path.normpath(os.path.join(os.path.dirname(src), raw))]
|
|
663
|
+
if "/" in raw:
|
|
664
|
+
candidates.append(os.path.normpath(raw))
|
|
665
|
+
parts = src.split("/")
|
|
666
|
+
depth = len(case_root.split("/"))
|
|
667
|
+
if src.startswith(case_root + "/") and len(parts) > depth + 1:
|
|
668
|
+
candidates.append(os.path.normpath("/".join(parts[:depth + 1]) + "/" + raw))
|
|
669
|
+
if mode == "wiki":
|
|
670
|
+
candidates += [candidate + ".md" for candidate in candidates if not candidate.endswith(".md")]
|
|
671
|
+
candidates = [candidate for candidate in candidates if not candidate.startswith("..")]
|
|
672
|
+
for candidate in candidates:
|
|
673
|
+
hit = exists_any(candidate)
|
|
674
|
+
if hit:
|
|
675
|
+
return hit, True
|
|
676
|
+
if mode != "wiki":
|
|
677
|
+
for candidate in candidates:
|
|
678
|
+
hit = prefix_unique(candidate)
|
|
679
|
+
if hit:
|
|
680
|
+
return hit, True
|
|
681
|
+
if mode == "wiki":
|
|
682
|
+
base = os.path.basename(raw)
|
|
683
|
+
options = by_base.get(base) or by_base.get(base + ".md") or []
|
|
684
|
+
if len(options) == 1:
|
|
685
|
+
return options[0], True
|
|
686
|
+
looks = "/" in raw and (raw.startswith(prefixes) or "." in os.path.basename(raw))
|
|
687
|
+
return None, looks
|
|
688
|
+
|
|
689
|
+
edges: dict[tuple[str, str], tuple[str, str]] = {}
|
|
690
|
+
broken: dict[tuple[str, str], str] = {}
|
|
691
|
+
|
|
692
|
+
def add_edge(src: str, dst: str, kind: str, raw: str) -> None:
|
|
693
|
+
if dst.rstrip("/") != src and (src, dst) not in edges:
|
|
694
|
+
edges[(src, dst)] = (kind, raw)
|
|
695
|
+
|
|
696
|
+
sources = [rel for rel, item in files.items() if item["ext"] == ".md" and item["size"] < 2_000_000
|
|
697
|
+
and not item["dataless"] and item["git_state"] != "ignored"]
|
|
698
|
+
for src in sources:
|
|
699
|
+
try:
|
|
700
|
+
text = (root / src).read_text(encoding="utf-8", errors="replace")
|
|
701
|
+
except OSError:
|
|
702
|
+
continue
|
|
703
|
+
if DERIVED_MARKER in text[:600]:
|
|
704
|
+
continue
|
|
705
|
+
seen = set()
|
|
706
|
+
if src.startswith(case_root + "/") and src.endswith("/CASE.md") and src.count("/") == case_root.count("/") + 2:
|
|
707
|
+
front = parse_frontmatter(text)
|
|
708
|
+
for key in DECLARED_KEYS:
|
|
709
|
+
values = front.get(key)
|
|
710
|
+
for value in values if isinstance(values, list) else []:
|
|
711
|
+
dst, looks = resolve(value, src, "path")
|
|
712
|
+
if dst:
|
|
713
|
+
add_edge(src, dst, "declared:" + key, value)
|
|
714
|
+
elif looks:
|
|
715
|
+
broken.setdefault((src, value), "declared:" + key)
|
|
716
|
+
seen.add(value)
|
|
717
|
+
for match in MD_LINK.finditer(text):
|
|
718
|
+
dst, looks = resolve(match.group(1), src, "link")
|
|
719
|
+
if dst:
|
|
720
|
+
add_edge(src, dst, "link", match.group(1))
|
|
721
|
+
elif looks and match.group(1) not in seen:
|
|
722
|
+
broken.setdefault((src, match.group(1)), "link")
|
|
723
|
+
for match in WIKI.finditer(text):
|
|
724
|
+
dst, _ = resolve(match.group(1), src, "wiki")
|
|
725
|
+
if dst:
|
|
726
|
+
add_edge(src, dst, "wikilink", match.group(1))
|
|
727
|
+
else:
|
|
728
|
+
broken.setdefault((src, match.group(1)), "wikilink")
|
|
729
|
+
for match in TICK.finditer(text):
|
|
730
|
+
raw = match.group(1)
|
|
731
|
+
if raw in seen or (" " in raw.strip() and not raw.startswith(prefixes)):
|
|
732
|
+
continue
|
|
733
|
+
if "/" not in raw and not re.search(r"\.\w{1,5}$", raw):
|
|
734
|
+
continue
|
|
735
|
+
dst, looks = resolve(raw, src, "path")
|
|
736
|
+
if dst:
|
|
737
|
+
add_edge(src, dst, "mention", raw)
|
|
738
|
+
elif looks and raw.startswith(prefixes):
|
|
739
|
+
broken.setdefault((src, raw), "mention")
|
|
740
|
+
for match in bare.finditer(text) if bare else ():
|
|
741
|
+
raw = match.group(1).rstrip(".,:;")
|
|
742
|
+
if "*" in raw or raw.endswith("/"):
|
|
743
|
+
continue
|
|
744
|
+
dst, looks = resolve(raw, src, "path")
|
|
745
|
+
if dst:
|
|
746
|
+
add_edge(src, dst, "mention", raw)
|
|
747
|
+
elif looks:
|
|
748
|
+
broken.setdefault((src, raw), "mention")
|
|
749
|
+
for match in CASE_ID.finditer(text):
|
|
750
|
+
hits = [cid for cid in case_ids if cid == match.group(0) or cid.startswith(match.group(0) + "-")]
|
|
751
|
+
if len(hits) == 1:
|
|
752
|
+
add_edge(src, f"{case_root}/{hits[0]}/CASE.md", "case-id", match.group(0))
|
|
753
|
+
for match in LEGACY_TASK.finditer(text):
|
|
754
|
+
target = legacy.get(match.group(1))
|
|
755
|
+
if target:
|
|
756
|
+
add_edge(src, target, "legacy-id", match.group(0))
|
|
757
|
+
for key in [key for key in broken if key[0] == src and key[1].startswith(match.group(0))]:
|
|
758
|
+
del broken[key]
|
|
759
|
+
return edges, broken
|
|
760
|
+
|
|
761
|
+
|
|
762
|
+
def classify_broken(raw: str, kind: str, prefixes: tuple[str, ...] = ()) -> str:
|
|
763
|
+
if re.match(r"^tasks/\d{8}", raw):
|
|
764
|
+
return "legacy-task-id"
|
|
765
|
+
if kind.startswith("declared") and (
|
|
766
|
+
(" " in raw.strip() and not raw.startswith(prefixes)) or ":" in raw or "http" in raw):
|
|
767
|
+
return "prose-not-path"
|
|
768
|
+
if kind == "wikilink" and "/" not in raw and "." not in raw:
|
|
769
|
+
return "wikilink-topic"
|
|
770
|
+
return "real-broken"
|
|
771
|
+
|
|
772
|
+
|
|
773
|
+
# ---------------------------------------------------------------- storage
|
|
774
|
+
|
|
775
|
+
class _Lock:
|
|
776
|
+
"""Exclusive rebuild lock; advisory where the platform offers it."""
|
|
777
|
+
|
|
778
|
+
def __init__(self, path: Path) -> None:
|
|
779
|
+
self.path = path
|
|
780
|
+
self.fd: int | None = None
|
|
781
|
+
|
|
782
|
+
def __enter__(self) -> "_Lock":
|
|
783
|
+
flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0)
|
|
784
|
+
try:
|
|
785
|
+
self.fd = os.open(self.path, flags, 0o600)
|
|
786
|
+
except OSError as exc:
|
|
787
|
+
raise AssetError(f"无法打开台账锁:{exc}", "state-lock") from exc
|
|
788
|
+
try:
|
|
789
|
+
if fcntl is not None:
|
|
790
|
+
fcntl.flock(self.fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
791
|
+
else:
|
|
792
|
+
import msvcrt
|
|
793
|
+
msvcrt.locking(self.fd, msvcrt.LK_NBLCK, 1)
|
|
794
|
+
except OSError as exc:
|
|
795
|
+
os.close(self.fd)
|
|
796
|
+
self.fd = None
|
|
797
|
+
raise AssetError("台账正在被另一个进程重建,请稍后重试", "index-locked") from exc
|
|
798
|
+
return self
|
|
799
|
+
|
|
800
|
+
def __exit__(self, *_: Any) -> None:
|
|
801
|
+
if self.fd is not None:
|
|
802
|
+
os.close(self.fd) # closing releases flock / msvcrt locks
|
|
803
|
+
self.fd = None
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
def _previous_hashes(db_path: Path) -> dict:
|
|
807
|
+
if not db_path.is_file():
|
|
808
|
+
return {}
|
|
809
|
+
try:
|
|
810
|
+
connection = sqlite3.connect(f"{db_path.as_uri()}?mode=ro", uri=True)
|
|
811
|
+
try:
|
|
812
|
+
return {row[0]: (row[1], row[2], row[3]) for row in connection.execute(
|
|
813
|
+
"SELECT path, size, mtime_ns, sha256 FROM files WHERE sha256 IS NOT NULL")}
|
|
814
|
+
finally:
|
|
815
|
+
connection.close()
|
|
816
|
+
except sqlite3.Error:
|
|
817
|
+
return {}
|
|
818
|
+
|
|
819
|
+
|
|
820
|
+
def _write_database(path: Path, registry: dict, fingerprint: str, project: Project) -> None:
|
|
821
|
+
temporary = path.parent / f".assets.{os.getpid()}.{time.time_ns()}.tmp"
|
|
822
|
+
prefixes = _prefixes(project)
|
|
823
|
+
connection = sqlite3.connect(temporary)
|
|
824
|
+
try:
|
|
825
|
+
connection.executescript(CREATE_SQL)
|
|
826
|
+
connection.execute(f"PRAGMA user_version = {ASSET_SCHEMA_VERSION}")
|
|
827
|
+
meta = {"schema_version": str(ASSET_SCHEMA_VERSION), "built_at": _now(), "fingerprint": fingerprint,
|
|
828
|
+
"project": str(project.root), "roots": json.dumps(project.roots, ensure_ascii=False),
|
|
829
|
+
"file_count": str(len(registry["files"])), "edge_count": str(len(registry["edges"])),
|
|
830
|
+
"denied": str(registry["denied"]), "git": "1" if registry["git"] else "0",
|
|
831
|
+
"non_shared": json.dumps(registry["non_shared"]),
|
|
832
|
+
"skipped": json.dumps(registry["skipped"], ensure_ascii=False)}
|
|
833
|
+
connection.executemany("INSERT INTO meta VALUES (?, ?)", meta.items())
|
|
834
|
+
connection.executemany(
|
|
835
|
+
"INSERT INTO containers VALUES (?,?,?,?,?,?,?,?,?)",
|
|
836
|
+
[(item["id"], item["kind"], item["title"], item["status"], item["sensitivity"], item["goal"],
|
|
837
|
+
json.dumps(item["tags"], ensure_ascii=False), json.dumps(item["aliases"], ensure_ascii=False),
|
|
838
|
+
item["updated_at"]) for item in registry["containers"].values()])
|
|
839
|
+
connection.executemany(
|
|
840
|
+
"INSERT INTO files VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
841
|
+
[(item["path"], item["name"], item["ext"], item["container"], item["container_kind"], item["role"],
|
|
842
|
+
item["size"], item["mtime_ns"], item["sha256"], item["git_state"], item["git_dirty"],
|
|
843
|
+
item["last_commit"], item["dataless"], item["title"], item["family"], item["dates"])
|
|
844
|
+
for item in registry["files"].values()])
|
|
845
|
+
connection.executemany("INSERT INTO edges VALUES (?,?,?,?)",
|
|
846
|
+
[(src, dst, kind, raw) for (src, dst), (kind, raw) in registry["edges"].items()])
|
|
847
|
+
connection.executemany("INSERT INTO broken VALUES (?,?,?,?)",
|
|
848
|
+
[(src, raw, kind, classify_broken(raw, kind, prefixes))
|
|
849
|
+
for (src, raw), kind in registry["broken"].items()])
|
|
850
|
+
connection.commit()
|
|
851
|
+
except BaseException:
|
|
852
|
+
connection.close()
|
|
853
|
+
temporary.unlink(missing_ok=True)
|
|
854
|
+
raise
|
|
855
|
+
connection.close()
|
|
856
|
+
os.chmod(temporary, 0o600)
|
|
857
|
+
os.replace(temporary, path)
|
|
858
|
+
|
|
859
|
+
|
|
860
|
+
def _open(project: Project) -> sqlite3.Connection:
|
|
861
|
+
path = project.state_dir() / DB_NAME
|
|
862
|
+
if not path.is_file():
|
|
863
|
+
raise AssetError("台账尚未建立,先运行 rebuild(或去掉 --no-refresh)", "index-missing")
|
|
864
|
+
try:
|
|
865
|
+
connection = sqlite3.connect(f"{path.as_uri()}?mode=ro", uri=True)
|
|
866
|
+
connection.row_factory = sqlite3.Row
|
|
867
|
+
version = connection.execute("PRAGMA user_version").fetchone()[0]
|
|
868
|
+
except sqlite3.Error as exc:
|
|
869
|
+
raise AssetError("台账无法读取,请运行 rebuild --force", "index-corrupt") from exc
|
|
870
|
+
if version != ASSET_SCHEMA_VERSION:
|
|
871
|
+
connection.close()
|
|
872
|
+
raise AssetError("台账格式与当前工具不一致,请运行 rebuild --force", "index-incompatible")
|
|
873
|
+
return connection
|
|
874
|
+
|
|
875
|
+
|
|
876
|
+
def _meta(connection: sqlite3.Connection) -> dict:
|
|
877
|
+
return {row["key"]: row["value"] for row in connection.execute("SELECT key, value FROM meta")}
|
|
878
|
+
|
|
879
|
+
|
|
880
|
+
def rebuild(project: Project, force: bool = False) -> dict:
|
|
881
|
+
state = project.state_dir(create=True)
|
|
882
|
+
db_path = state / DB_NAME
|
|
883
|
+
with _Lock(state / LOCK_NAME):
|
|
884
|
+
fingerprint, count = listing_fingerprint(project)
|
|
885
|
+
if not force and db_path.is_file():
|
|
886
|
+
try:
|
|
887
|
+
connection = _open(project)
|
|
888
|
+
try:
|
|
889
|
+
meta = _meta(connection)
|
|
890
|
+
finally:
|
|
891
|
+
connection.close()
|
|
892
|
+
if meta.get("fingerprint") == fingerprint:
|
|
893
|
+
return {"ok": True, "action": "no-op", "file_count": int(meta.get("file_count", 0)),
|
|
894
|
+
"built_at": meta.get("built_at", ""), "state_dir": str(state)}
|
|
895
|
+
except AssetError:
|
|
896
|
+
pass
|
|
897
|
+
registry = scan(project, _previous_hashes(db_path))
|
|
898
|
+
_write_database(db_path, registry, fingerprint, project)
|
|
899
|
+
return {"ok": True, "action": "rebuilt", "file_count": len(registry["files"]),
|
|
900
|
+
"edge_count": len(registry["edges"]), "broken_count": len(registry["broken"]),
|
|
901
|
+
"denied": registry["denied"], "built_at": _now(), "state_dir": str(state)}
|
|
902
|
+
|
|
903
|
+
|
|
904
|
+
def status(project: Project) -> dict:
|
|
905
|
+
base = {"ok": True, "project": str(project.root), "profile": "found" if project.profile_found else "missing",
|
|
906
|
+
"roots": project.roots, "state_dir": str(project.state_dir())}
|
|
907
|
+
try:
|
|
908
|
+
connection = _open(project)
|
|
909
|
+
except AssetError as exc:
|
|
910
|
+
return {**base, "status": exc.code.replace("index-", "")}
|
|
911
|
+
try:
|
|
912
|
+
meta = _meta(connection)
|
|
913
|
+
finally:
|
|
914
|
+
connection.close()
|
|
915
|
+
fingerprint, count = listing_fingerprint(project)
|
|
916
|
+
return {**base, "status": "fresh" if meta.get("fingerprint") == fingerprint else "stale",
|
|
917
|
+
"file_count": int(meta.get("file_count", 0)), "edge_count": int(meta.get("edge_count", 0)),
|
|
918
|
+
"built_at": meta.get("built_at", ""), "current_file_count": count}
|
|
919
|
+
|
|
920
|
+
|
|
921
|
+
def _connect(project: Project, refresh: bool) -> sqlite3.Connection:
|
|
922
|
+
if refresh:
|
|
923
|
+
rebuild(project)
|
|
924
|
+
return _open(project)
|
|
925
|
+
|
|
926
|
+
|
|
927
|
+
# ---------------------------------------------------------------- queries
|
|
928
|
+
|
|
929
|
+
def _containers(connection: sqlite3.Connection) -> dict[str, dict]:
|
|
930
|
+
return {row["id"]: dict(row) for row in connection.execute("SELECT * FROM containers")}
|
|
931
|
+
|
|
932
|
+
|
|
933
|
+
def _row_info(row: sqlite3.Row, info: dict, inbound: collections.Counter) -> dict:
|
|
934
|
+
container = info.get(row["container"], {})
|
|
935
|
+
return {"path": row["path"], "name": row["name"], "title": row["title"], "role": row["role"],
|
|
936
|
+
"container": row["container"], "container_kind": row["container_kind"],
|
|
937
|
+
"case_title": container.get("title", ""), "status": container.get("status", ""),
|
|
938
|
+
"modified": _iso(row["mtime_ns"] / 1e9), "size": row["size"],
|
|
939
|
+
"git": row["git_state"] + ("+dirty" if row["git_dirty"] else ""),
|
|
940
|
+
"dataless": bool(row["dataless"]), "referenced_by": inbound.get(row["path"], 0)}
|
|
941
|
+
|
|
942
|
+
|
|
943
|
+
def _group_families(rows: list, info: dict, inbound: collections.Counter, limit: int) -> tuple[list, int]:
|
|
944
|
+
"""rows: [(score, mtime_ns, row)] -> family groups, best first; the newest version leads each group."""
|
|
945
|
+
groups: dict[str, list] = collections.OrderedDict()
|
|
946
|
+
for score, mtime, row in rows:
|
|
947
|
+
groups.setdefault(row["family"], []).append((score, mtime, row))
|
|
948
|
+
out = []
|
|
949
|
+
for members in groups.values():
|
|
950
|
+
members.sort(key=lambda member: (version_rank(member[2]["name"]), member[1]), reverse=True)
|
|
951
|
+
head = _row_info(members[0][2], info, inbound)
|
|
952
|
+
head["versions"] = [{"path": member[2]["path"], "name": member[2]["name"],
|
|
953
|
+
"modified": _iso(member[1] / 1e9), "git": member[2]["git_state"]}
|
|
954
|
+
for member in members[1:]]
|
|
955
|
+
out.append((max(member[0] for member in members), members[0][1], head))
|
|
956
|
+
out.sort(key=lambda group: (group[0], group[1]), reverse=True)
|
|
957
|
+
return [group[2] for group in out[:limit]], len(out)
|
|
958
|
+
|
|
959
|
+
|
|
960
|
+
def _inbound_counts(connection: sqlite3.Connection) -> collections.Counter:
|
|
961
|
+
return collections.Counter(row[0] for row in connection.execute("SELECT dst FROM edges"))
|
|
962
|
+
|
|
963
|
+
|
|
964
|
+
def find(project: Project, terms: list[str], *, kind: str | None = None, ext: str | None = None,
|
|
965
|
+
container: str | None = None, content: bool = False, limit: int = 10, refresh: bool = True) -> dict:
|
|
966
|
+
connection = _connect(project, refresh)
|
|
967
|
+
try:
|
|
968
|
+
info, inbound = _containers(connection), _inbound_counts(connection)
|
|
969
|
+
text_terms, query_dates = parse_query(terms)
|
|
970
|
+
expanded = expand_terms(text_terms, project.synonyms())
|
|
971
|
+
ext_value = ("." + ext.lstrip(".").lower()) if ext else None
|
|
972
|
+
scored = []
|
|
973
|
+
for row in connection.execute("SELECT * FROM files"):
|
|
974
|
+
if (kind and row["role"] != kind) or (ext_value and row["ext"] != ext_value):
|
|
975
|
+
continue
|
|
976
|
+
if container and container.lower() not in row["container"].lower():
|
|
977
|
+
continue
|
|
978
|
+
if query_dates:
|
|
979
|
+
file_dates = set(row["dates"].split())
|
|
980
|
+
if not any((year + md in file_dates) if year else (md in file_dates) for year, md in query_dates):
|
|
981
|
+
continue
|
|
982
|
+
owner = info.get(row["container"], {})
|
|
983
|
+
name, title = row["name"].lower(), row["title"].lower()
|
|
984
|
+
head = " ".join([name, title, row["path"].lower()])
|
|
985
|
+
meta = " ".join([owner.get("title", ""), owner.get("goal", ""), owner.get("tags", ""),
|
|
986
|
+
owner.get("aliases", "")]).lower()
|
|
987
|
+
score, hit, body = 0, True, None
|
|
988
|
+
for alternatives in expanded:
|
|
989
|
+
if any(word in name or word in title for word in alternatives):
|
|
990
|
+
score += 5
|
|
991
|
+
elif any(word in head for word in alternatives):
|
|
992
|
+
score += 3
|
|
993
|
+
elif any(word in meta for word in alternatives):
|
|
994
|
+
score += 1
|
|
995
|
+
elif content and row["ext"] in TEXT_EXT and row["size"] <= CONTENT_LIMIT \
|
|
996
|
+
and not row["dataless"] and row["git_state"] != "ignored":
|
|
997
|
+
if body is None:
|
|
998
|
+
try:
|
|
999
|
+
body = (project.root / row["path"]).read_text(encoding="utf-8", errors="replace").lower()
|
|
1000
|
+
except OSError:
|
|
1001
|
+
body = ""
|
|
1002
|
+
if any(word in body for word in alternatives):
|
|
1003
|
+
score += 2
|
|
1004
|
+
else:
|
|
1005
|
+
hit = False
|
|
1006
|
+
else:
|
|
1007
|
+
hit = False
|
|
1008
|
+
if not hit:
|
|
1009
|
+
break
|
|
1010
|
+
if not hit:
|
|
1011
|
+
continue
|
|
1012
|
+
score += (4 if query_dates else 0) + ROLE_SCORE.get(row["role"], 0)
|
|
1013
|
+
score += {"active": 2, "waiting": 1}.get(owner.get("status", ""), 0)
|
|
1014
|
+
scored.append((score, row["mtime_ns"], row))
|
|
1015
|
+
groups, total = _group_families(scored, info, inbound, limit)
|
|
1016
|
+
return {"ok": True, "total": total, "files": len(scored), "returned": len(groups), "results": groups,
|
|
1017
|
+
"query": {"terms": text_terms, "dates": [f"{year or '*'}-{md}" for year, md in query_dates]}}
|
|
1018
|
+
finally:
|
|
1019
|
+
connection.close()
|
|
1020
|
+
|
|
1021
|
+
|
|
1022
|
+
def recent(project: Project, *, limit: int = 15, refresh: bool = True) -> dict:
|
|
1023
|
+
connection = _connect(project, refresh)
|
|
1024
|
+
try:
|
|
1025
|
+
info, inbound = _containers(connection), _inbound_counts(connection)
|
|
1026
|
+
rows = connection.execute("SELECT * FROM files WHERE role IN ('deliverable','draft','project-file') "
|
|
1027
|
+
"ORDER BY mtime_ns DESC LIMIT 400").fetchall()
|
|
1028
|
+
groups, _ = _group_families([(0, row["mtime_ns"], row) for row in rows], info, inbound, limit)
|
|
1029
|
+
return {"ok": True, "results": groups, "returned": len(groups)}
|
|
1030
|
+
finally:
|
|
1031
|
+
connection.close()
|
|
1032
|
+
|
|
1033
|
+
|
|
1034
|
+
def cases(project: Project, term: str | None = None, *, limit: int = 30, refresh: bool = True) -> dict:
|
|
1035
|
+
connection = _connect(project, refresh)
|
|
1036
|
+
try:
|
|
1037
|
+
counts = {row[0]: row[1] for row in connection.execute("SELECT container, COUNT(*) FROM files GROUP BY container")}
|
|
1038
|
+
groups = project.synonyms()
|
|
1039
|
+
rows = []
|
|
1040
|
+
for row in connection.execute("SELECT * FROM containers WHERE kind = 'case'"):
|
|
1041
|
+
haystack = " ".join([row["id"], row["title"], row["goal"], row["tags"], row["aliases"]]).lower()
|
|
1042
|
+
if term:
|
|
1043
|
+
words, _ = parse_query([term])
|
|
1044
|
+
if not all(any(word in haystack for word in alternatives)
|
|
1045
|
+
for alternatives in expand_terms(words, groups)):
|
|
1046
|
+
continue
|
|
1047
|
+
rows.append({"id": row["id"], "title": row["title"], "status": row["status"],
|
|
1048
|
+
"updated_at": row["updated_at"], "files": counts.get(row["id"], 0)})
|
|
1049
|
+
rows.sort(key=lambda item: item["updated_at"], reverse=True)
|
|
1050
|
+
rows.sort(key=lambda item: STATUS_ORDER.get(item["status"], 6))
|
|
1051
|
+
return {"ok": True, "total": len(rows), "results": rows[:limit]}
|
|
1052
|
+
finally:
|
|
1053
|
+
connection.close()
|
|
1054
|
+
|
|
1055
|
+
|
|
1056
|
+
def _inbound_rows(connection: sqlite3.Connection, path: str) -> list[dict]:
|
|
1057
|
+
out = [{"src": row["src"], "type": row["type"], "via": "file"}
|
|
1058
|
+
for row in connection.execute("SELECT src, type FROM edges WHERE dst = ?", (path,))]
|
|
1059
|
+
parent = os.path.dirname(path)
|
|
1060
|
+
while parent:
|
|
1061
|
+
for row in connection.execute("SELECT src, type FROM edges WHERE dst = ?", (parent + "/",)):
|
|
1062
|
+
out.append({"src": row["src"], "type": row["type"], "via": "directory:" + parent + "/"})
|
|
1063
|
+
parent = os.path.dirname(parent)
|
|
1064
|
+
return out
|
|
1065
|
+
|
|
1066
|
+
|
|
1067
|
+
def refs(project: Project, path: str, *, depth: int = 1, refresh: bool = True) -> dict:
|
|
1068
|
+
connection = _connect(project, refresh)
|
|
1069
|
+
try:
|
|
1070
|
+
path = path.strip()
|
|
1071
|
+
while path.startswith("./"):
|
|
1072
|
+
path = path[2:]
|
|
1073
|
+
if not connection.execute("SELECT 1 FROM files WHERE path = ?", (path,)).fetchone() and \
|
|
1074
|
+
not connection.execute("SELECT 1 FROM edges WHERE dst = ? OR dst = ?",
|
|
1075
|
+
(path, path.rstrip("/") + "/")).fetchone():
|
|
1076
|
+
raise AssetError(f"台账里没有这个路径:{path}", "unknown-path")
|
|
1077
|
+
outbound = [{"dst": row["dst"], "type": row["type"]}
|
|
1078
|
+
for row in connection.execute("SELECT dst, type FROM edges WHERE src = ? ORDER BY dst", (path,))]
|
|
1079
|
+
direct, seen_src = [], set()
|
|
1080
|
+
for row in _inbound_rows(connection, path):
|
|
1081
|
+
if row["src"] not in seen_src:
|
|
1082
|
+
seen_src.add(row["src"])
|
|
1083
|
+
direct.append(row)
|
|
1084
|
+
layers, seen = [direct], {path} | seen_src
|
|
1085
|
+
frontier = [row["src"] for row in direct]
|
|
1086
|
+
for _ in range(max(depth, 1) - 1):
|
|
1087
|
+
following = []
|
|
1088
|
+
for node in frontier:
|
|
1089
|
+
for row in _inbound_rows(connection, node):
|
|
1090
|
+
if row["src"] not in seen:
|
|
1091
|
+
seen.add(row["src"])
|
|
1092
|
+
following.append(row)
|
|
1093
|
+
if not following:
|
|
1094
|
+
break
|
|
1095
|
+
layers.append(following)
|
|
1096
|
+
frontier = [row["src"] for row in following]
|
|
1097
|
+
owner = connection.execute("SELECT container FROM files WHERE path = ?", (path,)).fetchone()
|
|
1098
|
+
affected = sorted({project.container_of(row["src"])[0] for layer in layers for row in layer}
|
|
1099
|
+
- ({owner["container"]} if owner else set()))
|
|
1100
|
+
return {"ok": True, "path": path, "outbound": outbound,
|
|
1101
|
+
"inbound": [dict(row, depth=index + 1) for index, layer in enumerate(layers) for row in layer],
|
|
1102
|
+
"affected_containers": affected}
|
|
1103
|
+
finally:
|
|
1104
|
+
connection.close()
|
|
1105
|
+
|
|
1106
|
+
|
|
1107
|
+
def check(project: Project, *, limit: int = 50, refresh: bool = True) -> dict:
|
|
1108
|
+
connection = _connect(project, refresh)
|
|
1109
|
+
try:
|
|
1110
|
+
info = _containers(connection)
|
|
1111
|
+
files = {row["path"]: dict(row) for row in connection.execute("SELECT * FROM files")}
|
|
1112
|
+
edges = [(row["src"], row["dst"], row["type"]) for row in connection.execute("SELECT src, dst, type FROM edges")]
|
|
1113
|
+
inbound: dict[str, set] = collections.defaultdict(set)
|
|
1114
|
+
declared: dict[str, set] = collections.defaultdict(set)
|
|
1115
|
+
reference_dirs = set()
|
|
1116
|
+
docs_dir = re.compile(r"^" + re.escape(project.case_root) + r"/[^/]+/docs(/|$)")
|
|
1117
|
+
for src, dst, kind in edges:
|
|
1118
|
+
inbound[dst.rstrip("/")].add(src)
|
|
1119
|
+
if kind.startswith("declared"):
|
|
1120
|
+
declared[project.container_of(src)[0]].add(dst.rstrip("/"))
|
|
1121
|
+
if dst.endswith("/") and docs_dir.match(dst):
|
|
1122
|
+
reference_dirs.add(dst.rstrip("/"))
|
|
1123
|
+
tiers: collections.Counter = collections.Counter()
|
|
1124
|
+
unclaimed: dict[str, list] = collections.defaultdict(list)
|
|
1125
|
+
directory_only: dict[str, list] = collections.defaultdict(list)
|
|
1126
|
+
for path, item in files.items():
|
|
1127
|
+
if item["container_kind"] != "case" or item["role"] not in ("deliverable", "draft") \
|
|
1128
|
+
or item["name"].startswith("~$"):
|
|
1129
|
+
continue
|
|
1130
|
+
if path in declared[item["container"]] or any(src != path for src in inbound.get(path, ())):
|
|
1131
|
+
tiers["file"] += 1
|
|
1132
|
+
elif any(path.startswith(directory + "/") for directory in reference_dirs):
|
|
1133
|
+
tiers["directory"] += 1
|
|
1134
|
+
directory_only[item["container"]].append(path)
|
|
1135
|
+
else:
|
|
1136
|
+
tiers["none"] += 1
|
|
1137
|
+
unclaimed[item["container"]].append(path)
|
|
1138
|
+
broken: dict[str, list] = collections.defaultdict(list)
|
|
1139
|
+
for row in connection.execute("SELECT src, raw, type, class FROM broken ORDER BY src, raw"):
|
|
1140
|
+
broken[row["class"]].append({"src": row["src"], "raw": row["raw"], "type": row["type"]})
|
|
1141
|
+
by_hash: dict[str, list] = collections.defaultdict(list)
|
|
1142
|
+
for path, item in files.items():
|
|
1143
|
+
if item["sha256"] and item["size"] > 10 * 1024:
|
|
1144
|
+
by_hash[item["sha256"]].append(path)
|
|
1145
|
+
duplicates = [sorted(paths) for paths in by_hash.values()
|
|
1146
|
+
if len(paths) > 1 and len({files[path]["container"] for path in paths}) > 1]
|
|
1147
|
+
upstream_newer = []
|
|
1148
|
+
for src, dst, kind in edges:
|
|
1149
|
+
source, target = files.get(src), files.get(dst)
|
|
1150
|
+
if not source or not target or source["container"] == target["container"] or kind == "case-id":
|
|
1151
|
+
continue
|
|
1152
|
+
if target["mtime_ns"] > source["mtime_ns"] and target["container_kind"] == source["container_kind"] == "case":
|
|
1153
|
+
upstream_newer.append({"src": src, "dst": dst, "dst_modified": _iso(target["mtime_ns"] / 1e9),
|
|
1154
|
+
"src_modified": _iso(source["mtime_ns"] / 1e9)})
|
|
1155
|
+
meta = _meta(connection)
|
|
1156
|
+
return {
|
|
1157
|
+
"ok": True, "built_at": meta.get("built_at", ""),
|
|
1158
|
+
"totals": {"files": len(files), "edges": len(edges), "denied": int(meta.get("denied", 0)),
|
|
1159
|
+
"git": dict(collections.Counter(item["git_state"] for item in files.values()))},
|
|
1160
|
+
"claim_tiers": dict(tiers),
|
|
1161
|
+
"unclaimed": {key: value[:limit] for key, value in sorted(unclaimed.items())},
|
|
1162
|
+
"directory_only": {key: len(value) for key, value in sorted(directory_only.items())},
|
|
1163
|
+
"broken": {key: value[:limit] for key, value in broken.items()},
|
|
1164
|
+
"broken_counts": {key: len(value) for key, value in broken.items()},
|
|
1165
|
+
"duplicates": duplicates[:limit],
|
|
1166
|
+
"upstream_newer": upstream_newer[:limit],
|
|
1167
|
+
"untracked_new": sorted(path for path, item in files.items() if item["git_state"] == "untracked")[:limit],
|
|
1168
|
+
"skipped": json.loads(meta.get("skipped", "[]"))[:limit],
|
|
1169
|
+
"case_status": dict(collections.Counter(item["status"] for item in info.values() if item["kind"] == "case")),
|
|
1170
|
+
}
|
|
1171
|
+
finally:
|
|
1172
|
+
connection.close()
|
|
1173
|
+
|
|
1174
|
+
|
|
1175
|
+
# ---------------------------------------------------------------- cli
|
|
1176
|
+
|
|
1177
|
+
def parser() -> argparse.ArgumentParser:
|
|
1178
|
+
result = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
1179
|
+
result.add_argument("--project", type=Path, default=None)
|
|
1180
|
+
result.add_argument("--output", choices=("text", "json"), default="text")
|
|
1181
|
+
# The same options are accepted after the subcommand, where people tend to type them.
|
|
1182
|
+
shared = argparse.ArgumentParser(add_help=False)
|
|
1183
|
+
shared.add_argument("--project", type=Path, default=argparse.SUPPRESS)
|
|
1184
|
+
shared.add_argument("--output", choices=("text", "json"), default=argparse.SUPPRESS)
|
|
1185
|
+
sub = result.add_subparsers(dest="command", required=True)
|
|
1186
|
+
|
|
1187
|
+
def command(name: str, help_text: str, refresh: bool = True) -> argparse.ArgumentParser:
|
|
1188
|
+
item = sub.add_parser(name, parents=[shared], help=help_text)
|
|
1189
|
+
if refresh:
|
|
1190
|
+
item.add_argument("--no-refresh", action="store_true", help="不先增量重建,直接读现有台账")
|
|
1191
|
+
return item
|
|
1192
|
+
|
|
1193
|
+
build = command("rebuild", "重建台账(文件清单未变时为 no-op)", refresh=False)
|
|
1194
|
+
build.add_argument("--force", action="store_true")
|
|
1195
|
+
command("status", "台账是否最新、位置与扫描根", refresh=False)
|
|
1196
|
+
search = command("find", "按标题、文件名、日期与同义词查找,同一系列的版本归为一组")
|
|
1197
|
+
search.add_argument("terms", nargs="*")
|
|
1198
|
+
search.add_argument("--kind", help="角色:deliverable / draft / source / data / image / script / process / project-file")
|
|
1199
|
+
search.add_argument("--ext")
|
|
1200
|
+
search.add_argument("--in", dest="container", help="限定 Case 或目录(片段匹配)")
|
|
1201
|
+
search.add_argument("--content", action="store_true", help="名称不命中时再查文本正文(≤1 MiB)")
|
|
1202
|
+
search.add_argument("--limit", type=int, default=10)
|
|
1203
|
+
latest = command("recent", "最近改动的交付物")
|
|
1204
|
+
latest.add_argument("--limit", type=int, default=15)
|
|
1205
|
+
listing = command("cases", "Case 列表(进行中在前)与各自登记的文件数")
|
|
1206
|
+
listing.add_argument("term", nargs="?")
|
|
1207
|
+
listing.add_argument("--limit", type=int, default=30)
|
|
1208
|
+
reference = command("refs", "谁引用它、它引用谁、改动会波及哪些 Case")
|
|
1209
|
+
reference.add_argument("path")
|
|
1210
|
+
reference.add_argument("--depth", type=int, default=1)
|
|
1211
|
+
inspect = command("check", "巡检:无人认领的交付物、断链、跨容器重复、上游更新")
|
|
1212
|
+
inspect.add_argument("--limit", type=int, default=50)
|
|
1213
|
+
return result
|
|
1214
|
+
|
|
1215
|
+
|
|
1216
|
+
def render_text(command: str, result: dict) -> str:
|
|
1217
|
+
if command in ("find", "recent"):
|
|
1218
|
+
lines = []
|
|
1219
|
+
for index, item in enumerate(result["results"], 1):
|
|
1220
|
+
owner = item["case_title"] or item["container"]
|
|
1221
|
+
lines.append(f"{index:>2}. {item['title'] or item['name']} [{owner}] {item['git']} {item['path']}")
|
|
1222
|
+
lines += [f" 版本 {version['name']} {version['path']}" for version in item["versions"]]
|
|
1223
|
+
return "\n".join(lines) or "没有找到。"
|
|
1224
|
+
if command == "cases":
|
|
1225
|
+
return "\n".join(f"{item['status'] or '-':9} {item['id']} {item['title']} ({item['files']} 个文件)"
|
|
1226
|
+
for item in result["results"]) or "没有 Case。"
|
|
1227
|
+
if command == "refs":
|
|
1228
|
+
lines = [result["path"]]
|
|
1229
|
+
lines += [f" ← {item['src']}" + ("" if item["via"] == "file" else f" ({item['via']})")
|
|
1230
|
+
for item in result["inbound"]]
|
|
1231
|
+
lines += [f" → {item['dst']}" for item in result["outbound"]]
|
|
1232
|
+
if result["affected_containers"]:
|
|
1233
|
+
lines.append(" 改动会波及:" + "、".join(result["affected_containers"]))
|
|
1234
|
+
return "\n".join(lines)
|
|
1235
|
+
if command == "check":
|
|
1236
|
+
totals, tiers, counts = result["totals"], result["claim_tiers"], result["broken_counts"]
|
|
1237
|
+
git = totals["git"]
|
|
1238
|
+
lines = [f"文件 {totals['files']} · 引用 {totals['edges']} · 拒绝登记的凭据类文件 {totals['denied']}",
|
|
1239
|
+
f"Git:已入库 {git.get('tracked', 0)} / 被忽略 {git.get('ignored', 0)} / "
|
|
1240
|
+
f"未提交 {git.get('untracked', 0)} / 非 Git {git.get('none', 0)}",
|
|
1241
|
+
f"交付物:文件级被引用 {tiers.get('file', 0)} · 仅目录覆盖 {tiers.get('directory', 0)} · "
|
|
1242
|
+
f"无人认领 {tiers.get('none', 0)}",
|
|
1243
|
+
f"断链:{counts.get('real-broken', 0)} · 说明文字 {counts.get('prose-not-path', 0)} · "
|
|
1244
|
+
f"旧 Task 路径 {counts.get('legacy-task-id', 0)} · 主题链接 {counts.get('wikilink-topic', 0)}"]
|
|
1245
|
+
lines += [f" 断链 {item['src']} → {item['raw']}" for item in result["broken"].get("real-broken", [])]
|
|
1246
|
+
lines += [f" 无人认领 {path}" for paths in result["unclaimed"].values() for path in paths]
|
|
1247
|
+
lines += [" 重复 " + " ≡ ".join(group) for group in result["duplicates"]]
|
|
1248
|
+
return "\n".join(lines)
|
|
1249
|
+
return json.dumps(result, ensure_ascii=False, indent=2)
|
|
1250
|
+
|
|
1251
|
+
|
|
1252
|
+
def run(project: Project, args: argparse.Namespace) -> dict:
|
|
1253
|
+
refresh = not getattr(args, "no_refresh", False)
|
|
1254
|
+
if args.command == "rebuild":
|
|
1255
|
+
return rebuild(project, force=args.force)
|
|
1256
|
+
if args.command == "status":
|
|
1257
|
+
return status(project)
|
|
1258
|
+
if args.command == "find":
|
|
1259
|
+
return find(project, args.terms, kind=args.kind, ext=args.ext, container=args.container,
|
|
1260
|
+
content=args.content, limit=args.limit, refresh=refresh)
|
|
1261
|
+
if args.command == "recent":
|
|
1262
|
+
return recent(project, limit=args.limit, refresh=refresh)
|
|
1263
|
+
if args.command == "cases":
|
|
1264
|
+
return cases(project, args.term, limit=args.limit, refresh=refresh)
|
|
1265
|
+
if args.command == "refs":
|
|
1266
|
+
return refs(project, args.path, depth=args.depth, refresh=refresh)
|
|
1267
|
+
return check(project, limit=args.limit, refresh=refresh)
|
|
1268
|
+
|
|
1269
|
+
|
|
1270
|
+
def main(argv: list[str] | None = None) -> int:
|
|
1271
|
+
args = parser().parse_args(argv)
|
|
1272
|
+
try:
|
|
1273
|
+
project = Project.discover(args.project, explicit=args.project is not None)
|
|
1274
|
+
result = run(project, args)
|
|
1275
|
+
except AssetError as exc:
|
|
1276
|
+
payload = {"ok": False, "code": exc.code, "error": str(exc)}
|
|
1277
|
+
if args.output == "json":
|
|
1278
|
+
print(json.dumps(payload, ensure_ascii=False))
|
|
1279
|
+
else:
|
|
1280
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
1281
|
+
return 2
|
|
1282
|
+
print(json.dumps(result, ensure_ascii=False, indent=2) if args.output == "json"
|
|
1283
|
+
else render_text(args.command, result))
|
|
1284
|
+
return 0 if result.get("ok") else 1
|
|
1285
|
+
|
|
1286
|
+
|
|
1287
|
+
if __name__ == "__main__":
|
|
1288
|
+
raise SystemExit(main())
|