draftgo-cli 3.0.1 → 3.0.33

package/resources/skill/scripts/draftgo_pull.py

@@ -8,7 +8,6 @@ DraftGo Pull Script
   python draftgo_pull.py nav            [nav_id ...]
   python draftgo_pull.py db_meta        [db_meta_id ...]
   python draftgo_pull.py aihub          [aihub_id ...]
-  python draftgo_pull.py external_apis  [api_id ...]
   python draftgo_pull.py system_config  [config_key ...]
   python draftgo_pull.py docs           [article_id ...]
   python draftgo_pull.py doc_categories [category_id ...]
@@ -19,7 +18,7 @@ DraftGo Pull Script
 """
 import json, sys
 from pathlib import Path
-import urllib.request, urllib.error
+import urllib.request, urllib.error, urllib.parse
 
 SCRIPT_DIR = Path(__file__).resolve().parent
 
@@ -43,28 +42,18 @@ DEFAULT_ROOT = find_project_root(SCRIPT_DIR)
 
 # ── 端点映射 ──
 ENDPOINTS = {
-    "pages":           "/api/pages/?page=1&page_size=500",
+    "pages":           "/api/pages/",
     "navigations":     "/api/navigations",
     "roles":           "/api/roles",
-    "users":           "/api/users?page=1&page_size=500",
+    "users":           "/api/users",
     "db_meta":         "/api/db-meta",
-    "aihub":           "/api/aihub?page=1&page_size=500",
-    "external_apis":   "/api/external-apis?page=1&page_size=500",
+    "aihub":           "/api/aihub",
     "system_config":   "/api/system/config",
     "doc_categories":  "/api/docs/categories?flat=true",
-    "docs":            "/api/docs/admin/articles?page=1&page_size=500",
-    "custom_scripts":  "/api/scripts/?page=1&page_size=500",
+    "docs":            "/api/docs/admin/articles",
+    "custom_scripts":  "/api/scripts/",
 }
 
-_LANG_EXT = {
-    "python": "py", "py": "py",
-    "javascript": "js", "js": "js",
-    "typescript": "ts", "ts": "ts",
-    "shell": "sh", "bash": "sh",
-    "go": "go",
-}
-
-
 # ── 通用工具函数 ──
 
 def load_config(root=None):
@@ -129,7 +118,31 @@ def safe_slug(s):
     return str(s).strip("/").replace("/", "_") or "root"
 
 
-def write_index(folder, items):
+def write_index(folder, items, preserve_existing=False, id_key="id"):
+    if preserve_existing:
+        index_path = folder / "index.json"
+        try:
+            existing = json.loads(index_path.read_text(encoding="utf-8"))
+            if not isinstance(existing, list):
+                existing = []
+        except (OSError, json.JSONDecodeError):
+            existing = []
+        replacements = {
+            str(item.get(id_key)): item for item in items if item.get(id_key) is not None
+        }
+        retained = []
+        seen = set()
+        for item in existing:
+            key = str(item.get(id_key)) if item.get(id_key) is not None else None
+            if key in replacements:
+                retained.append(replacements[key])
+                seen.add(key)
+            else:
+                retained.append(item)
+        items = retained + [
+            item for item in items
+            if item.get(id_key) is None or str(item.get(id_key)) not in seen
+        ]
     folder.mkdir(parents=True, exist_ok=True)
     (folder / "index.json").write_text(
         json.dumps(items, ensure_ascii=False, indent=2), encoding="utf-8"
@@ -138,7 +151,7 @@ def write_index(folder, items):
 
 # ── 各类型的 explode 逻辑 ──
 
-def explode_pages(out_dir, items):
+def explode_pages(out_dir, items, preserve_existing=False):
     index = []
     for p in items:
         pid = p.get("id", "x")
@@ -148,66 +161,95 @@ def explode_pages(out_dir, items):
         meta = {k: v for k, v in p.items() if k != "value"}
         meta["html_file"] = f".draftgo/pages/{fname}"
         index.append(meta)
-    write_index(out_dir, index)
+    write_index(out_dir, index, preserve_existing)
     return len(index)
 
 
-def explode_navigations(out_dir, items):
+def explode_navigations(out_dir, items, preserve_existing=False):
     index = []
     for item in items:
         iid = item.get("id", "x")
         name_slug = safe_slug(item.get("code") or item.get("route") or iid)
         fname = f"nav_{iid}_{name_slug}.html"
-        (out_dir / fname).write_text(item.get("html", ""), encoding="utf-8")
+        # 列表接口只返回导航元数据。详情拉取失败时不能以空内容覆盖本地导航。
+        if isinstance(item.get("html"), str):
+            (out_dir / fname).write_text(item["html"], encoding="utf-8")
+        else:
+            print(
+                f"  WARN nav_id={iid} 未取得 HTML，保留本地文件: {fname}",
+                file=sys.stderr,
+            )
         meta = {k: v for k, v in item.items() if k != "html"}
         meta["html_file"] = f".draftgo/navigations/{fname}"
         index.append(meta)
-    write_index(out_dir, index)
+    write_index(out_dir, index, preserve_existing)
     return len(index)
 
 
-def explode_docs(out_dir, items):
+def explode_docs(out_dir, items, preserve_existing=False):
     index = []
     for art in items:
         aid = art.get("id", "x")
         slug = safe_slug(art.get("slug") or art.get("title") or aid)
-        fname = f"article_{aid}_{slug}.md"
+        fname = f"article_{aid}_{slug}.html"
         content = art.get("content") or ""
         (out_dir / fname).write_text(content, encoding="utf-8")
         meta = {k: v for k, v in art.items() if k != "content"}
         meta["content_file"] = f".draftgo/docs/articles/{fname}"
         index.append(meta)
-    write_index(out_dir, index)
+    write_index(out_dir, index, preserve_existing)
     return len(index)
 
 
-def explode_scripts(out_dir, items):
+def explode_scripts(out_dir, items, preserve_existing=False):
     index = []
     for sc in items:
         sid = sc.get("id", "x")
         slug = safe_slug(sc.get("slug") or sc.get("name") or sid)
-        ext = _LANG_EXT.get(str(sc.get("language") or "").lower(), "txt")
-        fname = f"script_{sid}_{slug}.{ext}"
-        (out_dir / fname).write_text(sc.get("code") or "", encoding="utf-8")
-        meta = {k: v for k, v in sc.items() if k != "code"}
+        fname = f"script_{sid}_{slug}.go"
+        # 列表接口不含 code；详情拉取失败时绝不能用空内容覆盖受管源码。
+        if "code" in sc:
+            (out_dir / fname).write_text(sc["code"], encoding="utf-8")
+        else:
+            print(f"  WARN script_id={sid} 未取得代码，保留本地文件: {fname}", file=sys.stderr)
+        meta = {k: v for k, v in sc.items() if k not in ("code", "language")}
         meta["code_file"] = f".draftgo/custom_scripts/{fname}"
         index.append(meta)
-    write_index(out_dir, index)
+    write_index(out_dir, index, preserve_existing)
     return len(index)
 
 
 # ── 按类型拉取 ──
 
+def select_items(items, ids, id_key, type_name):
+    """筛选指定目标，并拒绝用空结果覆盖现有索引。"""
+    if not ids:
+        return items
+    wanted = [str(value) for value in ids]
+    found = {str(item.get(id_key)) for item in items if item.get(id_key) is not None}
+    missing = [value for value in wanted if value not in found]
+    if missing:
+        print(
+            f"  ERR {type_name} 云端未返回目标 {id_key}: {', '.join(missing)}",
+            file=sys.stderr,
+        )
+        return None
+    return [item for item in items if str(item.get(id_key)) in wanted]
+
 def pull_pages(server, token, root, ids=None):
     raw = fetch(server, token, ENDPOINTS["pages"])
     if raw is None:
-        return 0
+        return None
     items = extract_items(raw)
-    if ids:
-        items = [p for p in items if str(p.get("id")) in ids]
+    items = select_items(items, ids, "id", "pages")
+    if items is None:
+        return None
+    if any(not isinstance(p.get("value"), dict) or not isinstance(p["value"].get("html"), str) for p in items):
+        print("  ERR pages 响应缺少 HTML，未写入本地缓存", file=sys.stderr)
+        return None
     out = root / ".draftgo/pages"
     out.mkdir(parents=True, exist_ok=True)
-    count = explode_pages(out, items)
+    count = explode_pages(out, items, preserve_existing=bool(ids))
     print(f"  OK pages ({count} items)")
     return count
 
@@ -215,13 +257,33 @@ def pull_pages(server, token, root, ids=None):
 def pull_navigations(server, token, root, ids=None):
     raw = fetch(server, token, ENDPOINTS["navigations"])
     if raw is None:
-        return 0
+        return None
     items = extract_items(raw)
-    if ids:
-        items = [n for n in items if str(n.get("id")) in ids]
+    items = select_items(items, ids, "id", "navigations")
+    if items is None:
+        return None
+    hydrated = []
+    for item in items:
+        code = item.get("code")
+        if not code:
+            print(
+                f"  ERR nav_id={item.get('id', '?')} 缺少 code，未写入本地缓存",
+                file=sys.stderr,
+            )
+            return None
+        detail = unwrap(fetch(
+            server, token, f"/api/navigations/{urllib.parse.quote(str(code), safe='')}"
+        ))
+        if not isinstance(detail, dict) or not isinstance(detail.get("html"), str):
+            print(
+                f"  ERR nav_id={item.get('id', '?')} 未取得 HTML，未写入本地缓存",
+                file=sys.stderr,
+            )
+            return None
+        hydrated.append({**item, **detail})
     out = root / ".draftgo/navigations"
     out.mkdir(parents=True, exist_ok=True)
-    count = explode_navigations(out, items)
+    count = explode_navigations(out, hydrated, preserve_existing=bool(ids))
     print(f"  OK navigations ({count} items)")
     return count
 
@@ -230,50 +292,40 @@ def pull_simple(server, token, root, type_name, ids=None):
     """拉取仅需 index.json 的简单类型（db_meta, aihub, roles, users, system_config）。"""
     raw = fetch(server, token, ENDPOINTS[type_name])
     if raw is None:
-        return 0
+        return None
     items = extract_items(raw)
-    if ids:
-        id_key = "config_key" if type_name == "system_config" else "id"
-        items = [it for it in items if str(it.get(id_key)) in ids]
+    id_key = "config_key" if type_name == "system_config" else "id"
+    items = select_items(items, ids, id_key, type_name)
+    if items is None:
+        return None
     out = root / f".draftgo/{type_name}"
-    write_index(out, items)
+    write_index(out, items, preserve_existing=bool(ids), id_key=id_key)
     print(f"  OK {type_name} ({len(items)} items)")
     return len(items)
 
 
-def pull_external_apis(server, token, root, ids=None):
-    raw = fetch(server, token, ENDPOINTS["external_apis"])
-    if raw is None:
-        return 0
-    items = extract_items(raw)
-    if ids:
-        items = [it for it in items if str(it.get("id")) in ids]
-    hydrated = []
-    for it in items:
-        api_id = it.get("id")
-        detail = unwrap(fetch(server, token, f"/api/external-apis/{api_id}")) if api_id else None
-        hydrated.append({**it, **detail} if isinstance(detail, dict) else it)
-    out = root / ".draftgo/external_apis"
-    write_index(out, hydrated)
-    print(f"  OK external_apis ({len(hydrated)} items)")
-    return len(hydrated)
-
-
 def pull_docs(server, token, root, ids=None):
     raw = fetch(server, token, ENDPOINTS["docs"])
     if raw is None:
-        return 0
+        return None
     items = extract_items(raw)
-    if ids:
-        items = [it for it in items if str(it.get("id")) in ids]
+    items = select_items(items, ids, "id", "docs")
+    if items is None:
+        return None
     hydrated = []
     for it in items:
         aid = it.get("id")
         detail = unwrap(fetch(server, token, f"/api/docs/articles/{aid}")) if aid else None
-        hydrated.append({**it, **detail} if isinstance(detail, dict) else it)
+        if not isinstance(detail, dict) or not isinstance(detail.get("content"), str):
+            print(
+                f"  ERR article_id={aid or '?'} 未取得正文，未写入本地缓存",
+                file=sys.stderr,
+            )
+            return None
+        hydrated.append({**it, **detail})
     out = root / ".draftgo/docs/articles"
     out.mkdir(parents=True, exist_ok=True)
-    count = explode_docs(out, hydrated)
+    count = explode_docs(out, hydrated, preserve_existing=bool(ids))
     print(f"  OK docs ({count} articles)")
     return count
 
@@ -281,12 +333,13 @@ def pull_docs(server, token, root, ids=None):
 def pull_doc_categories(server, token, root, ids=None):
     raw = fetch(server, token, ENDPOINTS["doc_categories"])
     if raw is None:
-        return 0
+        return None
     items = extract_items(raw)
-    if ids:
-        items = [it for it in items if str(it.get("id")) in ids]
+    items = select_items(items, ids, "id", "doc_categories")
+    if items is None:
+        return None
     out = root / ".draftgo/doc_categories"
-    write_index(out, items)
+    write_index(out, items, preserve_existing=bool(ids))
     print(f"  OK doc_categories ({len(items)} items)")
     return len(items)
 
@@ -294,14 +347,26 @@ def pull_doc_categories(server, token, root, ids=None):
 def pull_custom_scripts(server, token, root, ids=None):
     raw = fetch(server, token, ENDPOINTS["custom_scripts"])
     if raw is None:
-        return 0
+        return None
     items = extract_items(raw)
-    if ids:
-        items = [it for it in items if str(it.get("id")) in ids]
+    items = select_items(items, ids, "id", "custom_scripts")
+    if items is None:
+        return None
+    hydrated = []
+    for it in items:
+        script_id = it.get("id")
+        detail = unwrap(fetch(server, token, f"/api/scripts/{script_id}")) if script_id else None
+        if not isinstance(detail, dict) or not isinstance(detail.get("code"), str):
+            print(
+                f"  ERR script_id={script_id or '?'} 未取得代码，未写入本地缓存",
+                file=sys.stderr,
+            )
+            return None
+        hydrated.append({**it, **detail})
     out = root / ".draftgo/custom_scripts"
     out.mkdir(parents=True, exist_ok=True)
-    count = explode_scripts(out, items)
-    print(f"  OK custom_scripts ({count} scripts)")
+    count = explode_scripts(out, hydrated, preserve_existing=bool(ids))
+    print(f"  OK custom_scripts ({count} scripts, code from details)")
     return count
 
 
@@ -316,7 +381,6 @@ PULL_HANDLERS = {
     "roles":           lambda s, t, r, ids=None: pull_simple(s, t, r, "roles", ids),
     "users":           lambda s, t, r, ids=None: pull_simple(s, t, r, "users", ids),
     "system_config":   lambda s, t, r, ids=None: pull_simple(s, t, r, "system_config", ids),
-    "external_apis":   pull_external_apis,
     "docs":            pull_docs,
     "doc_categories":  pull_doc_categories,
     "custom_scripts":  pull_custom_scripts,
@@ -325,14 +389,15 @@ PULL_HANDLERS = {
 
 def pull_all(server, token, root):
     """拉取全部类型，供 init 复用。"""
-    pull_pages(server, token, root)
-    pull_navigations(server, token, root)
+    results = [pull_pages(server, token, root), pull_navigations(server, token, root)]
     for t in ("db_meta", "aihub", "roles", "users", "system_config"):
-        pull_simple(server, token, root, t)
-    pull_external_apis(server, token, root)
-    pull_doc_categories(server, token, root)
-    pull_docs(server, token, root)
-    pull_custom_scripts(server, token, root)
+        results.append(pull_simple(server, token, root, t))
+    results.extend([
+        pull_doc_categories(server, token, root),
+        pull_docs(server, token, root),
+        pull_custom_scripts(server, token, root),
+    ])
+    return None if any(result is None for result in results) else results
 
 
 def main():
@@ -345,7 +410,8 @@ def main():
     root = DEFAULT_ROOT
 
     if mode == "--all":
-        pull_all(server, token, root)
+        if pull_all(server, token, root) is None:
+            sys.exit(1)
         return
 
     if mode not in PULL_HANDLERS:
@@ -353,7 +419,8 @@ def main():
         sys.exit(1)
 
     ids = sys.argv[2:] or None
-    PULL_HANDLERS[mode](server, token, root, ids)
+    if PULL_HANDLERS[mode](server, token, root, ids) is None:
+        sys.exit(1)
 
 
 if __name__ == "__main__":