@shulex/cxclawctl 0.1.2 → 0.1.4

package/skills/kb-builder/scripts/import_qa.py

@@ -1,219 +0,0 @@
-#!/usr/bin/env python3
-"""
-Import qa_plan into Solvea knowledge base as QA entries.
-
-Two input modes:
-
-1. --qa-file <path>   already-merged qa_plan_<ts>.json
-2. --from-dir <dir>   merge all *.json files in <dir> (each is a QA list).
-                      Useful when the agent wrote multiple small batches into
-                      /workspace/tmp/kb-builder/qa_parts/ to avoid the LLM max_tokens
-                      truncation that hits any single huge tool_call. The
-                      merged plan is written to OUTPUTS_DIR/qa_plan_<ts>.json
-                      for traceability before upload.
-
-QA item format (each list entry):
-[
-  {
-    "dir_path": ["Product Introduction", "Smart Plugs", "Kasa EP25"],
-    "question": "Kasa EP25 Smart Plug — What is the Kasa EP25 and what does it do?",
-    "answer": "The Kasa EP25 is ...\n\n![Product image](https://cdn.example.com/...)"
-  }
-]
-
-Usage:
-  python3 import_qa.py --qa-file /workspace/outputs/kb-builder/qa_plan_<ts>.json
-  python3 import_qa.py --qa-file /workspace/outputs/kb-builder/qa_plan_<ts>.json --dry-run
-  python3 import_qa.py --from-dir /workspace/tmp/kb-builder/qa_parts/
-  python3 import_qa.py --from-dir /workspace/tmp/kb-builder/qa_parts/ --dry-run
-
-SolveaClient() picks token / project_id / env from environment variables
-(SOLVEA_TOKEN / SOLVEA_PROD_TOKEN / SOLVEA_PROJECT_ID / SOLVEA_ENV).
-"""
-from __future__ import annotations
-import argparse, json, os, sys
-sys.path.insert(0, os.path.join(os.environ.get("SOLVEA_LIB_DIR", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "__lib__"))), "skill-runtime"))
-from bootstrap import *
-from pathlib import Path
-
-def validate_qa_item(item: object, index: int, max_depth: int = 3) -> tuple[list[str], str, str]:
-    """Return normalized QA fields or raise a clear schema error."""
-    if not isinstance(item, dict):
-        raise ValueError(f"QA item #{index} must be an object, got {type(item).__name__}")
-
-    dir_path = item.get("dir_path")
-    question = item.get("question")
-    answer = item.get("answer")
-
-    if not isinstance(dir_path, list) or not dir_path:
-        raise ValueError(
-            f"QA item #{index} dir_path must be a non-empty list of folder names, "
-            f'e.g. ["Product Introduction", "Speakers", "B0117-03018-BK"]; '
-            f"got {dir_path!r}"
-        )
-    if not all(isinstance(name, str) and name.strip() for name in dir_path):
-        raise ValueError(f"QA item #{index} dir_path entries must be non-empty strings: {dir_path!r}")
-    if len(dir_path) > max_depth:
-        raise ValueError(f"QA item #{index} dir_path has {len(dir_path)} levels; maximum is {max_depth}")
-    if any("/" in name or "\\" in name for name in dir_path):
-        raise ValueError(
-            f"QA item #{index} dir_path entries must be folder names, not slash paths: {dir_path!r}"
-        )
-    if not isinstance(question, str) or not question.strip():
-        raise ValueError(f"QA item #{index} question must be a non-empty string")
-    if not isinstance(answer, str) or not answer.strip():
-        raise ValueError(f"QA item #{index} answer must be a non-empty markdown string")
-
-    return [name.strip() for name in dir_path], question.strip(), answer.strip()
-
-
-def validate_plan(plan: object, max_depth: int = 3) -> list[tuple[list[str], str, str]]:
-    if not isinstance(plan, list):
-        raise ValueError(f"QA plan must be a list, got {type(plan).__name__}")
-    return [validate_qa_item(item, i + 1, max_depth=max_depth) for i, item in enumerate(plan)]
-
-
-def flatten_folders(nodes: list, result: list | None = None) -> list:
-    """Recursively flatten a nested folder tree into a flat list."""
-    if result is None:
-        result = []
-    for node in nodes:
-        result.append(node)
-        children = node.get("children", [])
-        if children:
-            flatten_folders(children, result)
-    return result
-
-
-def get_or_create_dir(client: SolveaClient, path: list[str], dir_cache: dict) -> str | int:
-    """Walk dir_path, creating missing folders. Returns final directory ID."""
-    parent_id = 0
-    current_path: list[str] = []
-    for name in path:
-        current_path.append(name)
-        key = "/".join(current_path)
-        if key in dir_cache:
-            parent_id = dir_cache[key]
-            continue
-        # Search in existing folders (API returns {"list": [nested tree]})
-        folders = client.list_directories()
-        if isinstance(folders, list):
-            all_folders = flatten_folders(folders)
-        else:
-            raw = folders.get("list") or folders.get("records") or folders.get("data") or []
-            all_folders = flatten_folders(raw)
-        match = next(
-            (f for f in all_folders
-             if f.get("name") == name and (f.get("parentId") or 0) == parent_id),
-            None,
-        )
-        if match:
-            fid = match["id"]
-        else:
-            result = client.create_folder(name, parent_id=parent_id)
-            fid = result.get("id") if isinstance(result, dict) else None
-            if not fid:
-                print(f"  [warn] Could not create folder '{name}' under parentId={parent_id}: {result}", file=sys.stderr)
-                fid = parent_id  # fallback: use parent
-        dir_cache[key] = fid
-        parent_id = fid
-    return parent_id
-
-
-def merge_dir(src_dir: Path) -> tuple[list, Path]:
-    """Merge every *.json in src_dir (each must be a QA list) into one list,
-    then write the result to OUTPUTS_DIR/qa_plan_{ts}.json for traceability.
-
-    Files are merged in filename-sorted order so the result is deterministic.
-    Files whose top-level JSON isn't a list are skipped with a warning.
-    """
-    if not src_dir.exists() or not src_dir.is_dir():
-        raise FileNotFoundError(f"--from-dir not a directory: {src_dir}")
-
-    merged: list = []
-    parts = sorted(src_dir.glob("*.json"))
-    if not parts:
-        raise FileNotFoundError(f"No *.json found under {src_dir}")
-
-    for p in parts:
-        try:
-            data = json.loads(p.read_text(encoding="utf-8"))
-        except Exception as e:
-            print(f"  [warn] skip {p.name}: {e}", file=sys.stderr)
-            continue
-        if not isinstance(data, list):
-            print(f"  [warn] skip {p.name}: top-level is not a list", file=sys.stderr)
-            continue
-        print(f"  [merge] {p.name}: +{len(data)} QA")
-        merged.extend(data)
-
-    OUTPUTS_DIR.mkdir(parents=True, exist_ok=True)
-    out = OUTPUTS_DIR / f"qa_plan_{timestamp()}.json"
-    out.write_text(json.dumps(merged, ensure_ascii=False, indent=2), encoding="utf-8")
-    print(f"[merge] {len(merged)} QA total -> {out}")
-    return merged, out
-
-
-def run(qa_file: str | None, from_dir: str | None, dry_run: bool, max_depth: int = 3) -> None:
-    if from_dir:
-        plan, qa_file_path = merge_dir(Path(from_dir))
-    else:
-        qa_file_path = Path(qa_file)
-        plan = json.loads(qa_file_path.read_text(encoding="utf-8"))
-
-    normalized_plan = validate_plan(plan, max_depth=max_depth)
-
-    print(f"[import] {len(normalized_plan)} QA entries to import (dry_run={dry_run}) source={qa_file_path}")
-
-    dir_cache: dict[str, str | int] = {}
-    ok = 0
-    fail = 0
-    client = None
-    if not dry_run:
-        from client import SolveaClient
-        client = SolveaClient()
-
-    for i, (dir_path, question, answer) in enumerate(normalized_plan):
-        if dry_run:
-            print(f"  [{i+1}] DRY dir={dir_path}")
-            print(f"         Q: {question[:80]}")
-            print(f"         A: {answer[:80]}...")
-            continue
-
-        try:
-            assert client is not None
-            dir_id = get_or_create_dir(client, dir_path, dir_cache) if dir_path else ""
-            client.upload_qa([{
-                "title": question,
-                "content": answer,
-                "directoryId": dir_id,
-            }])
-            ok += 1
-            print(f"  [{i+1}/{len(normalized_plan)}] OK  {'/'.join(dir_path)} | {question[:60]}")
-        except Exception as e:
-            fail += 1
-            print(f"  [{i+1}/{len(normalized_plan)}] FAIL {question[:60]}: {e}")
-
-    if not dry_run:
-        print(f"\n[import] Done: {ok} OK, {fail} failed")
-
-
-def main():
-    ap = argparse.ArgumentParser()
-    src = ap.add_mutually_exclusive_group(required=True)
-    src.add_argument("--qa-file", dest="qa_file",
-                     help="Path to a pre-merged qa_plan_<ts>.json")
-    src.add_argument("--from-dir", dest="from_dir",
-                     help="Merge all *.json under this directory (each = QA list); "
-                          "merged result is written to OUTPUTS_DIR/qa_plan_<ts>.json")
-    ap.add_argument("--dry-run", action="store_true", dest="dry_run")
-    ap.add_argument("--max-depth", type=int, default=3,
-                    help="Maximum QA directory depth (default: 3; override only when explicitly requested)")
-    args = ap.parse_args()
-    if args.max_depth < 1:
-        ap.error("--max-depth must be positive")
-    run(args.qa_file, args.from_dir, args.dry_run, args.max_depth)
-
-
-if __name__ == "__main__":
-    main()

package/skills/kb-builder/scripts/import_source_documents.py

@@ -1,143 +0,0 @@
-#!/usr/bin/env python3
-"""Import URL/Word/PDF sources as DOCUMENT knowledge.
-
-Input is a JSON list of records:
-  {"source": "https://...", "dir_path": ["一级", "二级", "三级"]}
-  {"source": "/mounted/chat-files/manual.docx", "dir_path": ["..."]}
-
-The default is a dry-run.  URL duplicates in the input are reported as
-``skipped_duplicate``; the API response is retained because the backend may
-also deduplicate URLs already present in the knowledge base.
-"""
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import sys
-from pathlib import Path
-
-sys.path.insert(0, os.path.join(os.environ.get("SOLVEA_LIB_DIR", os.path.abspath(
-    os.path.join(os.path.dirname(__file__), "..", "..", "__lib__"))), "skill-runtime"))
-from bootstrap import *  # noqa: F403
-
-
-def validate_record(item, index, max_depth=3):
-    if not isinstance(item, dict) or not isinstance(item.get("source"), str) or not item["source"].strip():
-        raise ValueError(f"source document #{index} needs a non-empty source")
-    path = item.get("dir_path", [])
-    if not isinstance(path, list) or any(not isinstance(x, str) or not x.strip() for x in path):
-        raise ValueError(f"source document #{index} dir_path must be a list of non-empty names")
-    if len(path) > max_depth:
-        raise ValueError(f"source document #{index} dir_path has {len(path)} levels; maximum is {max_depth}")
-    if any("/" in x or "\\" in x for x in path):
-        raise ValueError(f"source document #{index} dir_path entries cannot contain slashes")
-    source = item["source"].strip()
-    is_url = source.lower().startswith(("http://", "https://"))
-    if not is_url and not Path(source).is_file():
-        raise ValueError(f"source document #{index} file not found: {source}")
-    if not is_url and Path(source).suffix.lower() not in {".doc", ".docx", ".pdf"}:
-        raise ValueError(f"source document #{index} supports .doc/.docx/.pdf, got {Path(source).suffix}")
-    return {"source": source, "dir_path": [x.strip() for x in path], "is_url": is_url}
-
-
-def validate_plan(data, max_depth=3):
-    if not isinstance(data, list):
-        raise ValueError("source document plan must be a JSON list")
-    normalized = [validate_record(item, i + 1, max_depth) for i, item in enumerate(data)]
-    seen = set()
-    for item in normalized:
-        if item["is_url"]:
-            item["status"] = "skipped_duplicate" if item["source"] in seen else "pending"
-            seen.add(item["source"])
-        else:
-            item["status"] = "pending"
-    return normalized
-
-
-def _flatten(nodes, out=None):
-    out = [] if out is None else out
-    for node in nodes:
-        out.append(node)
-        _flatten(node.get("children", []), out)
-    return out
-
-
-def resolve_dir(client, path, cache):
-    parent = 0
-    prefix = []
-    for name in path:
-        prefix.append(name)
-        key = "/".join(prefix)
-        if key in cache:
-            parent = cache[key]
-            continue
-        raw = client.list_directories()
-        nodes = raw if isinstance(raw, list) else raw.get("list", raw.get("records", raw.get("data", [])))
-        match = next((x for x in _flatten(nodes) if x.get("name") == name and (x.get("parentId") or 0) == parent), None)
-        if match:
-            parent = match["id"]
-        else:
-            result = client.create_folder(name, parent_id=parent)
-            parent = result.get("id") if isinstance(result, dict) else None
-            if not parent:
-                raise RuntimeError(f"could not create directory {key!r}: {result}")
-        cache[key] = parent
-    return parent
-
-
-def run(plan_file, dry_run=True, max_depth=3):
-    data = json.loads(Path(plan_file).read_text(encoding="utf-8"))
-    plan = validate_plan(data, max_depth)
-    print(f"[source-documents] {len(plan)} records (dry_run={dry_run})")
-    for item in plan:
-        print(f"  [{item['status']}] {item['source']} -> {'/'.join(item['dir_path']) or '(root)'}")
-    if dry_run:
-        return plan
-
-    from client import SolveaClient
-    client = SolveaClient()
-    cache = {}
-    results = []
-    for item in plan:
-        if item["status"] == "skipped_duplicate":
-            results.append({**item, "result": None})
-            continue
-        try:
-            directory_id = resolve_dir(client, item["dir_path"], cache)
-            if item["is_url"]:
-                documents = [{"filePath": item["source"], "docType": "TC", "fileType": "website"}]
-                response = client.create_documents(documents, directory_id=directory_id, task_type="WEB")
-            else:
-                key = client.upload_temp_file(item["source"], for_knowledge=True)
-                # The file-document endpoint derives the loader from the uploaded
-                # object key; unlike WEBSITE, it does not require fileType.
-                documents = [{"filePath": key, "docType": "TC"}]
-                response = client.create_documents(documents, directory_id=directory_id, task_type="FILE")
-            if not response:
-                raise RuntimeError("document API returned no created or refreshed record")
-            results.append({**item, "status": "submitted", "directory_id": directory_id, "result": response})
-            print(f"  [submitted] {item['source']}")
-        except Exception as exc:
-            results.append({**item, "status": "failed", "error": str(exc)})
-            print(f"  [failed] {item['source']}: {exc}", file=sys.stderr)
-    out = OUTPUTS_DIR / f"source_document_import_{timestamp()}.json"
-    out.parent.mkdir(parents=True, exist_ok=True)
-    out.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
-    print(f"[source-documents] result={out}")
-    return results
-
-
-def main():
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--plan-file", required=True)
-    ap.add_argument("--execute", action="store_true", help="Submit documents after explicit user confirmation")
-    ap.add_argument("--max-depth", type=int, default=3)
-    args = ap.parse_args()
-    if args.max_depth < 1:
-        ap.error("--max-depth must be positive")
-    run(args.plan_file, dry_run=not args.execute, max_depth=args.max_depth)
-
-
-if __name__ == "__main__":
-    main()