@shulex/cxclawctl 0.0.0-stage → 0.1.2

package/skills/kb-builder/scripts/import_qa.py

@@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+"""
+Import qa_plan into Solvea knowledge base as QA entries.
+
+Two input modes:
+
+1. --qa-file <path>   already-merged qa_plan_<ts>.json
+2. --from-dir <dir>   merge all *.json files in <dir> (each is a QA list).
+                      Useful when the agent wrote multiple small batches into
+                      /workspace/tmp/kb-builder/qa_parts/ to avoid the LLM max_tokens
+                      truncation that hits any single huge tool_call. The
+                      merged plan is written to OUTPUTS_DIR/qa_plan_<ts>.json
+                      for traceability before upload.
+
+QA item format (each list entry):
+[
+  {
+    "dir_path": ["Product Introduction", "Smart Plugs", "Kasa EP25"],
+    "question": "Kasa EP25 Smart Plug — What is the Kasa EP25 and what does it do?",
+    "answer": "The Kasa EP25 is ...\n\n![Product image](https://cdn.example.com/...)"
+  }
+]
+
+Usage:
+  python3 import_qa.py --qa-file /workspace/outputs/kb-builder/qa_plan_<ts>.json
+  python3 import_qa.py --qa-file /workspace/outputs/kb-builder/qa_plan_<ts>.json --dry-run
+  python3 import_qa.py --from-dir /workspace/tmp/kb-builder/qa_parts/
+  python3 import_qa.py --from-dir /workspace/tmp/kb-builder/qa_parts/ --dry-run
+
+SolveaClient() picks token / project_id / env from environment variables
+(SOLVEA_TOKEN / SOLVEA_PROD_TOKEN / SOLVEA_PROJECT_ID / SOLVEA_ENV).
+"""
+from __future__ import annotations
+import argparse, json, os, sys
+sys.path.insert(0, os.path.join(os.environ.get("SOLVEA_LIB_DIR", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "__lib__"))), "skill-runtime"))
+from bootstrap import *
+from pathlib import Path
+
+def validate_qa_item(item: object, index: int, max_depth: int = 3) -> tuple[list[str], str, str]:
+    """Return normalized QA fields or raise a clear schema error."""
+    if not isinstance(item, dict):
+        raise ValueError(f"QA item #{index} must be an object, got {type(item).__name__}")
+
+    dir_path = item.get("dir_path")
+    question = item.get("question")
+    answer = item.get("answer")
+
+    if not isinstance(dir_path, list) or not dir_path:
+        raise ValueError(
+            f"QA item #{index} dir_path must be a non-empty list of folder names, "
+            f'e.g. ["Product Introduction", "Speakers", "B0117-03018-BK"]; '
+            f"got {dir_path!r}"
+        )
+    if not all(isinstance(name, str) and name.strip() for name in dir_path):
+        raise ValueError(f"QA item #{index} dir_path entries must be non-empty strings: {dir_path!r}")
+    if len(dir_path) > max_depth:
+        raise ValueError(f"QA item #{index} dir_path has {len(dir_path)} levels; maximum is {max_depth}")
+    if any("/" in name or "\\" in name for name in dir_path):
+        raise ValueError(
+            f"QA item #{index} dir_path entries must be folder names, not slash paths: {dir_path!r}"
+        )
+    if not isinstance(question, str) or not question.strip():
+        raise ValueError(f"QA item #{index} question must be a non-empty string")
+    if not isinstance(answer, str) or not answer.strip():
+        raise ValueError(f"QA item #{index} answer must be a non-empty markdown string")
+
+    return [name.strip() for name in dir_path], question.strip(), answer.strip()
+
+
+def validate_plan(plan: object, max_depth: int = 3) -> list[tuple[list[str], str, str]]:
+    if not isinstance(plan, list):
+        raise ValueError(f"QA plan must be a list, got {type(plan).__name__}")
+    return [validate_qa_item(item, i + 1, max_depth=max_depth) for i, item in enumerate(plan)]
+
+
+def flatten_folders(nodes: list, result: list | None = None) -> list:
+    """Recursively flatten a nested folder tree into a flat list."""
+    if result is None:
+        result = []
+    for node in nodes:
+        result.append(node)
+        children = node.get("children", [])
+        if children:
+            flatten_folders(children, result)
+    return result
+
+
+def get_or_create_dir(client: SolveaClient, path: list[str], dir_cache: dict) -> str | int:
+    """Walk dir_path, creating missing folders. Returns final directory ID."""
+    parent_id = 0
+    current_path: list[str] = []
+    for name in path:
+        current_path.append(name)
+        key = "/".join(current_path)
+        if key in dir_cache:
+            parent_id = dir_cache[key]
+            continue
+        # Search in existing folders (API returns {"list": [nested tree]})
+        folders = client.list_directories()
+        if isinstance(folders, list):
+            all_folders = flatten_folders(folders)
+        else:
+            raw = folders.get("list") or folders.get("records") or folders.get("data") or []
+            all_folders = flatten_folders(raw)
+        match = next(
+            (f for f in all_folders
+             if f.get("name") == name and (f.get("parentId") or 0) == parent_id),
+            None,
+        )
+        if match:
+            fid = match["id"]
+        else:
+            result = client.create_folder(name, parent_id=parent_id)
+            fid = result.get("id") if isinstance(result, dict) else None
+            if not fid:
+                print(f"  [warn] Could not create folder '{name}' under parentId={parent_id}: {result}", file=sys.stderr)
+                fid = parent_id  # fallback: use parent
+        dir_cache[key] = fid
+        parent_id = fid
+    return parent_id
+
+
+def merge_dir(src_dir: Path) -> tuple[list, Path]:
+    """Merge every *.json in src_dir (each must be a QA list) into one list,
+    then write the result to OUTPUTS_DIR/qa_plan_{ts}.json for traceability.
+
+    Files are merged in filename-sorted order so the result is deterministic.
+    Files whose top-level JSON isn't a list are skipped with a warning.
+    """
+    if not src_dir.exists() or not src_dir.is_dir():
+        raise FileNotFoundError(f"--from-dir not a directory: {src_dir}")
+
+    merged: list = []
+    parts = sorted(src_dir.glob("*.json"))
+    if not parts:
+        raise FileNotFoundError(f"No *.json found under {src_dir}")
+
+    for p in parts:
+        try:
+            data = json.loads(p.read_text(encoding="utf-8"))
+        except Exception as e:
+            print(f"  [warn] skip {p.name}: {e}", file=sys.stderr)
+            continue
+        if not isinstance(data, list):
+            print(f"  [warn] skip {p.name}: top-level is not a list", file=sys.stderr)
+            continue
+        print(f"  [merge] {p.name}: +{len(data)} QA")
+        merged.extend(data)
+
+    OUTPUTS_DIR.mkdir(parents=True, exist_ok=True)
+    out = OUTPUTS_DIR / f"qa_plan_{timestamp()}.json"
+    out.write_text(json.dumps(merged, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(f"[merge] {len(merged)} QA total -> {out}")
+    return merged, out
+
+
+def run(qa_file: str | None, from_dir: str | None, dry_run: bool, max_depth: int = 3) -> None:
+    if from_dir:
+        plan, qa_file_path = merge_dir(Path(from_dir))
+    else:
+        qa_file_path = Path(qa_file)
+        plan = json.loads(qa_file_path.read_text(encoding="utf-8"))
+
+    normalized_plan = validate_plan(plan, max_depth=max_depth)
+
+    print(f"[import] {len(normalized_plan)} QA entries to import (dry_run={dry_run}) source={qa_file_path}")
+
+    dir_cache: dict[str, str | int] = {}
+    ok = 0
+    fail = 0
+    client = None
+    if not dry_run:
+        from client import SolveaClient
+        client = SolveaClient()
+
+    for i, (dir_path, question, answer) in enumerate(normalized_plan):
+        if dry_run:
+            print(f"  [{i+1}] DRY dir={dir_path}")
+            print(f"         Q: {question[:80]}")
+            print(f"         A: {answer[:80]}...")
+            continue
+
+        try:
+            assert client is not None
+            dir_id = get_or_create_dir(client, dir_path, dir_cache) if dir_path else ""
+            client.upload_qa([{
+                "title": question,
+                "content": answer,
+                "directoryId": dir_id,
+            }])
+            ok += 1
+            print(f"  [{i+1}/{len(normalized_plan)}] OK  {'/'.join(dir_path)} | {question[:60]}")
+        except Exception as e:
+            fail += 1
+            print(f"  [{i+1}/{len(normalized_plan)}] FAIL {question[:60]}: {e}")
+
+    if not dry_run:
+        print(f"\n[import] Done: {ok} OK, {fail} failed")
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    src = ap.add_mutually_exclusive_group(required=True)
+    src.add_argument("--qa-file", dest="qa_file",
+                     help="Path to a pre-merged qa_plan_<ts>.json")
+    src.add_argument("--from-dir", dest="from_dir",
+                     help="Merge all *.json under this directory (each = QA list); "
+                          "merged result is written to OUTPUTS_DIR/qa_plan_<ts>.json")
+    ap.add_argument("--dry-run", action="store_true", dest="dry_run")
+    ap.add_argument("--max-depth", type=int, default=3,
+                    help="Maximum QA directory depth (default: 3; override only when explicitly requested)")
+    args = ap.parse_args()
+    if args.max_depth < 1:
+        ap.error("--max-depth must be positive")
+    run(args.qa_file, args.from_dir, args.dry_run, args.max_depth)
+
+
+if __name__ == "__main__":
+    main()

package/skills/kb-builder/scripts/import_source_documents.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""Import URL/Word/PDF sources as DOCUMENT knowledge.
+
+Input is a JSON list of records:
+  {"source": "https://...", "dir_path": ["一级", "二级", "三级"]}
+  {"source": "/mounted/chat-files/manual.docx", "dir_path": ["..."]}
+
+The default is a dry-run.  URL duplicates in the input are reported as
+``skipped_duplicate``; the API response is retained because the backend may
+also deduplicate URLs already present in the knowledge base.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+sys.path.insert(0, os.path.join(os.environ.get("SOLVEA_LIB_DIR", os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "..", "..", "__lib__"))), "skill-runtime"))
+from bootstrap import *  # noqa: F403
+
+
+def validate_record(item, index, max_depth=3):
+    if not isinstance(item, dict) or not isinstance(item.get("source"), str) or not item["source"].strip():
+        raise ValueError(f"source document #{index} needs a non-empty source")
+    path = item.get("dir_path", [])
+    if not isinstance(path, list) or any(not isinstance(x, str) or not x.strip() for x in path):
+        raise ValueError(f"source document #{index} dir_path must be a list of non-empty names")
+    if len(path) > max_depth:
+        raise ValueError(f"source document #{index} dir_path has {len(path)} levels; maximum is {max_depth}")
+    if any("/" in x or "\\" in x for x in path):
+        raise ValueError(f"source document #{index} dir_path entries cannot contain slashes")
+    source = item["source"].strip()
+    is_url = source.lower().startswith(("http://", "https://"))
+    if not is_url and not Path(source).is_file():
+        raise ValueError(f"source document #{index} file not found: {source}")
+    if not is_url and Path(source).suffix.lower() not in {".doc", ".docx", ".pdf"}:
+        raise ValueError(f"source document #{index} supports .doc/.docx/.pdf, got {Path(source).suffix}")
+    return {"source": source, "dir_path": [x.strip() for x in path], "is_url": is_url}
+
+
+def validate_plan(data, max_depth=3):
+    if not isinstance(data, list):
+        raise ValueError("source document plan must be a JSON list")
+    normalized = [validate_record(item, i + 1, max_depth) for i, item in enumerate(data)]
+    seen = set()
+    for item in normalized:
+        if item["is_url"]:
+            item["status"] = "skipped_duplicate" if item["source"] in seen else "pending"
+            seen.add(item["source"])
+        else:
+            item["status"] = "pending"
+    return normalized
+
+
+def _flatten(nodes, out=None):
+    out = [] if out is None else out
+    for node in nodes:
+        out.append(node)
+        _flatten(node.get("children", []), out)
+    return out
+
+
+def resolve_dir(client, path, cache):
+    parent = 0
+    prefix = []
+    for name in path:
+        prefix.append(name)
+        key = "/".join(prefix)
+        if key in cache:
+            parent = cache[key]
+            continue
+        raw = client.list_directories()
+        nodes = raw if isinstance(raw, list) else raw.get("list", raw.get("records", raw.get("data", [])))
+        match = next((x for x in _flatten(nodes) if x.get("name") == name and (x.get("parentId") or 0) == parent), None)
+        if match:
+            parent = match["id"]
+        else:
+            result = client.create_folder(name, parent_id=parent)
+            parent = result.get("id") if isinstance(result, dict) else None
+            if not parent:
+                raise RuntimeError(f"could not create directory {key!r}: {result}")
+        cache[key] = parent
+    return parent
+
+
+def run(plan_file, dry_run=True, max_depth=3):
+    data = json.loads(Path(plan_file).read_text(encoding="utf-8"))
+    plan = validate_plan(data, max_depth)
+    print(f"[source-documents] {len(plan)} records (dry_run={dry_run})")
+    for item in plan:
+        print(f"  [{item['status']}] {item['source']} -> {'/'.join(item['dir_path']) or '(root)'}")
+    if dry_run:
+        return plan
+
+    from client import SolveaClient
+    client = SolveaClient()
+    cache = {}
+    results = []
+    for item in plan:
+        if item["status"] == "skipped_duplicate":
+            results.append({**item, "result": None})
+            continue
+        try:
+            directory_id = resolve_dir(client, item["dir_path"], cache)
+            if item["is_url"]:
+                documents = [{"filePath": item["source"], "docType": "TC", "fileType": "website"}]
+                response = client.create_documents(documents, directory_id=directory_id, task_type="WEB")
+            else:
+                key = client.upload_temp_file(item["source"], for_knowledge=True)
+                # The file-document endpoint derives the loader from the uploaded
+                # object key; unlike WEBSITE, it does not require fileType.
+                documents = [{"filePath": key, "docType": "TC"}]
+                response = client.create_documents(documents, directory_id=directory_id, task_type="FILE")
+            if not response:
+                raise RuntimeError("document API returned no created or refreshed record")
+            results.append({**item, "status": "submitted", "directory_id": directory_id, "result": response})
+            print(f"  [submitted] {item['source']}")
+        except Exception as exc:
+            results.append({**item, "status": "failed", "error": str(exc)})
+            print(f"  [failed] {item['source']}: {exc}", file=sys.stderr)
+    out = OUTPUTS_DIR / f"source_document_import_{timestamp()}.json"
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(f"[source-documents] result={out}")
+    return results
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--plan-file", required=True)
+    ap.add_argument("--execute", action="store_true", help="Submit documents after explicit user confirmation")
+    ap.add_argument("--max-depth", type=int, default=3)
+    args = ap.parse_args()
+    if args.max_depth < 1:
+        ap.error("--max-depth must be positive")
+    run(args.plan_file, dry_run=not args.execute, max_depth=args.max_depth)
+
+
+if __name__ == "__main__":
+    main()