@shulex/cxclawctl 0.0.0-stage → 0.1.2

package/skills/kb-builder/scripts/batch_extract_ticket_qa.py

@@ -0,0 +1,333 @@
+#!/usr/bin/env python3
+"""
+Batch wrapper for extract_ticket_qa.py.
+
+Reads one xlsx, processes it page by page, stores page artifacts in state, then
+merges and de-duplicates into a single output JSON. It never uploads knowledge.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, os.path.join(os.environ.get("SOLVEA_LIB_DIR", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "__lib__"))), "skill-runtime"))
+from bootstrap import *
+
+from openai import OpenAI
+import pandas as pd
+
+from extract_ticket_qa import (
+    MAX_BATCH_SIZE,
+    MAX_PAGE_SIZE,
+    MAX_WORKERS,
+    build_thread,
+    chunked,
+    extract_batch,
+    resolve_columns,
+    select_rows,
+)
+
+
+STATE_ROOT = STATE_DIR / "ticket_extract"
+PAGES_DIR = STATE_ROOT / "pages"
+PROGRESS_FILE = STATE_ROOT / "progress.json"
+
+PRE_FILTER_PATTERNS = [
+    r"\btracking\b",
+    r"\bdelivery\b",
+    r"\bshipping\b",
+    r"\border status\b",
+    r"\brefund\b",
+    r"\breturn\b",
+    r"\bexchange\b",
+    r"\bpayment\b",
+    r"\binvoice\b",
+    r"\baccount\b",
+    r"\bplease send (a )?(photo|picture|image|receipt|order number)",
+    r"\bcontact (customer )?(support|service)\b",
+    r"\bsubmit (a )?(ticket|case)\b",
+    r"\bbroken on arrival\b",
+    r"\bdamaged on arrival\b",
+]
+
+POST_FILTER_PATTERNS = [
+    r"contact (customer )?(support|service)",
+    r"submit (a )?(ticket|case)",
+    r"send (a )?(photo|picture|image|receipt|order number)",
+    r"we will (issue|send|process)",
+]
+
+
+def normalize_question(question: str) -> str:
+    text = question.lower()
+    text = re.sub(r"[^a-z0-9]+", " ", text)
+    return re.sub(r"\s+", " ", text).strip()
+
+
+def should_pre_filter(thread: str) -> bool:
+    lowered = thread.lower()
+    return any(re.search(pattern, lowered) for pattern in PRE_FILTER_PATTERNS)
+
+
+def should_post_filter(item: dict) -> bool:
+    answer = str(item.get("answer", "")).lower()
+    return any(re.search(pattern, answer) for pattern in POST_FILTER_PATTERNS)
+
+
+def merge_and_dedupe(page_files: list[Path]) -> tuple[list[dict], dict]:
+    merged: list[dict] = []
+    seen: set[str] = set()
+    stats = {"input": 0, "post_filtered": 0, "deduped": 0}
+
+    for page_file in page_files:
+        data = json.loads(page_file.read_text(encoding="utf-8"))
+        for item in data.get("qa_pairs", []):
+            stats["input"] += 1
+            if should_post_filter(item):
+                stats["post_filtered"] += 1
+                continue
+            key = normalize_question(str(item.get("question", "")))
+            if not key:
+                continue
+            if key in seen:
+                stats["deduped"] += 1
+                continue
+            seen.add(key)
+            merged.append(item)
+
+    merged.sort(key=lambda item: (item.get("source_page") or 0, item.get("source_row") or 0))
+    stats["output"] = len(merged)
+    return merged, stats
+
+
+def _page_file(page: int) -> Path:
+    return PAGES_DIR / f"page_{page:03d}.json"
+
+
+def _write_json(path: Path, data: dict | list) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def process_page(
+    *,
+    df: pd.DataFrame,
+    page: int,
+    page_size: int,
+    brand: str,
+    messages_col: str | None,
+    title_col: str | None,
+    reply_col: str | None,
+    batch_size: int,
+    workers: int,
+    client,
+) -> dict:
+    page_df, window = select_rows(df, page=page, page_size=page_size)
+    columns = resolve_columns(page_df, messages_col=messages_col, title_col=title_col, reply_col=reply_col)
+
+    items: list[dict] = []
+    pre_filtered = 0
+    empty = 0
+    for source_row, (_, row) in enumerate(page_df.iterrows(), start=window.start_row):
+        thread = build_thread(row, columns)
+        if not thread:
+            empty += 1
+            continue
+        if should_pre_filter(thread):
+            pre_filtered += 1
+            continue
+        items.append({"source_row": source_row, "thread": thread})
+
+    qa_pairs: list[dict] = []
+    skipped = 0
+    failed = 0
+    batches = list(chunked(items, batch_size))
+
+    # Keep dependency local so tests can import this module without invoking model code.
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        futures = {pool.submit(extract_batch, client, batch, brand): len(batch) for batch in batches}
+        for future in as_completed(futures):
+            results = future.result()
+            if not results:
+                failed += futures[future]
+                continue
+            for result in results:
+                if result.get("skip_reason"):
+                    skipped += 1
+                    continue
+                question = str(result.get("question", "")).strip()
+                answer = str(result.get("answer", "")).strip()
+                if not question or not answer or len(answer) <= 20:
+                    failed += 1
+                    continue
+                qa_pairs.append({
+                    "source_page": page,
+                    "source_row": result.get("source_row"),
+                    "topic": result.get("topic", "troubleshooting"),
+                    "question": question,
+                    "answer": answer,
+                    "dir_path": [],
+                })
+
+    qa_pairs.sort(key=lambda item: item.get("source_row") or 0)
+    return {
+        "page": page,
+        "rows": {"start": window.start_row, "end": window.end_row, "total": window.total_rows},
+        "columns": {
+            "mode": columns.mode,
+            "messages_col": columns.messages_col,
+            "title_col": columns.title_col,
+            "reply_col": columns.reply_col,
+            "fallback_col": columns.fallback_col,
+        },
+        "stats": {
+            "page_rows": len(page_df),
+            "valid_threads": len(items),
+            "pre_filtered": pre_filtered,
+            "empty": empty,
+            "model_calls": len(batches),
+            "extracted": len(qa_pairs),
+            "skipped": skipped,
+            "failed": failed,
+        },
+        "qa_pairs": qa_pairs,
+    }
+
+
+def run(args) -> None:
+    if args.page_size > MAX_PAGE_SIZE:
+        raise ValueError(f"page-size {args.page_size} exceeds maximum {MAX_PAGE_SIZE}")
+    if args.batch_size > MAX_BATCH_SIZE:
+        raise ValueError(f"batch-size {args.batch_size} exceeds maximum {MAX_BATCH_SIZE}")
+    if args.workers < 1 or args.workers > MAX_WORKERS:
+        raise ValueError(f"workers must be 1-{MAX_WORKERS}, got {args.workers}")
+
+    print(f"[batch_extract] Reading {args.xlsx}")
+    df = pd.read_excel(args.xlsx)
+    total_rows = len(df)
+    total_pages = math.ceil(total_rows / args.page_size) if total_rows else 0
+    print(f"[batch_extract] rows={total_rows} pages={total_pages} page-size={args.page_size}")
+
+    STATE_ROOT.mkdir(parents=True, exist_ok=True)
+    PAGES_DIR.mkdir(parents=True, exist_ok=True)
+
+    # Sandbox injects OPENAI_BASE_URL + dummy OPENAI_API_KEY for the internal proxy.
+    client = OpenAI()
+    done_pages: list[int] = []
+    failed_pages: list[int] = []
+
+    for page in range(1, total_pages + 1):
+        out_file = _page_file(page)
+        if args.resume and out_file.exists():
+            print(f"[batch_extract] page {page}/{total_pages} exists, skip")
+            done_pages.append(page)
+            continue
+        try:
+            result = process_page(
+                df=df,
+                page=page,
+                page_size=args.page_size,
+                brand=args.brand,
+                messages_col=args.messages_col,
+                title_col=args.title_col,
+                reply_col=args.reply_col,
+                batch_size=args.batch_size,
+                workers=args.workers,
+                client=client,
+            )
+            _write_json(out_file, result)
+            done_pages.append(page)
+            print(f"[batch_extract] page {page}/{total_pages} done -> {out_file}")
+        except Exception as exc:
+            failed_pages.append(page)
+            print(f"[batch_extract] page {page}/{total_pages} failed: {exc}", file=sys.stderr)
+
+        _write_json(PROGRESS_FILE, {
+            "status": "running" if failed_pages else "extracting",
+            "xlsx": args.xlsx,
+            "brand": args.brand,
+            "total_rows": total_rows,
+            "total_pages": total_pages,
+            "page_size": args.page_size,
+            "batch_size": args.batch_size,
+            "workers": args.workers,
+            "messages_col": args.messages_col,
+            "title_col": args.title_col,
+            "reply_col": args.reply_col,
+            "done_pages": done_pages,
+            "failed_pages": failed_pages,
+        })
+
+    if failed_pages:
+        _write_json(PROGRESS_FILE, {
+            "status": "failed",
+            "xlsx": args.xlsx,
+            "brand": args.brand,
+            "total_rows": total_rows,
+            "total_pages": total_pages,
+            "done_pages": done_pages,
+            "failed_pages": failed_pages,
+        })
+        raise RuntimeError(f"QA extraction failed pages={failed_pages}; no merged output was generated")
+
+    page_files = [_page_file(page) for page in range(1, total_pages + 1) if _page_file(page).exists()]
+    merged, merge_stats = merge_and_dedupe(page_files)
+
+    OUTPUTS_DIR.mkdir(parents=True, exist_ok=True)
+    out_path = OUTPUTS_DIR / f"ticket_qa_merged_{timestamp()}.json"
+    _write_json(out_path, merged)
+
+    final_progress = {
+        "status": "completed_with_errors" if failed_pages else "completed",
+        "xlsx": args.xlsx,
+        "brand": args.brand,
+        "total_rows": total_rows,
+        "total_pages": total_pages,
+        "done_pages": done_pages,
+        "failed_pages": failed_pages,
+        "page_files": [str(path) for path in page_files],
+        "merged_output": str(out_path),
+        "merge_stats": merge_stats,
+    }
+    _write_json(PROGRESS_FILE, final_progress)
+
+    print(f"[batch_extract] merged -> {out_path}")
+    print(
+        "[batch_extract] stats "
+        f"input={merge_stats['input']} post_filtered={merge_stats['post_filtered']} "
+        f"deduped={merge_stats['deduped']} output={merge_stats['output']}"
+    )
+    print("[batch_extract] Not uploaded. Review merged output, then run import_qa.py dry-run after user confirmation.")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--xlsx", required=True)
+    parser.add_argument("--brand", required=True)
+    parser.add_argument("--messages-col", default=None)
+    parser.add_argument("--title-col", default=None)
+    parser.add_argument("--reply-col", default=None)
+    parser.add_argument("--page-size", type=int, default=100, dest="page_size")
+    parser.add_argument("--batch-size", type=int, default=5, dest="batch_size")
+    parser.add_argument("--workers", type=int, default=3)
+    parser.add_argument("--resume", action="store_true")
+    args = parser.parse_args()
+    try:
+        run(args)
+    except ValueError as exc:
+        print(f"[error] {exc}", file=sys.stderr)
+        sys.exit(2)
+    except RuntimeError as exc:
+        print(f"[error] {exc}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/skills/kb-builder/scripts/build_directory_plan.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""Build a KB directory plan from crawl_result JSON."""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+from pathlib import Path
+from urllib.parse import urlparse
+
+sys.path.insert(0, os.path.join(os.environ.get("SOLVEA_LIB_DIR", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "__lib__"))), "skill-runtime"))
+from bootstrap import *  # noqa: F401,F403
+
+
+def resolve_workspace_path(path: str | Path) -> Path:
+    p = Path(path)
+    if p.is_absolute() and str(p).startswith("/workspace"):
+        return WORKSPACE / p.relative_to("/workspace")
+    return p
+
+
+def virtual_workspace_path(path: Path) -> str:
+    try:
+        rel = path.resolve().relative_to(WORKSPACE.resolve())
+        return str(Path("/workspace") / rel)
+    except Exception:
+        return str(path)
+
+
+def clean_name(value: str, fallback: str) -> str:
+    value = re.sub(r"\s+", " ", (value or "")).strip()
+    value = re.sub(r"[<>\"{}|\\^`]", "", value).strip()
+    return value[:80] or fallback
+
+
+def policy_name(page: dict) -> str:
+    hay = f"{page.get('url', '')} {page.get('title', '')}".lower()
+    if "warranty" in hay or "guarantee" in hay:
+        return "Warranty Policy"
+    if "return" in hay or "refund" in hay:
+        return "Return Policy"
+    if "shipping" in hay or "delivery" in hay:
+        return "Shipping Policy"
+    if "privacy" in hay:
+        return "Privacy Policy"
+    if "terms" in hay:
+        return "Terms Policy"
+    return "Brand Policy"
+
+
+def infer_product_group(page: dict) -> str:
+    path = urlparse(page.get("url", "")).path.strip("/")
+    parts = [p for p in path.split("/") if p]
+    for marker in ("collections", "collection", "category", "categories"):
+        if marker in parts:
+            idx = parts.index(marker)
+            if idx + 1 < len(parts):
+                return clean_name(parts[idx + 1].replace("-", " ").title(), "Products")
+    return "Products"
+
+
+def build_tree(crawl: dict) -> list[dict]:
+    pages = crawl.get("pages") or []
+    products = [p for p in pages if p.get("type") == "product"]
+    policies = [p for p in pages if p.get("type") == "policy"]
+    help_pages = [p for p in pages if p.get("type") == "help_center"]
+
+    product_groups: dict[str, list[dict]] = {}
+    for page in products:
+        product_groups.setdefault(infer_product_group(page), []).append(page)
+
+    product_children = []
+    for group, group_pages in sorted(product_groups.items()):
+        product_children.append({
+            "name": group,
+            "children": [
+                {"name": clean_name(p.get("title") or urlparse(p.get("url", "")).path.rsplit("/", 1)[-1], "Product")}
+                for p in group_pages[:80]
+            ],
+        })
+
+    policy_names = sorted({policy_name(p) for p in policies})
+    tree = [
+        {
+            "name": "Product Introduction",
+            "children": product_children or [{"name": "Products", "children": []}],
+        },
+        {
+            "name": "Brand Policy",
+            "children": [{"name": name} for name in (policy_names or ["Brand Policy"])],
+        },
+    ]
+
+    if help_pages:
+        tree.append({
+            "name": "Help Center",
+            "children": [{"name": "Help Articles"}],
+        })
+
+    tree.append({"name": "Pending Assignment", "children": []})
+    return tree
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Build kb-builder directory plan from crawl result")
+    parser.add_argument("--crawl-result", required=True)
+    parser.add_argument("--output", default="")
+    parser.add_argument("--write-confirmed-tree", action="store_true",
+                        help="Also write /workspace/state/kb-builder/confirmed_tree.json after user confirmation")
+    args = parser.parse_args()
+
+    crawl_path = resolve_workspace_path(args.crawl_result)
+    crawl = json.loads(crawl_path.read_text(encoding="utf-8"))
+    tree = build_tree(crawl)
+    plan = {
+        "base_url": crawl.get("base_url", ""),
+        "source_crawl_result": args.crawl_result,
+        "tree": tree,
+        "stats": {
+            "pages": len(crawl.get("pages") or []),
+            "product": sum(1 for p in crawl.get("pages") or [] if p.get("type") == "product"),
+            "policy": sum(1 for p in crawl.get("pages") or [] if p.get("type") == "policy"),
+            "help_center": sum(1 for p in crawl.get("pages") or [] if p.get("type") == "help_center"),
+        },
+    }
+
+    OUTPUTS_DIR.mkdir(parents=True, exist_ok=True)
+    out = resolve_workspace_path(args.output) if args.output else OUTPUTS_DIR / f"directory_plan_{timestamp()}.json"
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(plan, ensure_ascii=False, indent=2), encoding="utf-8")
+
+    if args.write_confirmed_tree:
+        confirmed = STATE_DIR / "confirmed_tree.json"
+        confirmed.parent.mkdir(parents=True, exist_ok=True)
+        confirmed.write_text(json.dumps(tree, ensure_ascii=False, indent=2), encoding="utf-8")
+        print(f"[build_directory_plan] confirmed_tree={virtual_workspace_path(confirmed)}")
+
+    print(f"[build_directory_plan] output={virtual_workspace_path(out)} root_nodes={len(tree)} pages={plan['stats']['pages']}")
+
+
+if __name__ == "__main__":
+    main()