@gobi-starweaver/ws-cli 0.3.1 → 0.3.2

package/dist/vendor/hj_corpus/hj_index_search.py

@@ -0,0 +1,359 @@
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sqlite3
+import time
+import unicodedata
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+STATION_ROOT = Path(__file__).resolve().parents[2]
+LOG_DIR = STATION_ROOT / "08_运行日志与资产清单"
+# T5：hj 走 HJ_CORPUS_DB env（T4 透传桥·main.cjs resolveHjCorpusDb() 三级解析回写 process.env）。
+# 本 DEFAULT_DB 仅 fallback·封装态不触（hj server 读 env 入口·封装态指向 resources/data/hj_4_jt_core_fts.sqlite）。
+# 开发态 marker 在 → env 未设时 fallback 到站根/05_...（逐字保值）。
+DEFAULT_DB = STATION_ROOT / "05_本地资料索引" / "translate" / "corpus" / "indexes" / "hj_4_jt_fts.sqlite"
+DEFAULT_JSON = LOG_DIR / "HJ古籍索引只读接口验证_20260609.json"
+DEFAULT_TXT = LOG_DIR / "HJ古籍索引只读接口验证_20260609.txt"
+
+SPACE_RE = re.compile(r"\s+")
+
+EVIDENCE_POLICY = {
+    "evidence_role": "candidate_source_and_context_only",
+    "evidence_note": "本地古籍 FTS 命中只作出处候选和上下文依据，不作页码依据。",
+    "page_basis": False,
+    "page_warning": "页码、版本、卷次细目仍须回到拟引用版本或权威数据库逐项核对。",
+}
+
+
+def now_iso() -> str:
+    return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+
+
+def compact_query(query: str) -> str:
+    return SPACE_RE.sub(" ", query.strip())
+
+
+def normalize_with_map(text: str) -> tuple[str, list[int]]:
+    chars: list[str] = []
+    offsets: list[int] = []
+    for idx, ch in enumerate(text):
+        normalized = unicodedata.normalize("NFKC", ch)
+        for item in normalized:
+            category = unicodedata.category(item)
+            if category[0] in {"C", "P", "S", "Z"}:
+                continue
+            chars.append(item.lower())
+            offsets.append(idx)
+    return "".join(chars), offsets
+
+
+def normalize_query(query: str) -> str:
+    normalized, _ = normalize_with_map(query)
+    return normalized
+
+
+def fts_phrase(query: str) -> str:
+    return '"' + query.replace('"', '""') + '"'
+
+
+def open_hj_index(db_path: Path = DEFAULT_DB) -> sqlite3.Connection:
+    if not db_path.exists():
+        raise FileNotFoundError(f"HJ index not found: {db_path}")
+    conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA query_only=ON")
+    conn.execute("PRAGMA temp_store=MEMORY")
+    conn.execute("PRAGMA cache_size=-262144")
+    return conn
+
+
+def inspect_hj_index(db_path: Path = DEFAULT_DB) -> dict[str, Any]:
+    conn = open_hj_index(db_path)
+    try:
+        metadata = {
+            row["key"]: row["value"]
+            for row in conn.execute("SELECT key, value FROM metadata ORDER BY key")
+        }
+        return {
+            "db_path": str(db_path),
+            "opened_read_only": True,
+            "query_only": int(conn.execute("PRAGMA query_only").fetchone()[0]),
+            "doc_count": int(conn.execute("SELECT COUNT(*) FROM docs").fetchone()[0]),
+            "metadata": metadata,
+            "evidence_policy": EVIDENCE_POLICY,
+        }
+    finally:
+        conn.close()
+
+
+def search_fts(
+    conn: sqlite3.Connection,
+    normalized_query: str,
+    limit: int,
+    book_prefix: str | None = None,
+) -> list[sqlite3.Row]:
+    params: list[object] = [fts_phrase(normalized_query)]
+    where = ["docs_fts MATCH ?"]
+    if book_prefix:
+        where.append("d.book_base LIKE ?")
+        params.append(book_prefix.rstrip("/\\") + "%")
+    params.append(limit)
+    sql = f"""
+        SELECT
+            d.docid, d.rel_path, d.book_base, d.article_no,
+            d.title, d.source, d.body, d.char_count,
+            bm25(docs_fts) AS rank
+        FROM docs_fts
+        JOIN docs AS d ON d.docid = docs_fts.rowid
+        WHERE {' AND '.join(where)}
+        ORDER BY rank
+        LIMIT ?
+    """
+    return list(conn.execute(sql, params))
+
+
+def search_like(
+    conn: sqlite3.Connection,
+    query: str,
+    limit: int,
+    book_prefix: str | None = None,
+) -> list[sqlite3.Row]:
+    pattern = f"%{query}%"
+    params: list[object] = [pattern, pattern, pattern]
+    where = ["(d.title LIKE ? OR d.source LIKE ? OR d.body LIKE ?)"]
+    if book_prefix:
+        where.append("d.book_base LIKE ?")
+        params.append(book_prefix.rstrip("/\\") + "%")
+    params.append(limit)
+    sql = f"""
+        SELECT
+            d.docid, d.rel_path, d.book_base, d.article_no,
+            d.title, d.source, d.body, d.char_count,
+            0.0 AS rank
+        FROM docs AS d
+        WHERE {' AND '.join(where)}
+        LIMIT ?
+    """
+    return list(conn.execute(sql, params))
+
+
+def make_context(
+    text: str,
+    raw_query: str,
+    normalized_query: str,
+    width: int,
+) -> tuple[str, int, str]:
+    pos = text.find(raw_query)
+    end_pos = pos + len(raw_query) if pos >= 0 else -1
+    if pos < 0:
+        normalized_text, offsets = normalize_with_map(text)
+        folded_pos = normalized_text.find(normalized_query)
+        if folded_pos >= 0 and folded_pos < len(offsets):
+            pos = offsets[folded_pos]
+            last = min(folded_pos + len(normalized_query) - 1, len(offsets) - 1)
+            end_pos = offsets[last] + 1
+    if pos < 0 or end_pos < 0:
+        return text[: width * 2].replace("\n", " "), -1, ""
+    start = max(0, pos - width)
+    end = min(len(text), end_pos + width)
+    prefix = "..." if start > 0 else ""
+    suffix = "..." if end < len(text) else ""
+    return (prefix + text[start:end] + suffix).replace("\n", " "), pos, text[pos:end_pos]
+
+
+def row_to_candidate(
+    row: sqlite3.Row,
+    raw_query: str,
+    normalized_query: str,
+    context_chars: int,
+) -> dict[str, Any]:
+    best_context = ""
+    position = -1
+    matched_text = ""
+    hit_column = "body"
+    for name in ("title", "source", "body"):
+        context, pos, match = make_context(row[name] or "", raw_query, normalized_query, context_chars)
+        if pos >= 0:
+            best_context = context
+            position = pos
+            matched_text = match
+            hit_column = name
+            break
+        if not best_context:
+            best_context = context
+
+    return {
+        "docid": row["docid"],
+        "book_base": row["book_base"],
+        "rel_path": row["rel_path"],
+        "article_no": row["article_no"],
+        "title": row["title"],
+        "source": row["source"],
+        "char_count": row["char_count"],
+        "rank": row["rank"],
+        "hit_column": hit_column,
+        "position": position,
+        "matched_text": matched_text,
+        "context": best_context,
+        "candidate_source": {
+            "book_base": row["book_base"],
+            "title": row["title"],
+            "article_no": row["article_no"],
+            "rel_path": row["rel_path"],
+        },
+        "evidence_role": EVIDENCE_POLICY["evidence_role"],
+        "page_basis": False,
+        "page_warning": EVIDENCE_POLICY["page_warning"],
+    }
+
+
+def search_hj_index(
+    query: str,
+    *,
+    db_path: Path = DEFAULT_DB,
+    limit: int = 10,
+    context_chars: int = 80,
+    book_prefix: str | None = None,
+    allow_short_scan: bool = False,
+) -> dict[str, Any]:
+    raw_query = compact_query(query)
+    normalized_query = normalize_query(raw_query)
+    if not normalized_query:
+        raise ValueError("query is empty after normalization")
+    if limit < 1:
+        raise ValueError("limit must be positive")
+    if context_chars < 0:
+        raise ValueError("context_chars must be non-negative")
+
+    warnings: list[str] = []
+    conn = open_hj_index(db_path)
+    try:
+        started = time.perf_counter()
+        if len(normalized_query) >= 3:
+            rows = search_fts(conn, normalized_query, limit, book_prefix)
+            mode = "fts5-trigram"
+        elif allow_short_scan:
+            rows = search_like(conn, raw_query, limit, book_prefix)
+            mode = "like-scan-short-query"
+            warnings.append("短词查询未走 trigram 倒排索引，结果仅用于人工初筛。")
+        else:
+            rows = []
+            mode = "short-query-blocked"
+            warnings.append("少于 3 个规范化字符的查询不会命中 trigram 索引；必要时启用 allow_short_scan。")
+        elapsed_ms = (time.perf_counter() - started) * 1000
+        candidates = [
+            row_to_candidate(row, raw_query, normalized_query, context_chars)
+            for row in rows
+        ]
+    finally:
+        conn.close()
+
+    return {
+        "query": raw_query,
+        "normalized_query": normalized_query,
+        "searched_at": now_iso(),
+        "db_path": str(db_path),
+        "mode": mode,
+        "limit": limit,
+        "context_chars": context_chars,
+        "book_prefix": book_prefix,
+        "hit_count": len(candidates),
+        "elapsed_ms": elapsed_ms,
+        "warnings": warnings,
+        "evidence_policy": EVIDENCE_POLICY,
+        "results": candidates,
+    }
+
+
+def write_json_report(report: dict[str, Any], out_path: Path) -> None:
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    out_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def write_text_report(report: dict[str, Any], out_path: Path) -> None:
+    lines = [
+        "HJ古籍索引只读检索接口验证",
+        f"查询：{report['query']}",
+        f"规范化查询：{report['normalized_query']}",
+        f"数据库：{report['db_path']}",
+        f"模式：{report['mode']}",
+        f"命中数：{report['hit_count']}",
+        f"耗时毫秒：{report['elapsed_ms']:.3f}",
+        "",
+        "证据规则：",
+        f"- {EVIDENCE_POLICY['evidence_note']}",
+        f"- {EVIDENCE_POLICY['page_warning']}",
+    ]
+    if report["warnings"]:
+        lines.extend(["", "警告："])
+        lines.extend(f"- {warning}" for warning in report["warnings"])
+    lines.extend(["", "命中候选："])
+    for idx, item in enumerate(report["results"], start=1):
+        lines.extend(
+            [
+                "",
+                f"{idx}. {item['title']}",
+                f"   book_base/article：{item['book_base']} / {item['article_no']}",
+                f"   source：{item['source']}",
+                f"   hit：{item['hit_column']}@{item['position']} matched={item['matched_text'] or '(未定位)'}",
+                f"   context：{item['context']}",
+                "   page_basis：False；页码须另核版本。",
+            ]
+        )
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    out_path.write_text("\n".join(lines), encoding="utf-8")
+
+
+def command_stats(args: argparse.Namespace) -> None:
+    print(json.dumps(inspect_hj_index(Path(args.db)), ensure_ascii=False, indent=2))
+
+
+def command_search(args: argparse.Namespace) -> None:
+    report = search_hj_index(
+        args.query,
+        db_path=Path(args.db),
+        limit=args.limit,
+        context_chars=args.context,
+        book_prefix=args.book,
+        allow_short_scan=args.allow_short_scan,
+    )
+    if args.out_json:
+        write_json_report(report, Path(args.out_json))
+    if args.out_txt:
+        write_text_report(report, Path(args.out_txt))
+    print(json.dumps(report, ensure_ascii=False, indent=2))
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="中古史学术IDE HJ古籍索引只读检索接口")
+    parser.add_argument("--db", default=str(DEFAULT_DB), help="hj_4_jt_fts.sqlite path")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    stats = subparsers.add_parser("stats", help="inspect index metadata using read-only access")
+    stats.set_defaults(func=command_stats)
+
+    search = subparsers.add_parser("search", help="search HJ index as source candidates")
+    search.add_argument("query")
+    search.add_argument("--limit", type=int, default=10)
+    search.add_argument("--context", type=int, default=80)
+    search.add_argument("--book", help="filter by book_base prefix, e.g. A01/A01_100")
+    search.add_argument("--allow-short-scan", action="store_true")
+    search.add_argument("--out-json", default="")
+    search.add_argument("--out-txt", default="")
+    search.set_defaults(func=command_search)
+    return parser
+
+
+def main() -> None:
+    parser = build_parser()
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

package/package.json

@@ -1,26 +1,46 @@
 {
   "name": "@gobi-starweaver/ws-cli",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "北辰计划·望舒 Polaris·WS — 面向史学与人文研究的终端智能体（WS CLI）",
   "license": "SEE LICENSE IN LICENSE",
   "type": "module",
-  "main": "cli.js",
+  "homepage": "https://gobi-starweaver.cn",
   "bin": {
-    "ws": "cli.js"
+    "ws": "./dist/cli.js"
   },
+  "main": "./dist/cli.js",
   "files": [
-    "cli.js",
-    "chunks/**",
-    "templates/**",
-    "bundled/**",
-    "assets/**",
-    "mcp/**",
+    "dist/cli.js",
+    "dist/chunks/**",
+    "dist/templates/**",
+    "dist/bundled/**",
+    "dist/mcp/**",
+    "dist/vendor/**",
+    "dist/assets/**",
     "README.md",
-    "README-en.md",
     "LICENSE"
   ],
   "engines": {
     "node": ">=22"
   },
-  "dependencies": {}
+  "scripts": {
+    "typecheck": "tsc -p ./ --noEmit",
+    "bundle": "node ../../scripts/esbuild.config.js",
+    "build": "npm run typecheck && npm run bundle && node ../../scripts/copy-bundle-assets.js && node -e \"import('node:fs').then(f => f.chmodSync('dist/cli.js', 0o755))\"",
+    "prepublishOnly": "npm run build",
+    "format": "prettier --write .",
+    "test": "node ../../scripts/scan-theme-drift.mjs && node src/tests/run-tests.mjs"
+  },
+  "dependencies": {
+    "@gobi-starweaver/ws-core": "file:../core",
+    "chalk": "^5.6.2",
+    "ignore": "^7.0.5",
+    "ink": "^7.0.4",
+    "react": "^19.2.5",
+    "read-package-up": "^12.0.0",
+    "yargs": "^18.0.0"
+  },
+  "devDependencies": {
+    "@types/yargs": "^17.0.35"
+  }
 }

package/README-en.md

@@ -1,52 +0,0 @@
-<div align="center">
-
-<h1>北辰计划·望舒 Polaris·WS</h1>
-
-**A soft lamp for all who sail the sea of learning by night.**
-
-[中文](README.md) · English
-
-</div>
-
-望舒 (WS) is a terminal agent from the Beichen Project （北辰计划）, built for historical and humanistic research: in your command line, it inquires, reads, writes and revises with you through conversation — like a colleague sitting across your desk.
-
-The Beichen Project is the humanities-agent series of Gobi StarWeaver (GSH), and holds that an agent for the humanities should carry both human warmth and scholarly rigor.
-
-## Installation
-
-One command installs everything (auto-detects your OS and architecture, installs all dependencies, and talks to exactly one domain — our official site):
-
-**Windows** (paste in PowerShell, press Enter):
-
-```powershell
-irm https://gobi-starweaver.cn/dist/install.ps1 | iex
-```
-
-**macOS / Linux** (paste in a terminal, press Enter):
-
-```bash
-curl -fsSL https://gobi-starweaver.cn/dist/install.sh | bash
-```
-
-Then **close and reopen your terminal**, and run `ws` in any project directory. On first launch, sign in with your Beichen Project account to get started (subscription verification happens automatically with login).
-
-## What it does
-
-- **Conversation is the work**: state your topic and materials; 望舒 searches, reads, writes and revises with you, step by step.
-- **Historical sources at hand**: built-in Hanji corpus retrieval — dynastic histories, veritable records and collected works, with citations traceable to juan and page.
-- **Thinking with a breath**: while waiting, you can see footprints like "consulting the Zizhi Tongjian…"; the moment the answer arrives, all is calm again.
-- **Skills that grow**: the Skills and MCP plugin system lets 望舒 grow with your research habits.
-- **Sign in and rest easy**: once logged in and your subscription verified, every request is relayed securely through Beichen servers — no keys ever live on your machine.
-
-## Versioning
-
-望舒 carries two version numbers, each governing its own domain:
-
-- **Mind version v0.9**: the settled version of 望舒's persona, mind and writing style — how it thinks and how it writes.
-- **Release version 0.3.0**: the semantic version of the CLI software itself — installation, upgrades and issue reports refer to this one.
-
-## License and origin
-
-This software is **proprietary** — copyright Gobi StarWeaver Digital History Lab (GSH), all rights reserved; the portions originating from [DeepCode](https://github.com/lessweb/deepcode-cli) remain under the MIT License as required — see [LICENSE](LICENSE). 望舒 CLI is continuously developed by GSH.
-
-《史学智能生态系统》魏柯帆团队原创，知识产权受保护。