@nomadamas/k-skill 0.1.0

package/skills/joseon-sillok-search/scripts/sillok_search.py

@@ -0,0 +1,552 @@
+from __future__ import annotations
+
+import argparse
+import html
+import json
+import math
+import re
+import ssl
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import asdict, dataclass
+from http.cookiejar import CookieJar
+from typing import Any, Iterable
+
+try:
+    import requests
+except ImportError:  # pragma: no cover - optional runtime dependency
+    requests = None
+
+
+BASE_URL = "https://sillok.history.go.kr"
+SEARCH_URL = f"{BASE_URL}/search/searchResultList.do"
+DETAIL_URL_TEMPLATE = f"{BASE_URL}/id/{{article_id}}"
+DEFAULT_TIMEOUT = 30
+DEFAULT_LIMIT = 5
+MAX_PAGES = 20
+
+DEFAULT_HEADERS = {
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7",
+    "Content-Type": "application/x-www-form-urlencoded",
+    "Origin": BASE_URL,
+    "Referer": f"{BASE_URL}/main/main.do",
+    "User-Agent": (
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"
+    ),
+}
+
+COMMENT_PATTERN = re.compile(r"<!--.*?-->", re.S)
+BR_PATTERN = re.compile(r"<br\s*/?>", re.I)
+TAG_PATTERN = re.compile(r"<[^>]+>")
+WHITESPACE_PATTERN = re.compile(r"\s+")
+DIGITS_PATTERN = re.compile(r"\d+")
+DETAIL_FOOTER_PATTERN = re.compile(
+    r"\s*(?:[【〖](?:태백산사고본|국편영인본|분류)[】〗]|ⓒ\s*세종대왕기념사업회).*",
+    re.S,
+)
+
+CATEGORY_PATTERN = re.compile(
+    r"<a\s+href=\"javascript:searchCategory\('([^']*)'\);\"\s+class=\"cate-item[^\"]*\">(.*?)</a>",
+    re.S,
+)
+RESULT_PATTERN = re.compile(
+    r"<div\s+class=\"result-box\">.*?"
+    r"<a\s+href=\"javascript:goView\('([^']+)',\s*\d+\);\"\s+class=\"subject\">(.*?)</a>\s*"
+    r"<p\s+class=\"text\">(.*?)</p>",
+    re.S,
+)
+TITLE_HEAD_PATTERN = re.compile(
+    r"<div\s+class=\"title\">\s*<p\s+class=\"date\">(.*?)</p>\s*<h3>(.*?)</h3>",
+    re.S,
+)
+LEFT_VIEW_PATTERN = re.compile(
+    r"<div\s+class=\"view-item\s+left\">.*?<div\s+class=\"view-text\">(.*?)</div>",
+    re.S,
+)
+RIGHT_VIEW_PATTERN = re.compile(
+    r"<div\s+class=\"view-item\s+right\">.*?<div\s+class=\"view-text\">(.*?)</div>",
+    re.S,
+)
+CLASSIFICATION_PATTERN = re.compile(r"<li\s+class=\"view_font02\">\s*[【〖]분류[】〗]\s*(.*?)</li>", re.S)
+TOTAL_COUNT_PATTERN = re.compile(r"검색결과\s*<strong>(\d+)</strong>개")
+HIDDEN_VALUE_TEMPLATE = '<input[^>]+id="{field}"[^>]+value="([^"]*)"'
+
+KING_ACCESSION_YEARS = {
+    "태조": 1392,
+    "정종": 1399,
+    "태종": 1401,
+    "세종": 1418,
+    "문종": 1450,
+    "단종": 1452,
+    "세조": 1455,
+    "예종": 1468,
+    "성종": 1469,
+    "연산군": 1494,
+    "중종": 1506,
+    "인종": 1545,
+    "명종": 1545,
+    "선조": 1567,
+    "선조수정": 1567,
+    "광해군중초본": 1608,
+    "광해군정초본": 1608,
+    "인조": 1623,
+    "효종": 1649,
+    "현종": 1659,
+    "현종개수": 1659,
+    "숙종": 1674,
+    "숙종보궐정오": 1674,
+    "경종": 1720,
+    "경종수정": 1720,
+    "영조": 1724,
+    "정조": 1776,
+    "순조": 1800,
+    "헌종": 1834,
+    "철종": 1849,
+    "고종": 1863,
+    "순종": 1907,
+    "순종부록": 1910,
+}
+
+KING_ALIASES = {
+    "태조실록": "태조",
+    "정종실록": "정종",
+    "태종실록": "태종",
+    "세종실록": "세종",
+    "문종실록": "문종",
+    "단종실록": "단종",
+    "세조실록": "세조",
+    "예종실록": "예종",
+    "성종실록": "성종",
+    "연산군일기": "연산군",
+    "중종실록": "중종",
+    "인종실록": "인종",
+    "명종실록": "명종",
+    "선조실록": "선조",
+    "선조수정실록": "선조수정",
+    "광해군중초본": "광해군중초본",
+    "광해군정초본": "광해군정초본",
+    "인조실록": "인조",
+    "효종실록": "효종",
+    "현종실록": "현종",
+    "현종개수실록": "현종개수",
+    "숙종실록": "숙종",
+    "숙종보궐정오": "숙종보궐정오",
+    "경종실록": "경종",
+    "경종수정실록": "경종수정",
+    "영조실록": "영조",
+    "정조실록": "정조",
+    "순조실록": "순조",
+    "헌종실록": "헌종",
+    "철종실록": "철종",
+    "고종실록": "고종",
+    "순종실록": "순종",
+    "순종실록부록": "순종부록",
+    "순종부록": "순종부록",
+}
+
+for canonical in KING_ACCESSION_YEARS:
+    KING_ALIASES.setdefault(canonical, canonical)
+
+
+@dataclass(frozen=True)
+class SearchCategory:
+    label: str
+    count: int
+    token: str
+
+
+@dataclass(frozen=True)
+class ResultTitleMetadata:
+    king: str | None
+    regnal_year: int | None
+    gregorian_year: int | None
+    article_title: str
+
+
+@dataclass(frozen=True)
+class SearchResult:
+    article_id: str
+    url: str
+    title: str
+    article_title: str
+    summary: str
+    king: str | None
+    regnal_year: int | None
+    gregorian_year: int | None
+
+
+@dataclass(frozen=True)
+class SearchReport:
+    query: str
+    search_type: str
+    total_results: int
+    type_count: int
+    categories: list[SearchCategory]
+    items: list[SearchResult]
+
+
+@dataclass(frozen=True)
+class ArticleDetail:
+    article_id: str
+    url: str
+    header: str
+    title: str
+    translated_text: str
+    original_text: str
+    classification: str | None
+
+
+def clean_text(value: str | None) -> str:
+    text = COMMENT_PATTERN.sub(" ", value or "")
+    text = BR_PATTERN.sub("\n", text)
+    text = TAG_PATTERN.sub(" ", text)
+    text = html.unescape(text)
+    return WHITESPACE_PATTERN.sub(" ", text).strip()
+
+
+def clean_article_text(value: str | None) -> str:
+    text = COMMENT_PATTERN.sub(" ", value or "")
+    text = BR_PATTERN.sub("\n", text)
+    text = TAG_PATTERN.sub(" ", text)
+    text = html.unescape(text)
+    lines = [WHITESPACE_PATTERN.sub(" ", line).strip() for line in text.splitlines()]
+    text = "\n".join(line for line in lines if line)
+    text = DETAIL_FOOTER_PATTERN.sub("", text).strip()
+    return WHITESPACE_PATTERN.sub(" ", text).strip()
+
+
+def positive_int(raw_value: str) -> int:
+    value = int(raw_value)
+    if value <= 0:
+        raise argparse.ArgumentTypeError("must be a positive integer")
+    return value
+
+
+def normalize_king_name(value: str | None) -> str | None:
+    if value is None:
+        return None
+    cleaned = clean_text(value).replace(" ", "")
+    if not cleaned:
+        return None
+    return KING_ALIASES.get(cleaned, cleaned)
+
+
+def build_opener() -> urllib.request.OpenerDirector:
+    cookie_jar = CookieJar()
+    context = ssl.create_default_context()
+    return urllib.request.build_opener(
+        urllib.request.HTTPCookieProcessor(cookie_jar),
+        urllib.request.HTTPSHandler(context=context),
+    )
+
+
+def build_http_client() -> Any:
+    return build_opener()
+
+
+def should_fallback_to_opener(error: Exception) -> bool:
+    if requests is None:
+        return False
+
+    exceptions = getattr(requests, "exceptions", None)
+    http_error = getattr(exceptions, "HTTPError", None)
+    if isinstance(http_error, type) and isinstance(error, http_error):
+        return False
+
+    request_exception = getattr(exceptions, "RequestException", None)
+    if isinstance(request_exception, type) and isinstance(error, request_exception):
+        return True
+
+    return isinstance(error, OSError)
+
+
+def fetch_text(
+    opener: Any,
+    url: str,
+    *,
+    data: dict[str, str] | None = None,
+    timeout: int = DEFAULT_TIMEOUT,
+    referer: str | None = None,
+) -> str:
+    headers = dict(DEFAULT_HEADERS)
+    if referer is not None:
+        headers["Referer"] = referer
+
+    if requests is not None:
+        try:
+            if data is not None:
+                response = requests.post(url, data=data, timeout=timeout, headers=headers)
+            else:
+                response = requests.get(url, timeout=timeout, headers=headers)
+            response.raise_for_status()
+            return response.text
+        except Exception as error:  # noqa: BLE001
+            if opener is None or not should_fallback_to_opener(error):
+                raise RuntimeError(f"Sillok request failed for {url}: {error}") from error
+
+    body = urllib.parse.urlencode(data).encode("utf-8") if data is not None else None
+    request = urllib.request.Request(url, data=body, headers=headers, method="POST" if body else "GET")
+
+    try:
+        with opener.open(request, timeout=timeout) as response:
+            return response.read().decode("utf-8", "ignore")
+    except urllib.error.HTTPError as error:  # type: ignore[attr-defined]
+        raise RuntimeError(f"Sillok request failed with HTTP {error.code} for {url}") from error
+    except urllib.error.URLError as error:  # type: ignore[attr-defined]
+        raise RuntimeError(f"Sillok request failed for {url}: {error.reason}") from error
+
+
+def extract_hidden_int(html_text: str, field: str) -> int | None:
+    pattern = re.compile(HIDDEN_VALUE_TEMPLATE.format(field=re.escape(field)))
+    match = pattern.search(html_text)
+    if not match:
+        return None
+    digits = DIGITS_PATTERN.search(match.group(1))
+    return int(digits.group(0)) if digits else None
+
+
+def parse_result_title_metadata(title: str) -> ResultTitleMetadata:
+    cleaned_title = clean_text(title)
+    article_title = cleaned_title.split("/", 1)[1].strip() if "/" in cleaned_title else cleaned_title
+    metadata_match = re.search(r",\s*([^,]+?)\s+(즉위년|\d+년)\b", cleaned_title)
+
+    if not metadata_match:
+        return ResultTitleMetadata(None, None, None, article_title)
+
+    king = normalize_king_name(metadata_match.group(1))
+    year_token = metadata_match.group(2)
+    regnal_year = 1 if year_token == "즉위년" else int(year_token.removesuffix("년"))
+    accession_year = KING_ACCESSION_YEARS.get(king or "")
+    if accession_year is None:
+        gregorian_year = None
+    elif year_token == "즉위년":
+        gregorian_year = accession_year
+    else:
+        gregorian_year = accession_year + regnal_year
+
+    return ResultTitleMetadata(king, regnal_year, gregorian_year, article_title)
+
+
+def parse_search_results(html_text: str, *, query: str, search_type: str) -> SearchReport:
+    total_results = extract_hidden_int(html_text, "totalCount")
+    if total_results is None:
+        total_match = TOTAL_COUNT_PATTERN.search(html_text)
+        total_results = int(total_match.group(1)) if total_match else 0
+
+    count_field = {"k": "countK", "w": "countW", "m": "countM", "c": "countC"}.get(search_type, "")
+    type_count = extract_hidden_int(html_text, count_field) if count_field else None
+    if type_count is None:
+        type_count = total_results
+
+    categories: list[SearchCategory] = []
+    for token, label_html in CATEGORY_PATTERN.findall(html_text):
+        label = clean_text(label_html)
+        match = re.match(r"(.+?)\s*\((\d+)\)", label)
+        if not match:
+            continue
+        categories.append(SearchCategory(label=match.group(1), count=int(match.group(2)), token=token))
+
+    items: list[SearchResult] = []
+    for article_id, subject_html, summary_html in RESULT_PATTERN.findall(html_text):
+        title = clean_text(subject_html)
+        title = re.sub(r"^\d+\.\s*", "", title)
+        metadata = parse_result_title_metadata(title)
+        items.append(
+            SearchResult(
+                article_id=article_id,
+                url=DETAIL_URL_TEMPLATE.format(article_id=article_id),
+                title=title,
+                article_title=metadata.article_title,
+                summary=clean_text(summary_html),
+                king=metadata.king,
+                regnal_year=metadata.regnal_year,
+                gregorian_year=metadata.gregorian_year,
+            )
+        )
+
+    return SearchReport(
+        query=query,
+        search_type=search_type,
+        total_results=total_results,
+        type_count=type_count,
+        categories=categories,
+        items=items,
+    )
+
+
+def filter_results(
+    items: Iterable[SearchResult],
+    *,
+    king: str | None = None,
+    year: int | None = None,
+) -> list[SearchResult]:
+    normalized_king = normalize_king_name(king)
+    filtered: list[SearchResult] = []
+
+    for item in items:
+        if normalized_king is not None and item.king != normalized_king:
+            continue
+        if year is not None and item.gregorian_year != year:
+            continue
+        filtered.append(item)
+
+    return filtered
+
+
+def parse_detail_page(html_text: str, *, article_id: str) -> ArticleDetail:
+    title_head = TITLE_HEAD_PATTERN.search(html_text)
+    if not title_head:
+        raise ValueError("Unable to find the article header on the detail page.")
+
+    translated_match = LEFT_VIEW_PATTERN.search(html_text)
+    original_match = RIGHT_VIEW_PATTERN.search(html_text)
+    if not translated_match or not original_match:
+        raise ValueError("Unable to find translated/original article text on the detail page.")
+
+    classification_match = CLASSIFICATION_PATTERN.search(html_text)
+
+    return ArticleDetail(
+        article_id=article_id,
+        url=DETAIL_URL_TEMPLATE.format(article_id=article_id),
+        header=clean_text(title_head.group(1)),
+        title=clean_text(title_head.group(2)),
+        translated_text=clean_article_text(translated_match.group(1)),
+        original_text=clean_article_text(original_match.group(1)),
+        classification=clean_text(classification_match.group(1)) if classification_match else None,
+    )
+
+
+def build_search_payload(*, query: str, search_type: str, page_index: int) -> dict[str, str]:
+    return {
+        "topSearchWord": query,
+        "pageIndex": str(page_index),
+        "initPageUnit": "0",
+        "type": search_type,
+        "sillokType": "S",
+        "topSearchWord_ime": f'<span class="newbatang">{html.escape(query)}</span>',
+    }
+
+
+def fetch_search_page(
+    opener: urllib.request.OpenerDirector,
+    *,
+    query: str,
+    search_type: str,
+    page_index: int,
+    timeout: int,
+) -> SearchReport:
+    payload = build_search_payload(query=query, search_type=search_type, page_index=page_index)
+    html_text = fetch_text(opener, SEARCH_URL, data=payload, timeout=timeout, referer=f"{BASE_URL}/main/main.do")
+    return parse_search_results(html_text, query=query, search_type=search_type)
+
+
+def fetch_detail_page(
+    opener: urllib.request.OpenerDirector,
+    *,
+    article_id: str,
+    timeout: int,
+) -> ArticleDetail:
+    url = DETAIL_URL_TEMPLATE.format(article_id=article_id)
+    html_text = fetch_text(opener, url, timeout=timeout, referer=SEARCH_URL)
+    return parse_detail_page(html_text, article_id=article_id)
+
+
+def search_sillok(
+    query: str,
+    *,
+    king: str | None = None,
+    year: int | None = None,
+    limit: int = DEFAULT_LIMIT,
+    search_type: str = "k",
+    timeout: int = DEFAULT_TIMEOUT,
+) -> dict:
+    opener = build_http_client()
+    reports: list[SearchReport] = []
+    filtered_results: list[SearchResult] = []
+
+    page_index = 1
+    total_pages = 1
+    while page_index <= total_pages and page_index <= MAX_PAGES:
+        report = fetch_search_page(opener, query=query, search_type=search_type, page_index=page_index, timeout=timeout)
+        reports.append(report)
+        page_filtered = filter_results(report.items, king=king, year=year)
+        filtered_results.extend(page_filtered)
+
+        if page_index == 1:
+            page_size = len(report.items) or 1
+            total_pages = max(1, math.ceil((report.type_count or report.total_results or 0) / page_size))
+        if len(filtered_results) >= limit or not report.items:
+            break
+        page_index += 1
+
+    first_report = reports[0] if reports else SearchReport(query, search_type, 0, 0, [], [])
+    limited_results = filtered_results[:limit]
+    details = [fetch_detail_page(opener, article_id=item.article_id, timeout=timeout) for item in limited_results]
+    detail_map = {detail.article_id: detail for detail in details}
+
+    serialized_items = []
+    for item in limited_results:
+        detail = detail_map.get(item.article_id)
+        serialized_items.append(
+            {
+                **asdict(item),
+                "detail": asdict(detail) if detail else None,
+                "excerpt": detail.translated_text[:280] if detail and detail.translated_text else item.summary[:280],
+            }
+        )
+
+    return {
+        "query": query,
+        "type": search_type,
+        "filters": {"king": normalize_king_name(king), "year": year, "limit": limit},
+        "total_results": first_report.total_results,
+        "type_count": first_report.type_count,
+        "returned_count": len(serialized_items),
+        "categories": [asdict(category) for category in first_report.categories],
+        "items": serialized_items,
+    }
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Search Joseon Sillok records from sillok.history.go.kr")
+    parser.add_argument("--query", required=True, help="Search keyword to send to the Joseon Sillok site")
+    parser.add_argument("--king", help="Optional king filter, e.g. 세종 or 세종실록")
+    parser.add_argument("--year", type=positive_int, help="Optional Gregorian year filter, e.g. 1443")
+    parser.add_argument("--limit", type=positive_int, default=DEFAULT_LIMIT, help="Number of results to return")
+    parser.add_argument(
+        "--type",
+        dest="search_type",
+        choices=["k", "w"],
+        default="k",
+        help="Search translated text (k) or original text (w)",
+    )
+    parser.add_argument("--timeout", type=positive_int, default=DEFAULT_TIMEOUT, help="HTTP timeout in seconds")
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv)
+
+    try:
+        report = search_sillok(
+            args.query,
+            king=args.king,
+            year=args.year,
+            limit=args.limit,
+            search_type=args.search_type,
+            timeout=args.timeout,
+        )
+    except Exception as error:  # noqa: BLE001
+        print(json.dumps({"error": str(error)}, ensure_ascii=False, indent=2), file=sys.stderr)
+        return 1
+
+    print(json.dumps(report, ensure_ascii=False, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/skills/joseon-sillok-search/skill.json

@@ -0,0 +1,8 @@
+{
+  "name": "joseon-sillok-search",
+  "description": "Search Joseon Dynasty Annals records from the official sillok.history.go.kr site with keyword search plus optional king/year narrowing.",
+  "profiles": [
+    "lookup"
+  ],
+  "frontmatter": "name: joseon-sillok-search\ndescription: Search Joseon Dynasty Annals records from the official sillok.history.go.kr site with keyword search plus optional king/year narrowing.\nlicense: MIT\nmetadata:\n  category: history\n  locale: ko-KR\n  phase: v1"
+}