agent-sin 0.1.12 → 0.1.15

package/builtin-skills/nightly-topic-knowledge/_topics_lib.py

@@ -0,0 +1,415 @@
+"""Helpers for the nightly-topic-knowledge skill.
+
+Reads daily memory / conversation log / notes, builds the LLM prompt, parses
+the JSON response, and merges results into per-topic knowledge files with caps.
+File I/O is deliberately kept here so main.py stays focused on orchestration.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from datetime import datetime, timedelta, timezone
+from typing import Any
+from zoneinfo import ZoneInfo
+
+
+DEFAULT_TIMEZONE = "Asia/Tokyo"
+DEFAULT_MAX_CHARS = 60000
+
+# Topic ids are now open-ended — the model invents them per user so that
+# someone who suddenly gets into movies ends up with a movies.json file
+# instead of an "other" bucket. Style examples shown to the model below.
+TOPIC_ID_MAX_LEN = 48
+TOPIC_ID_PATTERN = re.compile(r"[a-z0-9]+(?:-[a-z0-9]+)*$")
+TOPIC_STYLE_EXAMPLES = [
+    "agent-sin",
+    "movies",
+    "ai-models",
+    "health",
+    "english-learning",
+    "personal-finance",
+]
+
+FIELD_CAPS: dict[str, int] = {
+    "stable_facts": 12,
+    "preferences": 12,
+    "recent_focus": 8,
+    "open_questions": 8,
+    "useful_context": 10,
+    "avoid_assumptions": 8,
+}
+
+LIST_FIELDS = list(FIELD_CAPS.keys())
+
+
+# ---------- date helpers ----------
+
+def resolve_target_date(date_arg: str | None, tz_name: str) -> str:
+    if date_arg:
+        # input.schema already validated YYYY-MM-DD; keep a defensive parse.
+        datetime.strptime(date_arg, "%Y-%m-%d")
+        return date_arg
+    tz = ZoneInfo(tz_name)
+    now = datetime.now(tz)
+    yesterday = (now - timedelta(days=1)).date()
+    return yesterday.isoformat()
+
+
+def split_date(date_str: str) -> tuple[str, str, str]:
+    yyyy, mm, dd = date_str.split("-")
+    return yyyy, mm, dd
+
+
+# ---------- source readers ----------
+
+def read_daily_memory(memory_dir: str, date_str: str) -> tuple[str | None, str]:
+    yyyy, mm, _ = split_date(date_str)
+    path = os.path.join(memory_dir, "daily", yyyy, mm, f"{date_str}.md")
+    text = _read_text(path)
+    return (path if text else None), text
+
+
+def read_conversation_log(logs_dir: str, date_str: str) -> tuple[str | None, str]:
+    path = os.path.join(logs_dir, "conversations", f"{date_str}.jsonl")
+    raw = _read_text(path)
+    if not raw:
+        return (None, "")
+    lines = []
+    for line in raw.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            entry = json.loads(line)
+        except Exception:
+            continue
+        role = str(entry.get("role") or entry.get("source") or "").strip()
+        content = str(entry.get("content") or "").strip()
+        if not content:
+            continue
+        ts = str(entry.get("ts") or "").strip()
+        header = " ".join(part for part in (ts, role) if part)
+        lines.append(f"[{header}]\n{content}".rstrip())
+    body = "\n\n".join(lines)
+    return (path if body else None), body
+
+
+def read_notes(notes_dir: str, date_str: str) -> tuple[str | None, str]:
+    yyyy, mm, _ = split_date(date_str)
+    path = os.path.join(notes_dir, yyyy, mm, f"{date_str}.md")
+    text = _read_text(path)
+    return (path if text else None), text
+
+
+def _read_text(path: str) -> str:
+    if not path or not os.path.isfile(path):
+        return ""
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            return f.read()
+    except Exception:
+        return ""
+
+
+def bundle_sources(parts: list[tuple[str, str]], max_chars: int) -> tuple[str, int]:
+    """Concatenate (label, text) pairs into a single bounded blob."""
+    chunks: list[str] = []
+    total = 0
+    for label, text in parts:
+        text = (text or "").strip()
+        if not text:
+            continue
+        header = f"## {label}\n"
+        remaining = max_chars - total - len(header)
+        if remaining <= 0:
+            break
+        if len(text) > remaining:
+            text = text[:remaining] + "\n... [truncated]"
+        chunk = header + text
+        chunks.append(chunk)
+        total += len(chunk) + 2  # joiner
+    return ("\n\n".join(chunks), total)
+
+
+# ---------- LLM prompt + parsing ----------
+
+def build_prompt(date_str: str, source_bundle: str, known_topic_ids: list[str] | None = None) -> str:
+    schema = {
+        "topics": [
+            {
+                "topic_id": "<short kebab-case ASCII id you choose for this theme>",
+                "summary": "<1-3 short sentences capturing the current understanding>",
+                "recent_focus": ["<recently emphasized concrete sub-themes>"],
+                "stable_facts": ["<long-lasting, non-time-sensitive facts>"],
+                "preferences": ["<user preferences or decision criteria>"],
+                "open_questions": ["<unresolved points to revisit>"],
+                "useful_context": ["<short context lines that help future replies>"],
+                "avoid_assumptions": ["<things to NOT assume>"],
+            }
+        ]
+    }
+    instructions = [
+        "You are condensing one day of an individual user's conversations, daily memory, and notes",
+        "into per-topic knowledge that an assistant can re-use in future chats.",
+        "Hard rules:",
+        "- Output ONE JSON object only. No prose, no markdown fences.",
+        "- Choose topic_ids freely — there is no allow-list. One topic per coherent theme.",
+        "- topic_id must be short kebab-case ASCII (lowercase a-z, 0-9, hyphen). 1-48 chars.",
+        f"  Examples: {', '.join(TOPIC_STYLE_EXAMPLES)}.",
+        "- Reuse an existing topic_id when the theme matches (see the list below). Invent a new id only for genuinely new themes.",
+        "- Do NOT preserve everything. Keep only items that will still be useful in future conversations.",
+        "- Drop small talk, transient emotions, short-term task progress, and any secrets / credentials.",
+        "- Prefer the user's continuing interests, judgement criteria, preferences, and spec decisions.",
+        "- Each list item must be a short standalone sentence (one line, < 200 chars).",
+        "- Newer signals matter more than older ones.",
+        "- Empty arrays are fine; omit fields you cannot fill with care.",
+        "- Respond in the same language the user mostly used in the sources (Japanese in most cases).",
+        "",
+        f"Date: {date_str}",
+        "",
+    ]
+    if known_topic_ids:
+        instructions.append("Existing topic ids (prefer reusing these when they fit):")
+        instructions.extend(f"- {tid}" for tid in known_topic_ids)
+        instructions.append("")
+    instructions.extend(
+        [
+            "JSON schema (output shape):",
+            json.dumps(schema, ensure_ascii=False, indent=2),
+            "",
+            "Sources:",
+            source_bundle or "(empty)",
+        ]
+    )
+    return "\n".join(instructions)
+
+
+def sanitize_topic_id(raw: object) -> str | None:
+    """Normalize a model-supplied topic_id to safe kebab-case or return None."""
+    if not isinstance(raw, str):
+        return None
+    text = raw.strip().lower()
+    # Drop anything that isn't a-z / 0-9; collapse runs into a single hyphen.
+    text = re.sub(r"[^a-z0-9]+", "-", text)
+    text = text.strip("-")
+    if not text:
+        return None
+    if len(text) > TOPIC_ID_MAX_LEN:
+        text = text[:TOPIC_ID_MAX_LEN].rstrip("-")
+    if not TOPIC_ID_PATTERN.match(text):
+        return None
+    return text
+
+
+_JSON_OBJECT_PATTERN = re.compile(r"\{[\s\S]*\}")
+
+
+def parse_llm_response(text: str) -> dict[str, Any]:
+    raw = (text or "").strip()
+    if not raw:
+        raise ValueError("Empty LLM response")
+    # Drop common code-fence wrappers before trying strict parse.
+    fenced = re.match(r"^```(?:json)?\s*([\s\S]*?)\s*```$", raw)
+    if fenced:
+        raw = fenced.group(1).strip()
+    try:
+        return json.loads(raw)
+    except Exception:
+        pass
+    # Fall back to the first balanced-looking JSON object in the text.
+    match = _JSON_OBJECT_PATTERN.search(raw)
+    if not match:
+        raise ValueError("LLM response did not contain JSON")
+    return json.loads(match.group(0))
+
+
+def normalize_topics(parsed: dict[str, Any]) -> list[dict[str, Any]]:
+    raw_topics = parsed.get("topics")
+    if not isinstance(raw_topics, list):
+        return []
+    out: list[dict[str, Any]] = []
+    seen_ids: set[str] = set()
+    for entry in raw_topics:
+        if not isinstance(entry, dict):
+            continue
+        tid = sanitize_topic_id(entry.get("topic_id"))
+        if not tid or tid in seen_ids:
+            continue
+        seen_ids.add(tid)
+        topic: dict[str, Any] = {"topic_id": tid}
+        summary = entry.get("summary")
+        if isinstance(summary, str) and summary.strip():
+            topic["summary"] = summary.strip()
+        for field in LIST_FIELDS:
+            items = entry.get(field)
+            if not isinstance(items, list):
+                continue
+            cleaned: list[str] = []
+            for item in items:
+                if not isinstance(item, str):
+                    continue
+                value = item.strip()
+                if value:
+                    cleaned.append(value[:300])
+            if cleaned:
+                topic[field] = cleaned
+        out.append(topic)
+    return out
+
+
+def list_existing_topic_ids(memory_dir: str) -> list[str]:
+    folder = os.path.join(memory_dir, "topic-knowledge", "topics")
+    if not os.path.isdir(folder):
+        return []
+    ids: list[str] = []
+    for name in os.listdir(folder):
+        if not name.endswith(".json"):
+            continue
+        tid = name[:-5]
+        if TOPIC_ID_PATTERN.match(tid):
+            ids.append(tid)
+    return sorted(ids)
+
+
+# ---------- merge logic ----------
+
+def topic_path(memory_dir: str, topic_id: str) -> str:
+    return os.path.join(memory_dir, "topic-knowledge", "topics", f"{topic_id}.json")
+
+
+def index_path(memory_dir: str) -> str:
+    return os.path.join(memory_dir, "topic-knowledge", "index.json")
+
+
+def run_path(memory_dir: str, date_str: str) -> str:
+    return os.path.join(memory_dir, "topic-knowledge", "runs", f"{date_str}.json")
+
+
+def load_json(path: str) -> dict[str, Any] | None:
+    if not os.path.isfile(path):
+        return None
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except Exception:
+        return None
+
+
+def write_json(path: str, data: dict[str, Any]) -> None:
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    tmp = path + ".tmp"
+    with open(tmp, "w", encoding="utf-8") as f:
+        json.dump(data, f, ensure_ascii=False, indent=2)
+        f.write("\n")
+    os.replace(tmp, path)
+
+
+def merge_topic(
+    existing: dict[str, Any] | None,
+    update: dict[str, Any],
+    date_str: str,
+    now_iso: str,
+) -> dict[str, Any]:
+    topic_id = update["topic_id"]
+    base: dict[str, Any] = dict(existing or {})
+    base["topic_id"] = topic_id
+    base.setdefault("name", topic_id)
+
+    # New summary always wins (newer signals matter more), but keep the old
+    # one if the LLM did not produce a fresh summary this round.
+    if "summary" in update:
+        base["summary"] = update["summary"]
+    elif "summary" not in base:
+        base["summary"] = ""
+
+    # Merge each list field with caps; new items prepended so the most recent
+    # distillation surfaces first, then older items fill remaining slots.
+    for field, cap in FIELD_CAPS.items():
+        existing_items = base.get(field) or []
+        if not isinstance(existing_items, list):
+            existing_items = []
+        new_items = update.get(field) or []
+        merged: list[str] = []
+        seen: set[str] = set()
+        for item in list(new_items) + list(existing_items):
+            if not isinstance(item, str):
+                continue
+            key = re.sub(r"\s+", " ", item).strip().lower()
+            if not key or key in seen:
+                continue
+            seen.add(key)
+            merged.append(item.strip())
+            if len(merged) >= cap:
+                break
+        base[field] = merged
+
+    source_dates = base.get("source_dates")
+    if not isinstance(source_dates, list):
+        source_dates = []
+    if date_str not in source_dates:
+        source_dates.append(date_str)
+    # keep last 30 source dates only
+    base["source_dates"] = sorted(set(source_dates))[-30:]
+
+    base["last_updated"] = now_iso
+    base["freshness_score"] = _freshness_score(base["source_dates"], date_str)
+    base["confidence"] = _confidence(update)
+    return base
+
+
+def _freshness_score(source_dates: list[str], reference: str) -> float:
+    if not source_dates:
+        return 0.0
+    try:
+        ref = datetime.strptime(reference, "%Y-%m-%d").date()
+    except Exception:
+        return 0.0
+    score = 0.0
+    for d in source_dates[-14:]:
+        try:
+            day = datetime.strptime(d, "%Y-%m-%d").date()
+        except Exception:
+            continue
+        age_days = (ref - day).days
+        if age_days < 0:
+            continue
+        score += max(0.0, 1.0 - age_days / 14.0)
+    return round(min(1.0, score / 7.0), 3)
+
+
+def _confidence(update: dict[str, Any]) -> float:
+    populated = sum(1 for field in LIST_FIELDS if update.get(field))
+    if "summary" in update:
+        populated += 1
+    return round(min(1.0, populated / 5.0), 3)
+
+
+def build_index(
+    existing: dict[str, Any] | None,
+    topic_files: list[dict[str, Any]],
+    now_iso: str,
+) -> dict[str, Any]:
+    by_id: dict[str, dict[str, Any]] = {}
+    if isinstance(existing, dict):
+        for entry in (existing.get("topics") or []):
+            if isinstance(entry, dict) and entry.get("topic_id"):
+                by_id[entry["topic_id"]] = entry
+    for topic in topic_files:
+        tid = topic["topic_id"]
+        by_id[tid] = {
+            "topic_id": tid,
+            "name": topic.get("name", tid),
+            "summary": topic.get("summary", ""),
+            "last_updated": topic.get("last_updated", now_iso),
+        }
+    ordered = sorted(by_id.values(), key=lambda item: item["topic_id"])
+    # recent = topics touched in the latest run (passed in via topic_files)
+    recent = [topic["topic_id"] for topic in topic_files]
+    return {
+        "version": 1,
+        "updated_at": now_iso,
+        "topics": ordered,
+        "recent": recent,
+    }