@agentunion/kite 1.0.7 → 1.2.0

package/extensions/services/web/vendor/storage/store.py

@@ -0,0 +1,507 @@
+"""File-based storage engine using JSONL and Markdown."""
+
+from __future__ import annotations
+
+import json
+import uuid
+from collections import defaultdict
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any
+
+import aiofiles
+import aiofiles.os
+
+from .. import config as cfg
+
+
+def _data_path(*parts: str) -> Path:
+    return cfg.data_dir().joinpath(*parts)
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+# ---------------------------------------------------------------------------
+# Generic JSONL helpers
+# ---------------------------------------------------------------------------
+
+async def append_jsonl(filepath: Path, record: dict[str, Any]) -> None:
+    """Append a single JSON line to *filepath*."""
+    filepath.parent.mkdir(parents=True, exist_ok=True)
+    async with aiofiles.open(filepath, "a", encoding="utf-8") as f:
+        await f.write(json.dumps(record, ensure_ascii=False) + "\n")
+
+
+async def read_jsonl(filepath: Path) -> list[dict[str, Any]]:
+    """Read all lines from a JSONL file, returning a list of dicts."""
+    if not filepath.exists():
+        return []
+    records: list[dict[str, Any]] = []
+    async with aiofiles.open(filepath, "r", encoding="utf-8") as f:
+        async for line in f:
+            line = line.strip()
+            if line:
+                try:
+                    records.append(json.loads(line))
+                except json.JSONDecodeError:
+                    continue
+    return records
+
+
+async def read_jsonl_paginated(
+    filepath: Path,
+    page: int = 1,
+    page_size: int = 20,
+    filter_fn=None,
+) -> tuple[list[dict[str, Any]], int]:
+    """Read JSONL with optional filter and pagination.
+
+    Returns (page_records, total_matching).
+    """
+    all_records = await read_jsonl(filepath)
+    if filter_fn:
+        all_records = [r for r in all_records if filter_fn(r)]
+    # newest first
+    all_records.reverse()
+    total = len(all_records)
+    start = (page - 1) * page_size
+    end = start + page_size
+    return all_records[start:end], total
+
+
+async def update_jsonl_record(
+    filepath: Path, match_key: str, match_value: Any, updates: dict[str, Any]
+) -> bool:
+    """Update first record in JSONL where record[match_key] == match_value."""
+    if not filepath.exists():
+        return False
+    records = await read_jsonl(filepath)
+    found = False
+    for rec in records:
+        if rec.get(match_key) == match_value:
+            rec.update(updates)
+            found = True
+            break
+    if found:
+        async with aiofiles.open(filepath, "w", encoding="utf-8") as f:
+            for rec in records:
+                await f.write(json.dumps(rec, ensure_ascii=False) + "\n")
+    return found
+
+
+async def delete_jsonl_record(filepath: Path, match_key: str, match_value: Any) -> bool:
+    """Delete first record where record[match_key] == match_value."""
+    if not filepath.exists():
+        return False
+    records = await read_jsonl(filepath)
+    new_records = [r for r in records if r.get(match_key) != match_value]
+    if len(new_records) == len(records):
+        return False
+    async with aiofiles.open(filepath, "w", encoding="utf-8") as f:
+        for rec in new_records:
+            await f.write(json.dumps(rec, ensure_ascii=False) + "\n")
+    return True
+
+
+# ---------------------------------------------------------------------------
+# Markdown helpers
+# ---------------------------------------------------------------------------
+
+async def write_md(filepath: Path, content: str) -> None:
+    filepath.parent.mkdir(parents=True, exist_ok=True)
+    async with aiofiles.open(filepath, "w", encoding="utf-8") as f:
+        await f.write(content)
+
+
+async def read_md(filepath: Path) -> str | None:
+    if not filepath.exists():
+        return None
+    async with aiofiles.open(filepath, "r", encoding="utf-8") as f:
+        return await f.read()
+
+
+# ---------------------------------------------------------------------------
+# Domain-specific storage
+# ---------------------------------------------------------------------------
+
+# --- Contacts ---
+
+def contacts_path() -> Path:
+    return _data_path("contacts", "contacts.jsonl")
+
+
+async def list_contacts(page: int = 1, page_size: int = 50, query: str | None = None):
+    def _filter(r):
+        if not query:
+            return True
+        q = query.lower()
+        return (
+            q in r.get("name", "").lower()
+            or q in r.get("phone", "").lower()
+            or q in r.get("company", "").lower()
+            or q in r.get("notes", "").lower()
+            or any(q in t.lower() for t in r.get("tags", []))
+        )
+    return await read_jsonl_paginated(contacts_path(), page, page_size, _filter)
+
+
+async def get_contact(contact_id: str) -> dict[str, Any] | None:
+    records = await read_jsonl(contacts_path())
+    for r in records:
+        if r.get("id") == contact_id:
+            return r
+    return None
+
+
+async def find_contact_by_phone(phone: str) -> dict[str, Any] | None:
+    records = await read_jsonl(contacts_path())
+    for r in records:
+        if r.get("phone") == phone:
+            return r
+    return None
+
+
+async def search_contacts(query: str) -> list[dict[str, Any]]:
+    records = await read_jsonl(contacts_path())
+    q = query.lower()
+    return [
+        r for r in records
+        if q in r.get("name", "").lower()
+        or q in r.get("phone", "").lower()
+        or q in r.get("company", "").lower()
+        or q in r.get("notes", "").lower()
+    ]
+
+
+async def add_contact(contact: dict[str, Any]) -> dict[str, Any]:
+    if "id" not in contact:
+        contact["id"] = str(uuid.uuid4())
+    contact.setdefault("created_at", _now_iso())
+    contact.setdefault("updated_at", _now_iso())
+    contact.setdefault("source", "manual")
+    contact.setdefault("tags", [])
+    await append_jsonl(contacts_path(), contact)
+    return contact
+
+
+async def update_contact(contact_id: str, updates: dict[str, Any]) -> bool:
+    updates["updated_at"] = _now_iso()
+    return await update_jsonl_record(contacts_path(), "id", contact_id, updates)
+
+
+async def delete_contact(contact_id: str) -> bool:
+    return await delete_jsonl_record(contacts_path(), "id", contact_id)
+
+
+# --- SMS ---
+
+def sms_path() -> Path:
+    return _data_path("sms", "messages.jsonl")
+
+
+async def save_sms(record: dict[str, Any]) -> dict[str, Any]:
+    if "id" not in record:
+        record["id"] = str(uuid.uuid4())
+    record.setdefault("timestamp", _now_iso())
+    await append_jsonl(sms_path(), record)
+    return record
+
+
+async def list_sms(
+    page: int = 1, page_size: int = 20, phone_number: str | None = None
+):
+    def _filter(r):
+        if phone_number and r.get("phone_number") != phone_number:
+            return False
+        return True
+    return await read_jsonl_paginated(sms_path(), page, page_size, _filter)
+
+
+# --- Tasks ---
+
+def tasks_path() -> Path:
+    return _data_path("tasks", "tasks.jsonl")
+
+
+async def save_task(record: dict[str, Any]) -> None:
+    await append_jsonl(tasks_path(), record)
+
+
+async def update_task(task_id: str, updates: dict[str, Any]) -> None:
+    await update_jsonl_record(tasks_path(), "task_id", task_id, updates)
+
+
+async def get_task(task_id: str) -> dict[str, Any] | None:
+    records = await read_jsonl(tasks_path())
+    for r in records:
+        if r.get("task_id") == task_id:
+            return r
+    return None
+
+
+async def list_tasks(
+    page: int = 1, page_size: int = 20, **filters
+) -> tuple[list[dict[str, Any]], int]:
+    def _filter(r):
+        for k, v in filters.items():
+            if v is not None and r.get(k) != v:
+                return False
+        return True
+    return await read_jsonl_paginated(tasks_path(), page, page_size, _filter)
+
+
+# --- Dev Log ---
+
+def devlog_path() -> Path:
+    return _data_path("devlog", "records.jsonl")
+
+
+async def add_devlog(record: dict[str, Any]) -> dict[str, Any]:
+    if "id" not in record:
+        record["id"] = str(uuid.uuid4())
+    record.setdefault("created_at", _now_iso())
+    record.setdefault("status", "pending")
+    await append_jsonl(devlog_path(), record)
+    return record
+
+
+async def list_devlog(
+    page: int = 1, page_size: int = 20, status: str | None = None
+) -> tuple[list[dict[str, Any]], int]:
+    def _filter(r):
+        if status and r.get("status") != status:
+            return False
+        return True
+    return await read_jsonl_paginated(devlog_path(), page, page_size, _filter)
+
+
+async def get_devlog(record_id: str) -> dict[str, Any] | None:
+    records = await read_jsonl(devlog_path())
+    for r in records:
+        if r.get("id") == record_id:
+            return r
+    return None
+
+
+async def update_devlog(record_id: str, updates: dict[str, Any]) -> bool:
+    updates["updated_at"] = _now_iso()
+    # Auto-set completed_at when status changes to done
+    if updates.get("status") == "done":
+        updates.setdefault("completed_at", _now_iso())
+    elif updates.get("status") and updates["status"] != "done":
+        updates["completed_at"] = None
+    return await update_jsonl_record(devlog_path(), "id", record_id, updates)
+
+
+async def delete_devlog(record_id: str) -> bool:
+    return await delete_jsonl_record(devlog_path(), "id", record_id)
+
+
+def _devlog_archive_dir() -> Path:
+    return _data_path("devlog", "archive")
+
+
+async def archive_devlog(cutoff_iso: str | None = None) -> dict[str, Any]:
+    """Archive done records completed before the given cutoff time.
+
+    If *cutoff_iso* is provided (ISO-8601 string), use it as the cutoff.
+    Otherwise default to the end of the day before yesterday (UTC).
+
+    For records without ``completed_at``, fall back to ``updated_at`` then
+    ``created_at`` so that old done records without an explicit completion
+    timestamp can still be archived.
+
+    Returns {archived_count, files, skipped_no_time, total_done, cutoff}.
+    """
+    records = await read_jsonl(devlog_path())
+    if not records:
+        return {"archived_count": 0, "files": [], "skipped_no_time": 0,
+                "total_done": 0, "cutoff": None, "message": "records.jsonl 为空"}
+
+    # Determine cutoff
+    if cutoff_iso:
+        try:
+            cutoff = datetime.fromisoformat(cutoff_iso.replace("Z", "+00:00"))
+        except (ValueError, TypeError):
+            return {"archived_count": 0, "files": [],
+                    "message": f"无效的时间格式: {cutoff_iso}"}
+    else:
+        now = datetime.now(timezone.utc)
+        cutoff = (now - timedelta(days=2)).replace(
+            hour=23, minute=59, second=59, microsecond=999999
+        )
+
+    to_archive: list[dict[str, Any]] = []
+    to_keep: list[dict[str, Any]] = []
+    skipped_no_time = 0
+    total_done = 0
+
+    for rec in records:
+        if rec.get("status") == "done":
+            total_done += 1
+            # Try completed_at → updated_at → created_at
+            time_str = (rec.get("completed_at")
+                        or rec.get("updated_at")
+                        or rec.get("created_at"))
+            if not time_str:
+                skipped_no_time += 1
+                to_keep.append(rec)
+                continue
+            try:
+                t = datetime.fromisoformat(time_str.replace("Z", "+00:00"))
+                if t <= cutoff:
+                    # Ensure completed_at is set for archived records
+                    if not rec.get("completed_at"):
+                        rec["completed_at"] = time_str
+                    to_archive.append(rec)
+                    continue
+            except (ValueError, TypeError):
+                skipped_no_time += 1
+        to_keep.append(rec)
+
+    if not to_archive:
+        return {
+            "archived_count": 0,
+            "files": [],
+            "skipped_no_time": skipped_no_time,
+            "total_done": total_done,
+            "cutoff": cutoff.isoformat(),
+            "message": f"没有符合条件的记录（共 {total_done} 条已完成，截止时间 {cutoff.isoformat()}）",
+        }
+
+    # Group by date (YYYYMMDD based on completed_at)
+    grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for rec in to_archive:
+        try:
+            completed = datetime.fromisoformat(
+                rec["completed_at"].replace("Z", "+00:00")
+            )
+            date_str = completed.strftime("%Y%m%d")
+        except (ValueError, TypeError):
+            date_str = "unknown"
+        grouped[date_str].append(rec)
+
+    archive_dir = _devlog_archive_dir()
+    archive_dir.mkdir(parents=True, exist_ok=True)
+
+    written_files: list[str] = []
+    for date_str, recs in sorted(grouped.items()):
+        filepath = archive_dir / f"{date_str}.jsonl"
+        # Append to existing archive file (in case of repeated archiving)
+        async with aiofiles.open(filepath, "a", encoding="utf-8") as f:
+            for rec in recs:
+                await f.write(json.dumps(rec, ensure_ascii=False) + "\n")
+        written_files.append(f"{date_str}.jsonl")
+
+    # Rewrite records.jsonl with only kept records
+    async with aiofiles.open(devlog_path(), "w", encoding="utf-8") as f:
+        for rec in to_keep:
+            await f.write(json.dumps(rec, ensure_ascii=False) + "\n")
+
+    await _save_archive_meta(cutoff.isoformat())
+
+    return {"archived_count": len(to_archive), "files": written_files,
+            "cutoff": cutoff.isoformat(), "total_done": total_done}
+
+
+def _devlog_archive_meta_path() -> Path:
+    return _data_path("devlog", "archive", "meta.json")
+
+
+async def _save_archive_meta(cutoff_iso: str) -> None:
+    """Save last archive cutoff time."""
+    meta_path = _devlog_archive_meta_path()
+    meta_path.parent.mkdir(parents=True, exist_ok=True)
+    async with aiofiles.open(meta_path, "w", encoding="utf-8") as f:
+        await f.write(json.dumps({"last_cutoff": cutoff_iso}, ensure_ascii=False))
+
+
+async def get_last_archive_cutoff() -> str | None:
+    """Return the last archive cutoff ISO string, or None if never archived."""
+    meta_path = _devlog_archive_meta_path()
+    if not meta_path.exists():
+        return None
+    try:
+        async with aiofiles.open(meta_path, "r", encoding="utf-8") as f:
+            data = json.loads(await f.read())
+        return data.get("last_cutoff")
+    except (json.JSONDecodeError, OSError):
+        return None
+
+
+async def list_devlog_archives() -> list[str]:
+    """List archive date strings (without extension), sorted newest first."""
+    archive_dir = _devlog_archive_dir()
+    if not archive_dir.exists():
+        return []
+    files = sorted(
+        (f.stem for f in archive_dir.iterdir() if f.suffix == ".jsonl"),
+        reverse=True,
+    )
+    return files
+
+
+async def read_devlog_archive(date_str: str) -> list[dict[str, Any]]:
+    """Read all records from a specific archive file."""
+    filepath = _devlog_archive_dir() / f"{date_str}.jsonl"
+    return await read_jsonl(filepath)
+
+
+# --- Stats ---
+
+async def get_stats() -> dict[str, Any]:
+    from datetime import timedelta
+
+    all_tasks = await read_jsonl(tasks_path())
+    # Only count tasks that became actual calls (have a direction field)
+    calls = [t for t in all_tasks if t.get("direction") is not None]
+    sms_records = await read_jsonl(sms_path())
+    contacts_list = await read_jsonl(contacts_path())
+
+    now = datetime.now(timezone.utc)
+    today_start = now.replace(hour=0, minute=0, second=0, microsecond=0)
+    week_start = today_start - timedelta(days=today_start.weekday())
+
+    total_duration = sum(c.get("duration_seconds", 0) for c in calls)
+
+    calls_today = 0
+    calls_this_week = 0
+    for c in calls:
+        started = c.get("started_at", "")
+        if started:
+            try:
+                t = datetime.fromisoformat(started.replace("Z", "+00:00"))
+                if t >= today_start:
+                    calls_today += 1
+                if t >= week_start:
+                    calls_this_week += 1
+            except (ValueError, TypeError):
+                pass
+
+    calls_by_result: dict[str, int] = {}
+    for c in calls:
+        r = c.get("result", "unknown")
+        calls_by_result[r] = calls_by_result.get(r, 0) + 1
+
+    calls_by_direction: dict[str, int] = {}
+    for c in calls:
+        d = c.get("direction", "unknown")
+        calls_by_direction[d] = calls_by_direction.get(d, 0) + 1
+
+    sms_sent = sum(1 for s in sms_records if s.get("direction") == "outgoing")
+    sms_received = sum(1 for s in sms_records if s.get("direction") == "incoming")
+
+    return {
+        "total_calls": len(calls),
+        "total_duration_seconds": total_duration,
+        "avg_duration_seconds": round(total_duration / max(len(calls), 1), 1),
+        "calls_today": calls_today,
+        "calls_this_week": calls_this_week,
+        "calls_by_result": calls_by_result,
+        "calls_by_direction": calls_by_direction,
+        "total_sms_sent": sms_sent,
+        "total_sms_received": sms_received,
+        "total_contacts": len(contacts_list),
+    }