@researai/deepscientist 1.5.11 → 1.5.12

package/src/deepscientist/prompts/builder.py

@@ -257,7 +257,7 @@ class PromptBuilder:
             f"- bound_external_connector_count: {surface_context['bound_external_connector_count']}",
             "- surface_rule: treat web, TUI, and connector threads as one continuous quest, but adapt the amount of detail to the active surface.",
             "- surface_reply_rule: use artifact.interact(...) for durable user-visible continuity; do not dump raw internal tool chatter into connector replies.",
-            "- connector_contract_rule: load connector-specific prompt fragments only for the active or bound external connector; do not load unused connector contracts.",
+            "- connector_contract_rule: choose the active connector surface from the latest inbound external user turn when one exists; otherwise fall back to the bound external connector; keep purely local web/TUI turns on the local surface even if the quest is externally bound.",
         ]
 
         if connector == "qq":
@@ -316,12 +316,23 @@ class PromptBuilder:
             if str(parsed.get("connector") or "").strip().lower() == "local":
                 continue
             bound_external.append(parsed)
-        active = bound_external[0] if bound_external else None
-        origin = "bound_external_binding" if active is not None else "latest_user_source"
-        if active is None and latest_user_parsed is not None:
-            latest_connector = str(latest_user_parsed.get("connector") or "").strip().lower()
-            if latest_connector and latest_connector != "local":
-                active = latest_user_parsed
+        latest_connector = str((latest_user_parsed or {}).get("connector") or "").strip().lower()
+        if latest_connector and latest_connector != "local":
+            active = latest_user_parsed
+            origin = "latest_user_source"
+        elif latest_user is not None:
+            return {
+                "latest_user_source": latest_user_source,
+                "active_surface": "local",
+                "active_connector": "local",
+                "active_chat_type": "local",
+                "active_chat_id": "default",
+                "active_connector_origin": "latest_user_source_local",
+                "bound_external_connector_count": len(bound_external),
+            }
+        else:
+            active = bound_external[0] if bound_external else None
+            origin = "bound_external_binding" if active is not None else "none"
         if active is None:
             return {
                 "latest_user_source": latest_user_source,
@@ -687,18 +698,51 @@ class PromptBuilder:
         startup_contract = snapshot.get("startup_contract")
         if isinstance(startup_contract, dict):
             value = str(startup_contract.get("custom_profile") or "").strip().lower()
-            if value in {"continue_existing_state", "revision_rebuttal", "freeform"}:
+            if value in {"continue_existing_state", "review_audit", "revision_rebuttal", "freeform"}:
                 return value
         return "freeform"
 
+    @staticmethod
+    def _baseline_execution_policy(snapshot: dict) -> str:
+        startup_contract = snapshot.get("startup_contract")
+        if isinstance(startup_contract, dict):
+            value = str(startup_contract.get("baseline_execution_policy") or "").strip().lower()
+            if value in {"auto", "must_reproduce_or_verify", "reuse_existing_only", "skip_unless_blocking"}:
+                return value
+        return "auto"
+
+    @staticmethod
+    def _review_followup_policy(snapshot: dict) -> str:
+        startup_contract = snapshot.get("startup_contract")
+        if isinstance(startup_contract, dict):
+            value = str(startup_contract.get("review_followup_policy") or "").strip().lower()
+            if value in {"audit_only", "auto_execute_followups", "user_gated_followups"}:
+                return value
+        return "audit_only"
+
+    @staticmethod
+    def _manuscript_edit_mode(snapshot: dict) -> str:
+        startup_contract = snapshot.get("startup_contract")
+        if isinstance(startup_contract, dict):
+            value = str(startup_contract.get("manuscript_edit_mode") or "").strip().lower()
+            if value in {"none", "copy_ready_text", "latex_required"}:
+                return value
+        return "none"
+
     def _research_delivery_policy_block(self, snapshot: dict) -> str:
         need_research_paper = self._need_research_paper(snapshot)
         launch_mode = self._launch_mode(snapshot)
         custom_profile = self._custom_profile(snapshot)
+        baseline_execution_policy = self._baseline_execution_policy(snapshot)
+        review_followup_policy = self._review_followup_policy(snapshot)
+        manuscript_edit_mode = self._manuscript_edit_mode(snapshot)
         lines = [
             f"- need_research_paper: {need_research_paper}",
             f"- launch_mode: {launch_mode}",
             f"- custom_profile: {custom_profile if launch_mode == 'custom' else 'n/a'}",
+            f"- review_followup_policy: {review_followup_policy if custom_profile == 'review_audit' else 'n/a'}",
+            f"- baseline_execution_policy: {baseline_execution_policy if launch_mode == 'custom' else 'n/a'}",
+            f"- manuscript_edit_mode: {manuscript_edit_mode if custom_profile in {'review_audit', 'revision_rebuttal'} else 'n/a'}",
             f"- delivery_mode: {'paper_required' if need_research_paper else 'algorithm_first'}",
             "- idea_stage_rule: every accepted idea submission should normally create a new branch/worktree and a new user-visible research node.",
             "- idea_draft_rule: before `artifact.submit_idea(...)`, first finish a concise durable Markdown draft for the chosen route; keep `idea.md` compact and `draft.md` richer.",
@@ -713,7 +757,7 @@ class PromptBuilder:
             lines.extend(
                 [
                     "- custom_launch_rule: do not force the canonical full-research path when the custom startup contract is narrower.",
-                    "- custom_context_rule: treat `entry_state_summary`, `review_summary`, and `custom_brief` as active runtime context rather than decorative metadata.",
+                    "- custom_context_rule: treat `entry_state_summary`, `review_summary`, `review_materials`, and `custom_brief` as active runtime context rather than decorative metadata.",
                 ]
             )
             if custom_profile == "continue_existing_state":
@@ -723,6 +767,31 @@ class PromptBuilder:
                         "- reuse_first_rule: trust-rank and reconcile existing assets before deciding to rerun anything costly.",
                     ]
                 )
+            elif custom_profile == "review_audit":
+                lines.extend(
+                    [
+                        "- review_entry_rule: treat the current draft/paper state as the active contract; open `review` before more writing or finalization.",
+                        "- review_routing_rule: if that audit finds real evidence gaps, route to `analysis-campaign`, `baseline`, `scout`, or `write` instead of polishing blindly.",
+                    ]
+                )
+                if review_followup_policy == "auto_execute_followups":
+                    lines.extend(
+                        [
+                            "- review_followup_rule: after the audit artifacts are durable, continue automatically into the required experiments, manuscript deltas, and review-closure work instead of stopping at the audit report.",
+                        ]
+                    )
+                elif review_followup_policy == "user_gated_followups":
+                    lines.extend(
+                        [
+                            "- review_followup_rule: after the audit artifacts are durable, package the next expensive follow-up step into one structured decision instead of continuing silently.",
+                        ]
+                    )
+                else:
+                    lines.extend(
+                        [
+                            "- review_followup_rule: stop after the durable audit artifacts and route recommendation unless the user later asks for execution follow-up.",
+                        ]
+                    )
             elif custom_profile == "revision_rebuttal":
                 lines.extend(
                     [
@@ -736,6 +805,36 @@ class PromptBuilder:
                         "- freeform_entry_rule: prefer the custom brief over the default stage order and open only the skills actually needed.",
                     ]
                 )
+            if baseline_execution_policy == "must_reproduce_or_verify":
+                lines.extend(
+                    [
+                        "- baseline_execution_rule: before reviewer-linked follow-up work, explicitly verify or recover the rebuttal-critical baseline/comparator instead of assuming the stored evidence is still trustworthy.",
+                    ]
+                )
+            elif baseline_execution_policy == "reuse_existing_only":
+                lines.extend(
+                    [
+                        "- baseline_execution_rule: prefer the existing trusted baseline/results and do not rerun them unless you find concrete inconsistency, corruption, or missing-evidence problems.",
+                    ]
+                )
+            elif baseline_execution_policy == "skip_unless_blocking":
+                lines.extend(
+                    [
+                        "- baseline_execution_rule: do not spend time on baseline reruns by default; only open `baseline` if a named review/rebuttal issue truly depends on a missing comparator or unusable prior evidence.",
+                    ]
+                )
+            if manuscript_edit_mode == "latex_required":
+                lines.extend(
+                    [
+                        "- manuscript_edit_rule: when manuscript revision is needed, treat the provided LaTeX tree or `paper/latex/` as the authoritative writing surface; if LaTeX source is unavailable, produce LaTeX-ready replacement text and make that blocker explicit instead of pretending the manuscript was edited.",
+                    ]
+                )
+            elif manuscript_edit_mode == "copy_ready_text":
+                lines.extend(
+                    [
+                        "- manuscript_edit_rule: when manuscript revision is needed, provide section-level copy-ready replacement text and explicit deltas even if no LaTeX source is available.",
+                    ]
+                )
         if need_research_paper:
             lines.extend(
                 [
@@ -783,7 +882,10 @@ class PromptBuilder:
             "- interaction_protocol: first message may be plain conversation; after that, treat artifact.interact threads and mailbox polls as the main continuity spine across TUI, web, and connectors",
             "- mailbox_protocol: artifact.interact(include_recent_inbound_messages=True) is the queued human-message mailbox; when it returns user text, treat that input as higher priority than background subtasks until it has been acknowledged",
             "- acknowledgment_protocol: after artifact.interact returns any human message, immediately send one substantive artifact.interact(...) follow-up; if the active connector runtime already emitted a transport-level receipt acknowledgement, do not send a redundant receipt-only message; if answerable, answer directly, otherwise state the short plan, nearest checkpoint, and that the current background subtask is paused",
-            "- progress_protocol: emit artifact.interact(kind='progress', reply_mode='threaded', ...) at real human-meaningful checkpoints; if no natural checkpoint appears during active user-relevant work, prefer a concise keepalive once work has crossed roughly 10 tool calls with a human-meaningful delta, and do not drift beyond roughly 20 tool calls or about 15 minutes without a user-visible update",
+            "- progress_protocol: emit artifact.interact(kind='progress', reply_mode='threaded', ...) at real human-meaningful checkpoints; if no natural checkpoint appears during active user-relevant work, prefer a concise keepalive once work has crossed roughly 6 tool calls with a human-meaningful delta, and do not drift beyond roughly 12 tool calls or about 8 minutes without a user-visible update",
+            "- stage_kickoff_protocol: after entering any stage or companion skill, send one user-visible artifact.interact progress update within the first 3 tool calls of substantial work",
+            "- read_plan_keepalive_protocol: if work is still mostly reading, searching, comparison, or planning, do not wait too long for a 'big result'; send one concise user-visible checkpoint after about 5 consecutive tool calls if the user would otherwise see silence",
+            "- subtask_boundary_protocol: send a user-visible update whenever the active subtask changes materially, especially across intake -> audit, audit -> experiment planning, experiment planning -> run launch, run result -> drafting, or drafting -> review/rebuttal",
             "- smoke_then_detach_protocol: for baseline reproduction, main experiments, and analysis experiments, first validate the command path with a bounded smoke test; once the smoke test passes, launch the real long run with bash_exec(mode='detach', ...) and usually leave timeout_seconds unset rather than guessing a fake deadline",
             "- progress_first_monitoring_protocol: when supervising a long-running bash_exec session, judge health by forward progress rather than by whether the final artifact has already appeared within a short window",
             "- delta_monitoring_protocol: compare deltas such as new sample counters, new task counters, new saved files, new last_output_seq values, or changed last_progress payloads; if any of these move forward, treat the run as alive and keep observing",
@@ -804,7 +906,7 @@ class PromptBuilder:
             "- respect_protocol: write user-facing updates as natural, respectful, easy-to-follow chat; do not sound like a formal status report or internal tool log",
             "- omission_protocol: for ordinary user-facing updates, omit file paths, artifact ids, branch/worktree ids, session ids, raw commands, raw logs, and internal tool names unless the user asked for them or needs them to act",
             "- compaction_protocol: ordinary artifact.interact progress updates should usually fit in 2 to 4 short sentences and should not read like a monitoring transcript or execution diary",
-            "- tool_call_keepalive_protocol: for active multi-step work outside long detached experiment waits, prefer sending one concise artifact.interact progress update after roughly 10 tool calls when there is already a human-meaningful delta, and do not exceed roughly 20 tool calls or about 15 minutes without a user-visible checkpoint",
+            "- tool_call_keepalive_protocol: for active multi-step work outside long detached experiment waits, prefer sending one concise artifact.interact progress update after roughly 6 tool calls when there is already a human-meaningful delta, and do not exceed roughly 12 tool calls or about 8 minutes without a user-visible checkpoint",
             "- human_progress_shape_protocol: ordinary progress updates should usually make three things explicit in human language: the current task, the main difficulty or latest real progress, and the concrete next measure you will take",
             "- milestone_graduation_protocol: keep ordinary subtask completions concise; upgrade to a richer milestone report only when a stage-significant deliverable or route-changing checkpoint becomes durably true",
             "- eta_visibility_protocol: for baseline reproduction, main experiments, analysis experiments, and other important long-running phases, progress updates should also make the expected time to the next meaningful result, next milestone, or next user-visible update explicit; use roughly 10 to 30 minutes as the normal update window, and if the ETA is unreliable, say that and give a realistic next check-in window instead",

package/src/deepscientist/quest/service.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
 import copy
+from collections import deque
 from contextlib import contextmanager
+from datetime import UTC, datetime, timedelta
 import hashlib
 import subprocess
 import json
@@ -23,7 +25,7 @@ from ..connector_runtime import conversation_identity_key, normalize_conversatio
 from ..gitops import current_branch, export_git_graph, head_commit, init_repo
 from ..home import repo_root
 from ..registries import BaselineRegistry
-from ..shared import append_jsonl, ensure_dir, generate_id, read_json, read_jsonl, read_text, read_yaml, resolve_within, run_command, sha256_text, slugify, utc_now, write_json, write_text, write_yaml
+from ..shared import append_jsonl, ensure_dir, generate_id, iter_jsonl, read_json, read_jsonl, read_jsonl_tail, read_text, read_yaml, resolve_within, run_command, sha256_text, slugify, utc_now, write_json, write_text, write_yaml
 from ..skills import SkillInstaller
 from ..web_search import extract_web_search_payload
 from .layout import (
@@ -42,6 +44,126 @@ _UNSET = object()
 _NUMERIC_QUEST_ID_PATTERN = re.compile(r"^\d{1,10}$")
 _MAX_NUMERIC_QUEST_ID_VALUE = 9_999_999_999
 _NUMERIC_QUEST_ID_PAD_WIDTH = 3
+_CRASH_AUTO_RESUME_WINDOW = timedelta(hours=24)
+_JSONL_CACHE_MAX_BYTES = 4 * 1024 * 1024
+_CODEX_HISTORY_TAIL_LIMIT = 400
+_JSONL_STREAM_CHUNK_BYTES = 64 * 1024
+_EVENTS_OVERSIZED_LINE_BYTES = 8 * 1024 * 1024
+_OVERSIZED_EVENT_PREFIX_BYTES = 4096
+_EVENT_TYPE_BYTES_RE = re.compile(rb'"(?:type|event_type)"\s*:\s*"([^"]+)"')
+_EVENT_TOOL_NAME_BYTES_RE = re.compile(rb'"tool_name"\s*:\s*"([^"]+)"')
+_EVENT_RUN_ID_BYTES_RE = re.compile(rb'"run_id"\s*:\s*"([^"]+)"')
+
+
+def _oversized_event_placeholder(*, prefix: bytes, line_bytes: int) -> dict[str, Any]:
+    def _extract(pattern: re.Pattern[bytes]) -> str | None:
+        match = pattern.search(prefix)
+        if match is None:
+            return None
+        try:
+            return match.group(1).decode("utf-8", errors="ignore").strip() or None
+        except Exception:
+            return None
+
+    event_type = _extract(_EVENT_TYPE_BYTES_RE) or "runner.tool_result"
+    tool_name = _extract(_EVENT_TOOL_NAME_BYTES_RE)
+    run_id = _extract(_EVENT_RUN_ID_BYTES_RE)
+    summary = f"Omitted oversized quest event payload ({line_bytes} bytes) while reading event history."
+    payload: dict[str, Any] = {
+        "type": event_type,
+        "status": "omitted",
+        "summary": summary,
+        "oversized_event": True,
+        "oversized_bytes": line_bytes,
+    }
+    if tool_name:
+        payload["tool_name"] = tool_name
+    if run_id:
+        payload["run_id"] = run_id
+    return payload
+
+
+def _iter_jsonl_records_safely(
+    path: Path,
+    *,
+    oversized_line_bytes: int = _EVENTS_OVERSIZED_LINE_BYTES,
+):
+    if not path.exists():
+        return
+    with path.open("rb") as handle:
+        buffer = bytearray()
+        prefix = bytearray()
+        current_bytes = 0
+        oversized = False
+        while True:
+            chunk = handle.read(_JSONL_STREAM_CHUNK_BYTES)
+            if not chunk:
+                break
+            start = 0
+            while start <= len(chunk):
+                newline_index = chunk.find(b"\n", start)
+                has_newline = newline_index >= 0
+                segment = chunk[start:newline_index] if has_newline else chunk[start:]
+
+                if oversized:
+                    current_bytes += len(segment)
+                    if has_newline:
+                        yield _oversized_event_placeholder(prefix=bytes(prefix), line_bytes=current_bytes)
+                        prefix = bytearray()
+                        current_bytes = 0
+                        oversized = False
+                        start = newline_index + 1
+                        continue
+                    break
+
+                next_bytes = current_bytes + len(segment)
+                if next_bytes > oversized_line_bytes:
+                    combined_prefix = bytes(buffer)
+                    remaining = max(0, _OVERSIZED_EVENT_PREFIX_BYTES - len(combined_prefix))
+                    if remaining:
+                        combined_prefix += segment[:remaining]
+                    prefix = bytearray(combined_prefix)
+                    buffer.clear()
+                    current_bytes = next_bytes
+                    oversized = True
+                    if has_newline:
+                        yield _oversized_event_placeholder(prefix=bytes(prefix), line_bytes=current_bytes)
+                        prefix = bytearray()
+                        current_bytes = 0
+                        oversized = False
+                        start = newline_index + 1
+                        continue
+                    break
+
+                buffer.extend(segment)
+                current_bytes = next_bytes
+                if has_newline:
+                    raw = bytes(buffer).strip()
+                    buffer.clear()
+                    line_bytes = current_bytes
+                    current_bytes = 0
+                    if raw:
+                        try:
+                            payload = json.loads(raw)
+                        except json.JSONDecodeError:
+                            payload = None
+                        if isinstance(payload, dict):
+                            yield payload
+                    start = newline_index + 1
+                    continue
+                break
+
+        if oversized:
+            yield _oversized_event_placeholder(prefix=bytes(prefix), line_bytes=current_bytes)
+        elif buffer:
+            raw = bytes(buffer).strip()
+            if raw:
+                try:
+                    payload = json.loads(raw)
+                except json.JSONDecodeError:
+                    payload = None
+                if isinstance(payload, dict):
+                    yield payload
 
 
 class QuestService:
@@ -808,21 +930,15 @@ class QuestService:
             getattr(stat, "st_mtime_ns", int(stat.st_mtime * 1_000_000_000)),
             stat.st_size,
         )
+        if stat.st_size > _JSONL_CACHE_MAX_BYTES:
+            with self._jsonl_cache_lock:
+                self._jsonl_cache.pop(cache_key, None)
+            return read_jsonl(path)
         with self._jsonl_cache_lock:
             cached = self._jsonl_cache.get(cache_key)
             if cached and cached.get("state") == state:
                 return cached.get("records") or []
-        items: list[dict[str, Any]] = []
-        for line in path.read_text(encoding="utf-8").splitlines():
-            line = line.strip()
-            if not line:
-                continue
-            try:
-                payload = json.loads(line)
-            except json.JSONDecodeError:
-                continue
-            if isinstance(payload, dict):
-                items.append(payload)
+        items = read_jsonl(path)
         with self._jsonl_cache_lock:
             self._jsonl_cache[cache_key] = {
                 "state": state,
@@ -830,6 +946,57 @@ class QuestService:
             }
         return items
 
+    @staticmethod
+    def _read_jsonl_cursor_slice(
+        path: Path,
+        *,
+        after: int = 0,
+        before: int | None = None,
+        limit: int = 200,
+        tail: bool = False,
+    ) -> tuple[list[tuple[int, dict[str, Any]]], int, bool]:
+        normalized_limit = max(int(limit or 0), 0)
+        if not path.exists():
+            return [], 0, False
+        if normalized_limit <= 0:
+            total = sum(1 for _ in _iter_jsonl_records_safely(path))
+            return [], total, False
+
+        if before is not None:
+            stop_cursor = max(int(before) - 1, 0)
+            window: deque[tuple[int, dict[str, Any]]] = deque(maxlen=normalized_limit)
+            total = 0
+            for payload in _iter_jsonl_records_safely(path):
+                total += 1
+                if total >= before:
+                    break
+                window.append((total, payload))
+            has_more = bool(window and window[0][0] > 1)
+            return list(window), total, has_more
+
+        if tail:
+            window = deque(maxlen=normalized_limit)
+            total = 0
+            for payload in _iter_jsonl_records_safely(path):
+                total += 1
+                window.append((total, payload))
+            has_more = total > len(window)
+            return list(window), total, has_more
+
+        collected: list[tuple[int, dict[str, Any]]] = []
+        total = 0
+        saw_more = False
+        normalized_after = max(int(after or 0), 0)
+        for payload in _iter_jsonl_records_safely(path):
+            total += 1
+            if total <= normalized_after:
+                continue
+            if len(collected) < normalized_limit:
+                collected.append((total, payload))
+                continue
+            saw_more = True
+        return collected, total, saw_more
+
     @staticmethod
     def _path_state(path: Path) -> tuple[int, int, int] | None:
         if not path.exists():
@@ -1592,6 +1759,12 @@ class QuestService:
             if not active_run_id and status != "running":
                 continue
             previous_status = status or "running"
+            last_transition_at = self._runtime_recovery_timestamp(runtime_state, quest_data)
+            recoverable = self._runtime_recovery_eligible(
+                previous_status=previous_status,
+                active_run_id=active_run_id or None,
+                last_transition_at=last_transition_at,
+            )
             self.update_runtime_state(
                 quest_root=quest_root,
                 status="stopped",
@@ -1602,6 +1775,8 @@ class QuestService:
                 f"Recovered quest from stale runtime state; previous status `{previous_status}`"
                 + (f", abandoned run `{active_run_id}`." if active_run_id else ".")
             )
+            if recoverable:
+                summary = f"{summary} Auto-resume is eligible within the 24-hour recovery window."
             append_jsonl(
                 quest_root / ".ds" / "events.jsonl",
                 {
@@ -1610,6 +1785,8 @@ class QuestService:
                     "quest_id": quest_root.name,
                     "previous_status": previous_status,
                     "abandoned_run_id": active_run_id or None,
+                    "last_transition_at": last_transition_at,
+                    "recoverable": recoverable,
                     "status": "stopped",
                     "summary": summary,
                     "created_at": utc_now(),
@@ -1620,11 +1797,53 @@ class QuestService:
                     "quest_id": quest_root.name,
                     "previous_status": previous_status,
                     "abandoned_run_id": active_run_id or None,
+                    "last_transition_at": last_transition_at,
+                    "recoverable": recoverable,
                     "status": "stopped",
                 }
             )
         return reconciled
 
+    @staticmethod
+    def _parse_runtime_timestamp(value: Any) -> datetime | None:
+        normalized = str(value or "").strip()
+        if not normalized:
+            return None
+        candidate = normalized.replace("Z", "+00:00")
+        try:
+            parsed = datetime.fromisoformat(candidate)
+        except ValueError:
+            return None
+        if parsed.tzinfo is None:
+            parsed = parsed.replace(tzinfo=UTC)
+        return parsed.astimezone(UTC)
+
+    def _runtime_recovery_timestamp(self, runtime_state: dict[str, Any], quest_data: dict[str, Any]) -> str | None:
+        for candidate in (
+            runtime_state.get("last_transition_at"),
+            quest_data.get("updated_at"),
+            quest_data.get("created_at"),
+        ):
+            parsed = self._parse_runtime_timestamp(candidate)
+            if parsed is None:
+                continue
+            return parsed.isoformat()
+        return None
+
+    def _runtime_recovery_eligible(
+        self,
+        *,
+        previous_status: str,
+        active_run_id: str | None,
+        last_transition_at: str | None,
+    ) -> bool:
+        if previous_status != "running" and not str(active_run_id or "").strip():
+            return False
+        parsed = self._parse_runtime_timestamp(last_transition_at)
+        if parsed is None:
+            return False
+        return datetime.now(UTC) - parsed <= _CRASH_AUTO_RESUME_WINDOW
+
     def history(self, quest_id: str, limit: int = 100) -> list[dict]:
         return self._read_cached_jsonl(self._quest_root(quest_id) / ".ds" / "conversations" / "main.jsonl")[-limit:]
 
@@ -1730,40 +1949,37 @@ class QuestService:
         limit: int = 200,
         tail: bool = False,
     ) -> dict:
-        records = self._read_cached_jsonl(self._quest_root(quest_id) / ".ds" / "events.jsonl")
+        event_path = self._quest_root(quest_id) / ".ds" / "events.jsonl"
         normalized_limit = max(limit, 0)
         direction = "after"
         if before is not None:
             direction = "before"
-            end = max(int(before) - 1, 0)
-            start = max(end - normalized_limit, 0)
-            sliced = records[start:end]
         elif tail and normalized_limit > 0:
             direction = "tail"
-            start = max(len(records) - normalized_limit, 0)
-            sliced = records[start : start + normalized_limit]
-        else:
-            start = max(after, 0)
-            sliced = records[start : start + normalized_limit]
+        sliced_records, total_records, has_more = self._read_jsonl_cursor_slice(
+            event_path,
+            after=after,
+            before=before,
+            limit=normalized_limit,
+            tail=tail,
+        )
         enriched = []
-        for index, item in enumerate(sliced, start=start + 1):
+        for cursor, item in sliced_records:
             enriched.append(
                 {
-                    "cursor": index,
-                    "event_id": item.get("event_id") or f"evt-{quest_id}-{index}",
+                    "cursor": cursor,
+                    "event_id": item.get("event_id") or f"evt-{quest_id}-{cursor}",
                     **item,
                 }
             )
         if before is not None:
-            next_cursor = start + len(sliced)
+            next_cursor = enriched[-1]["cursor"] if enriched else max(min(int(before or 0) - 1, total_records), 0)
+        elif tail:
+            next_cursor = total_records
         else:
-            next_cursor = len(records) if tail else start + len(sliced)
+            next_cursor = enriched[-1]["cursor"] if enriched else max(int(after or 0), 0)
         oldest_cursor = enriched[0]["cursor"] if enriched else None
         newest_cursor = enriched[-1]["cursor"] if enriched else None
-        if before is not None:
-            has_more = start > 0
-        else:
-            has_more = start > 0 if tail else next_cursor < len(records)
         return {
             "quest_id": quest_id,
             "cursor": next_cursor,
@@ -3705,7 +3921,7 @@ def _parse_codex_history(history_root: Path, *, quest_id: str, run_id: str, skil
     entries: list[dict] = []
     known_tool_names: dict[str, str] = {}
 
-    for raw in read_jsonl(history_path):
+    for raw in read_jsonl_tail(history_path, _CODEX_HISTORY_TAIL_LIMIT):
         timestamp = raw.get("timestamp")
         event = raw.get("event")
         if not isinstance(event, dict):