dsh-compound-engineering 0.0.0-stage → 0.1.0

package/skills/ce-compound/scripts/session-history/extract-errors.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+"""Extract error signals from a DeepSeek Harness (dsh) session file.
+
+Usage:
+  python3 extract-errors.py <session-file> [--output PATH]
+  cat <session.jsonl> | python3 extract-errors.py
+
+Passing the session path instead of piping through stdin is required for dsh
+sessions, which are zstd-compressed containers of concatenated frames; the path
+form decodes them through the bundled `dsh_session` reader. The stdin form only
+works for already-decompressed JSONL (e.g. a dsh session piped through the
+reader), and reports no errors when the stream is not dsh-shaped.
+
+Finds failed tool calls / commands and outputs them with timestamps.
+
+When --output PATH is given, the extracted error log is written to PATH and
+stdout receives only a one-line JSON status (_meta with wrote/bytes/stats).
+This lets callers route bulk content to a scratch file without round-tripping
+extraction bytes through orchestrator tool results.
+
+Without --output, extracted content goes to stdout and ends with a _meta line.
+"""
+import argparse
+import io
+import os
+import sys
+import json
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+import dsh_session  # noqa: E402
+
+parser = argparse.ArgumentParser(add_help=True)
+parser.add_argument(
+    "session_file",
+    nargs="?",
+    help="Session file to read. Omit to read JSONL from stdin. Required for compressed dsh sessions.",
+)
+parser.add_argument(
+    "--output",
+    metavar="PATH",
+    help="Write extracted errors to PATH instead of stdout. Stdout receives a one-line _meta status.",
+)
+args = parser.parse_args()
+
+# Session logs are UTF-8; force UTF-8 I/O so Windows' cp1252 default can't crash
+# on non-Latin-1 content like emoji, smart quotes, or box-drawing (#1258).
+# newline="\n" keeps bash-side consumers of our output (tr/xargs/grep) from
+# seeing a Windows CRLF as part of the last field on every line.
+sys.stdin.reconfigure(encoding="utf-8", errors="replace")
+sys.stdout.reconfigure(encoding="utf-8", errors="replace", newline="\n")
+
+_original_stdout = sys.stdout
+if args.output:
+    sys.stdout = io.StringIO()
+
+stats = {"lines": 0, "parse_errors": 0, "errors_found": 0}
+
+# Top-level record types only dsh emits. Anything else in the stream is a
+# non-dsh or framework record we deliberately ignore.
+_DSH_RECORD_TYPES = ("user/message", "assistant/message", "tool/result", "step/start")
+
+
+def summarize_error(raw):
+    """Extract a short error summary instead of dumping the full payload."""
+    text = str(raw).strip()
+    # Take the first non-empty line as the error message
+    for line in text.split("\n"):
+        line = line.strip()
+        if line:
+            return line[:200]
+    return text[:200]
+
+
+def _dsh_result_text(content):
+    """Flatten a dsh tool-result payload into a one-line summary."""
+    if isinstance(content, str):
+        return summarize_error(content)
+    if isinstance(content, list):
+        parts = []
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == "text":
+                parts.append(block.get("text", ""))
+            elif isinstance(block, str):
+                parts.append(block)
+        return summarize_error("\n".join(parts))
+    return summarize_error(content)
+
+
+def handle_dsh(obj):
+    if obj.get("type") == "tool/result":
+        data = obj.get("data", {})
+        message = data.get("message", {})
+        content = message.get("content", [])
+        is_error = False
+        summary = ""
+        if isinstance(content, list):
+            for block in content:
+                if isinstance(block, dict) and block.get("type") == "tool-result":
+                    if block.get("isError"):
+                        is_error = True
+                        summary = _dsh_result_text(block.get("content"))
+        # dsh also records structured failures outside the content blocks.
+        error = data.get("error")
+        if isinstance(error, dict):
+            is_error = True
+            if not summary:
+                summary = summarize_error(f"{error.get('name', 'error')}: {error.get('code', '')}")
+        if not is_error:
+            return
+        ts = dsh_session.iso_from_millis(obj.get("time"))[:19]
+        call_id = message.get("source", {}).get("callId", "")
+        print(f"[{ts}] [error] tool_call={call_id}: {summary}")
+        print("---")
+        stats["errors_found"] += 1
+
+
+def is_dsh_record(obj):
+    """Report whether a parsed record can only come from a dsh session."""
+    t = obj.get("type")
+    if t in _DSH_RECORD_TYPES:
+        return True
+    # The session header is dsh only when it stamps createdAt in epoch millis;
+    # other harnesses open with an ISO `timestamp` string instead.
+    return t == "session" and isinstance(obj.get("createdAt"), (int, float))
+
+
+# Read the whole stream once: dsh decoding is not seekable, and error detection
+# needs the full record set anyway.
+objects = []
+_source = None
+_close_source = False
+if args.session_file:
+    if dsh_session.is_session_file(args.session_file):
+        _source = dsh_session.read_lines(args.session_file)
+    elif dsh_session.available():
+        # A path we cannot decode as a dsh session has no dsh errors to report.
+        _source = []
+    else:
+        _source = open(args.session_file, "r", encoding="utf-8", errors="replace")
+        _close_source = True
+else:
+    _source = sys.stdin
+
+for line in _source:
+    line = line.strip()
+    if not line:
+        continue
+    stats["lines"] += 1
+    try:
+        obj = json.loads(line)
+    except (json.JSONDecodeError, KeyError):
+        stats["parse_errors"] += 1
+        continue
+    objects.append(obj)
+
+if hasattr(_source, "close"):
+    _source.close()
+
+# Only stream dsh records; a non-dsh session yields no findings rather than
+# misparsed noise from another harness's schema.
+for obj in objects:
+    if not is_dsh_record(obj):
+        continue
+    try:
+        handle_dsh(obj)
+    except KeyError:
+        stats["parse_errors"] += 1
+
+print(json.dumps({"_meta": True, **stats}))
+
+if args.output:
+    body = sys.stdout.getvalue()
+    sys.stdout = _original_stdout
+    with open(args.output, "w", encoding="utf-8", newline="\n") as f:
+        f.write(body)
+    bytes_written = os.path.getsize(args.output)
+    print(json.dumps({"_meta": True, "wrote": args.output, "bytes": bytes_written, **stats}))

package/skills/ce-compound/scripts/session-history/extract-metadata.py

@@ -0,0 +1,301 @@
+#!/usr/bin/env python3
+"""Extract session metadata from DeepSeek Harness (dsh) session files.
+
+Batch mode (preferred — one invocation for all files):
+  python3 extract-metadata.py /path/to/dir/session-*.jsonl.zstd
+  python3 extract-metadata.py file1.jsonl.zstd file2.jsonl.zstd
+
+Single-file mode (stdin):
+  head -20 <session.jsonl> | python3 extract-metadata.py
+
+Detects dsh sessions from the JSONL structure. Non-dsh files are skipped.
+Outputs one JSON object per file, one per line.
+Includes a final _meta line with processing stats.
+
+dsh sessions are zstd-compressed (`session*.jsonl.zstd`, one file per session
+holding concatenated frames); they are decoded through `dsh_session.read_lines`
+so every frame is read, not just the first. Decoding needs Python 3.14+
+(`compression.zstd`); on older interpreters every file reports as unmatched.
+"""
+import sys
+import json
+import os
+
+# LF-only line endings: callers may pipe our JSON lines through bash tools
+# (tr/xargs/grep) where a Windows CRLF leaves a stray \r on every record.
+sys.stdout.reconfigure(encoding="utf-8", errors="replace", newline="\n")
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+import dsh_session  # noqa: E402
+
+MAX_LINES = 200  # Resumed dsh sessions front-load dozens of non-message records before the first user record
+
+
+def try_dsh(lines):
+    """DeepSeek Harness sessions.
+
+    The dsh header is `{"type": "session", "cwd": ..., "createdAt": <epoch ms>}`.
+    `createdAt` in epoch milliseconds is the field that separates a dsh header
+    from any other harness that opens with a cwd-bearing session record.
+    """
+    for line in lines:
+        try:
+            obj = json.loads(line.strip())
+        except (json.JSONDecodeError, KeyError):
+            continue
+        if obj.get("type") != "session" or "cwd" not in obj:
+            continue
+        if not isinstance(obj.get("createdAt"), (int, float)):
+            return None
+        return {
+            "platform": "dsh",
+            "cwd": obj.get("cwd", ""),
+            "session": obj.get("id", ""),
+            "ts": dsh_session.iso_from_millis(obj.get("createdAt")),
+        }
+    return None
+
+
+def extract_from_lines(lines):
+    return try_dsh(lines)
+
+
+def get_last_timestamp(filepath):
+    """Return the ISO time of the last timestamped record in a dsh session."""
+    objects = dsh_session.read_objects(filepath)
+    for obj in reversed(objects):
+        stamp = dsh_session.iso_from_millis(obj.get("time"))
+        if stamp:
+            return stamp
+    return None
+
+
+def _extract_user_assistant_text(filepath):
+    """Return concatenated user + assistant text content from a dsh session file.
+
+    Skips JSONL metadata field names and values (session id, timestamps, type
+    tags), tool calls, tool results, and thinking blocks. Only content the user
+    or the model actually said is included.
+
+    Without this filtering, common topic words like "session" would match every
+    session file via its id field, drowning out real content matches.
+    """
+    chunks = []
+    if not dsh_session.is_session_file(filepath):
+        return ""
+    try:
+        objects = []
+        # dsh sessions are zstd-compressed and hold concatenated frames, so
+        # plain line iteration would yield binary garbage.
+        for line in dsh_session.read_lines(filepath):
+            try:
+                objects.append(json.loads(line.strip()))
+            except (json.JSONDecodeError, ValueError):
+                continue
+
+        for obj in objects:
+            t = obj.get("type")
+
+            # dsh: slash-tagged top-level records. User text lives at
+            # data.content[], assistant text at data.message.content[].
+            # dsh also stores framework context (runtime snapshots, the skill
+            # catalog, timestamps) as user rows, so only `source.kind == "user"`
+            # counts as something the person said.
+            if t == "user/message":
+                source = obj.get("data", {}).get("source")
+                if isinstance(source, dict) and source.get("kind") not in ("user", "goal"):
+                    continue
+                chunks.extend(dsh_session.content_text(obj.get("data", {}).get("content")))
+                continue
+            if t == "assistant/message":
+                message = obj.get("data", {}).get("message", {})
+                chunks.extend(dsh_session.content_text(message.get("content")))
+                continue
+    except (OSError, IOError):
+        pass
+    return "\n".join(chunks)
+
+
+def count_keyword_matches(filepath, keywords):
+    """Case-insensitive substring count for each keyword in user/assistant text.
+
+    Returns a dict {original_keyword: count}. Scans only content the user or
+    assistant said — not JSONL metadata, tool calls, tool outputs, or thinking
+    blocks — so common topic words like "session" do not false-match against
+    the session id field.
+    """
+    text_lower = _extract_user_assistant_text(filepath).lower()
+    return {kw: text_lower.count(kw.lower()) for kw in keywords}
+
+
+def process_file(filepath):
+    """Extract metadata only. Keyword scanning is done separately so callers
+    can apply cheap filters (e.g. --cwd-filter) before paying the full-file
+    content scan cost.
+
+    Returns (result, error): (dict, None) on success, (None, None) when the
+    file is readable but is not a dsh session, (None, filepath) on read error."""
+    try:
+        size = os.path.getsize(filepath)
+        if not dsh_session.is_session_file(filepath):
+            return None, None
+        if not dsh_session.available():
+            return None, None
+        lines = dsh_session.read_lines(filepath, MAX_LINES)
+        result = extract_from_lines(lines)
+        if result:
+            result["file"] = filepath
+            result["size"] = size
+            return result, None
+        else:
+            return None, None
+    except (OSError, IOError) as e:
+        return None, filepath
+
+
+def _normalize_cwd(path):
+    path = path.replace("\\", "/").strip()
+    if len(path) > 1 and path.endswith("/") and not (
+        len(path) == 3 and path[1] == ":"
+    ):
+        path = path.rstrip("/")
+    # Windows drive paths are case-insensitive. Do not rewrite POSIX /d/...
+    # to D:/ — those are different filesystems.
+    if len(path) >= 2 and path[1] == ":":
+        path = path[0].upper() + ":" + path[2:].casefold()
+    return path
+
+
+def _is_abs_cwd(path):
+    if os.path.isabs(path):
+        return True
+    return len(path) >= 2 and path[1] == ":"
+
+
+def _cwd_paths_related(session_cwd, cwd_filter):
+    if session_cwd == cwd_filter:
+        return True
+    prefix = cwd_filter if cwd_filter.endswith("/") else cwd_filter + "/"
+    if session_cwd.startswith(prefix):
+        return True
+    prefix = session_cwd if session_cwd.endswith("/") else session_cwd + "/"
+    return cwd_filter.startswith(prefix)
+
+
+def _attach_timestamps(result, filepath):
+    # dsh already reported `createdAt` as ISO 8601; the last record's `time`
+    # is read separately because decompression is not seekable.
+    last_ts = get_last_timestamp(filepath)
+    if last_ts:
+        result["last_ts"] = last_ts
+
+
+def cwd_matches_filter(session_cwd, cwd_filter):
+    if not session_cwd or not cwd_filter:
+        return True
+    session = _normalize_cwd(session_cwd)
+    filt = _normalize_cwd(cwd_filter)
+    if not session or not filt:
+        return True
+    if _is_abs_cwd(filt):
+        return _cwd_paths_related(session, filt)
+    return filt in session.split("/")
+
+
+# Parse arguments: files and optional --cwd-filter / --keyword
+files = []
+cwd_filter = None
+keywords = None
+args = sys.argv[1:]
+i = 0
+while i < len(args):
+    if args[i] == "--cwd-filter" and i + 1 < len(args):
+        cwd_filter = args[i + 1]
+        i += 2
+    elif args[i] == "--keyword" and i + 1 < len(args):
+        keywords = [k for k in args[i + 1].split(",") if k]
+        i += 2
+    elif not args[i].startswith("-"):
+        files.append(args[i])
+        i += 1
+    else:
+        i += 1
+
+if files:
+    # Batch mode: process all files
+    processed = 0
+    parse_errors = 0
+    no_metadata = 0
+    filtered = 0
+    matched = 0
+    for filepath in files:
+        if not dsh_session.is_session_file(filepath):
+            continue
+        result, error = process_file(filepath)
+        processed += 1
+        if result:
+            # Apply CWD filter first: cheap metadata-only check. Skip
+            # sessions from other repos before paying the full-file keyword
+            # scan cost — dsh discovery lists across projects, so without
+            # this ordering --keyword would scan files that are immediately
+            # discarded.
+            if cwd_filter:
+                session_cwd = result.get("cwd")
+                if session_cwd:
+                    if not cwd_matches_filter(session_cwd, cwd_filter):
+                        filtered += 1
+                        continue
+                elif result.get("platform") == "dsh":
+                    filtered += 1
+                    continue
+            _attach_timestamps(result, filepath)
+            # Apply keyword scan only after cheap filters pass.
+            if keywords:
+                matches = count_keyword_matches(filepath, keywords)
+                result["keyword_matches"] = matches
+                result["match_count"] = sum(matches.values())
+                if result["match_count"] == 0:
+                    continue
+                matched += 1
+            print(json.dumps(result))
+        elif error:
+            parse_errors += 1
+        else:
+            no_metadata += 1
+
+    meta = {"_meta": True, "files_processed": processed, "parse_errors": parse_errors}
+    if no_metadata:
+        meta["files_without_metadata"] = no_metadata
+    if filtered:
+        meta["filtered_by_cwd"] = filtered
+    if keywords:
+        meta["files_matched"] = matched
+    print(json.dumps(meta))
+else:
+    # No file arguments: either single-file stdin mode or empty xargs invocation.
+    # When xargs runs us with no input (e.g., discover found no files), stdin is
+    # empty or a TTY — emit a clean zero-file result instead of a false parse error.
+    if sys.stdin.isatty():
+        lines = []
+    else:
+        lines = list(sys.stdin)
+
+    if not lines:
+        # No input at all — zero-file result (clean exit for empty pipelines).
+        # When --keyword was supplied, emit files_matched: 0 so callers relying
+        # on its presence to terminate quickly in zero-match scans see a
+        # consistent shape with the batch-mode no-match case.
+        meta = {"_meta": True, "files_processed": 0, "parse_errors": 0}
+        if keywords:
+            meta["files_matched"] = 0
+        print(json.dumps(meta))
+    else:
+        # Genuine single-file stdin mode (backward compatible)
+        result = extract_from_lines(lines)
+        meta = {"_meta": True, "files_processed": 1, "parse_errors": 0}
+        if result:
+            print(json.dumps(result))
+        else:
+            meta["files_without_metadata"] = 1
+        print(json.dumps(meta))