@misterhuydo/sentinel 1.5.39 → 1.5.41

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@misterhuydo/sentinel",
-  "version": "1.5.39",
+  "version": "1.5.41",
   "description": "Sentinel — Autonomous DevOps Agent installer and manager",
   "bin": {
     "sentinel": "./bin/sentinel.js"

package/python/sentinel/__init__.py

@@ -1 +1 @@
-__version__ = "1.5.39"
+__version__ = "1.5.41"

package/python/sentinel/sentinel_boss.py

@@ -152,15 +152,20 @@ COMPLETE TOOL REFERENCE
 ── Log Management ─────────────────────────────────────────────────────────────
 
  6. fetch_logs          Run fetch_log.sh on demand — pull fresh logs from servers now.
-                        Supports --debug and parameter overrides.
-                        "fetch logs", "fetch logs for SSOLWA", "fetch without filter"
+                        When grep_filter is set, results go to TEMP files (workspace/fetched_temp/)
+                        and do NOT affect the main rolling logs. Temp files are cleared on every
+                        custom fetch, and filter_logs searches them automatically.
+                        Use grep_filter for INFO-level or feature-specific patterns that the default
+                        WARN/ERROR filter would miss.
+                        "fetch logs", "fetch SSOLWA with filter provision/phone", "fetch without filter"
 
  7. search_logs         Live SSH grep on production servers using GREP_FILTER.
                         Falls back to cached files if SSH unavailable.
                         "search logs for illegal PIN in 1881", "find NullPointerException in STS"
 
- 8. filter_logs         Instant keyword/regex search on locally-synced logs. No SSH, sub-second.
-                        Supports since_hours, case options.
+ 8. filter_logs         Instant keyword/regex search on locally-synced logs + any temp fetch results.
+                        No SSH, sub-second. Also searches workspace/fetched_temp/ if a custom
+                        fetch was recently run.
                         "filter logs for TryDig", "errors last 6h", "find appid=X in STS logs"
 
  9. tail_log            Last N lines of a log source live, no filter.
@@ -545,9 +550,6 @@ When to act vs. when to ask:
   tasks, fixes, releases). Always call get_status or list_recent_commits first to verify live
   state. Session memory is a snapshot — tasks complete, commits land, queues drain between turns.
   If you remember "task X was in-flight", check whether it finished before telling the user to wait.
-- NEVER repeat a deployment assertion you made in a prior turn without re-verifying it.
-  If you said "version X is not deployed" in a previous message and the user asks again, do NOT
-  just echo the same claim — check list_recent_commits or startup logs fresh before answering.
 - Prefer filter_logs over search_logs when synced logs are available — it's instant and never causes session timeout.
   Use search_logs (live SSH fetch) when:
   • The user explicitly wants live/real-time data
@@ -556,15 +558,27 @@ When to act vs. when to ask:
     (synced logs may simply be stale — do NOT conclude the change isn't live yet; fetch live first)
   When filter_logs returns no hits after a recent release, always retry with search_logs before
   telling the user the log line isn't there.
-- NEVER infer deployment status from the absence of a feature log line.
-  A specific log line only appears when that exact code path is executed by a real user/request.
-  Zero hits means: nobody has triggered that path yet. It says NOTHING about whether the release
-  deployed. DO NOT say "release X has not deployed" or "the servers are still on the old version"
-  based on this. If you must comment on deployment status, say:
-  "The log line hasn't appeared yet — this means the endpoint hasn't been called since the
-  release, not that the release is missing."
-  To actually verify a release is live, search for startup/version lines:
-  search_logs with query "Starting|started in|version|initialized" — do not guess from feature log absence.
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+LOG RESULTS DO NOT PROVE DEPLOYMENT STATUS
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+You are FORBIDDEN from asserting "release X is not deployed" or "servers are still on old version"
+based solely on log search results — not from zero hits, not from finding old lines without new ones.
+
+WHY: A log line only appears when that exact code path executes. Finding 0 hits or only old log
+lines says nothing about what code version is running. The new line simply hasn't been triggered yet.
+
+WRONG — NEVER SAY THESE:
+  "Zero hits for 'provision/phone called by appId'. Release 6.29.34 has not deployed."
+  "Found 3 old DEBUG entries but no new INFO line — servers are still on the previous version."
+  "The new logging code from 6.29.34 is still not deployed."
+
+CORRECT:
+  "No 'provision/phone called by appId' line yet — the endpoint hasn't logged that path since
+  the last fetch. This says nothing about whether 6.29.34 is deployed."
+
+To verify if a release is live, offer to search for startup log lines:
+  search_logs with query "Starting|started in|version|initialized"
+NEVER repeat a deployment assertion from a prior turn without doing this check first.
 - If a tool call will take a moment (search, fetch, pull), prefix your reply with a brief "working" line ending in "..." before the results, e.g. "Searching SSOLWA for TryDig activity..." then the actual output.
   Never just say a working line and stop — always follow it with the results in the same message.
 
@@ -1156,7 +1170,14 @@ _TOOLS = [
             "Run fetch_log.sh for one or all configured log sources to pull the latest logs "
             "from remote servers right now. Use for: 'fetch logs', 'run fetch_log.sh', "
             "'grab latest logs from SSOLWA', 'try fetch_log.sh for STS', "
-            "'pull logs from server', 'get fresh logs'."
+            "'pull logs from server', 'get fresh logs'.\n\n"
+            "IMPORTANT: When a grep_filter is provided, results go to a TEMPORARY location "
+            "(workspace/fetched_temp/) and do NOT overwrite the main rolling logs. "
+            "The temp files are cleared on every custom fetch. "
+            "After a custom fetch, filter_logs will automatically search the temp results too. "
+            "Use grep_filter whenever the user wants to find INFO-level or feature-specific log lines "
+            "(e.g. 'provision/phone', 'appId', startup messages) — the default filter only captures "
+            "WARN|ERROR|FATAL|Exception|Error lines."
         ),
         "input_schema": {
             "type": "object",
@@ -1176,7 +1197,11 @@ _TOOLS = [
                 },
                 "grep_filter": {
                     "type": "string",
-                    "description": "Override GREP_FILTER (regex). Pass 'none' to disable filtering.",
+                    "description": (
+                        "Custom grep filter (regex). Results saved to temp files, main logs untouched. "
+                        "Pass 'none' to fetch all lines unfiltered. "
+                        "Use when searching for INFO-level or feature-specific patterns."
+                    ),
                 },
             },
         },
@@ -2814,104 +2839,44 @@ async def _run_tool(name: str, inputs: dict, cfg_loader, store, slack_client=Non
             return json.dumps({"error": f"Invalid regex: {e}"})
 
         synced_base = Path("workspace/synced")
-        if not synced_base.exists():
-            return json.dumps({
-                "error": "No synced logs found.",
-                "hint": "Log sync runs every SYNC_INTERVAL_SECONDS (default 300s). "
-                        "If just started, wait a minute then try again.",
-            })
+        temp_base   = Path(cfg_loader.sentinel.workspace_dir) / "fetched_temp"
 
         # Build cutoff timestamp for since_hours filter
         cutoff = None
         if since_hours:
             cutoff = _datetime.now(_tz.utc) - timedelta(hours=int(since_hours))
 
-        # Determine which source directories to search
-        if source_f:
-            src_dirs = [d for d in sorted(synced_base.iterdir())
-                        if d.is_dir() and source_f in d.name.lower()]
-        else:
-            src_dirs = [d for d in sorted(synced_base.iterdir()) if d.is_dir()]
-
-        if not src_dirs:
-            available = [d.name for d in synced_base.iterdir() if d.is_dir()]
-            return json.dumps({
-                "error": f"No synced source matching '{source_f}'",
-                "available_sources": available,
-            })
-
-        results = []
-        total_matches = 0
-        for src_dir in src_dirs:
-            for log_file in sorted(src_dir.glob("*")):
-                try:
-                    lines = log_file.read_text(encoding="utf-8", errors="replace").splitlines()
-                    matches = []
-                    for line in lines:
-                        if not pat.search(line):
-                            continue
-                        if cutoff:
-                            # Try to parse timestamp from line
-                            from .log_fetcher import _parse_line_ts
-                            ts = _parse_line_ts(line)
-                            if ts and ts < cutoff:
-                                continue
-                        matches.append(line[:300])
-                        if len(matches) >= max_matches:
-                            break
-                    if matches:
-                        results.append({
-                            "source": src_dir.name,
-                            "file": log_file.name,
-                            "matches": matches,
-                        })
-                        total_matches += len(matches)
-                except Exception:
-                    pass
-
-        if not results:
-            return json.dumps({
-                "query": query_f,
-                "total_matches": 0,
-                "sources_searched": [d.name for d in src_dirs],
-                "note": "No matches found in synced logs.",
-            })
+        # Collect candidate directories from both synced/ and fetched_temp/
+        def _collect_dirs(base):
+            if not base.exists():
+                return []
+            if source_f:
+                return [d for d in sorted(base.iterdir()) if d.is_dir() and source_f in d.name.lower()]
+            return [d for d in sorted(base.iterdir()) if d.is_dir()]
 
+        src_dirs      = _collect_dirs(synced_base)
+        temp_dirs     = _collect_dirs(temp_base)
+        all_search_dirs = src_dirs + [(d, True) for d in temp_dirs]  # True = is_temp
 
-        try:
-            pat = _re.compile(query_f, case_flag)
-        except _re.error as e:
-            return json.dumps({"error": f"Invalid regex: {e}"})
+        # Flatten to (dir, is_temp) pairs
+        search_pairs = [(d, False) for d in src_dirs] + [(d, True) for d in temp_dirs]
 
-        synced_base = Path("workspace/synced")
-        if not synced_base.exists():
+        if not search_pairs:
+            available = ([d.name for d in synced_base.iterdir() if d.is_dir()] if synced_base.exists() else [])
             return json.dumps({
-                "error": "No synced logs found.",
+                "error": f"No synced or temp source matching '{source_f}'" if source_f else "No logs found.",
+                "available_sources": available,
                 "hint": "Log sync runs every SYNC_INTERVAL_SECONDS (default 300s). "
                         "If just started, wait a minute then try again.",
             })
 
-        cutoff = None
-        if since_hours:
-            cutoff = _datetime.now(_tz.utc) - timedelta(hours=int(since_hours))
-
-        if source_f:
-            src_dirs = [d for d in sorted(synced_base.iterdir())
-                        if d.is_dir() and source_f in d.name.lower()]
-        else:
-            src_dirs = [d for d in sorted(synced_base.iterdir()) if d.is_dir()]
-
-        if not src_dirs:
-            available = [d.name for d in synced_base.iterdir() if d.is_dir()]
-            return json.dumps({
-                "error": f"No synced source matching '{source_f}'",
-                "available_sources": available,
-            })
-
-        all_matches = []   # list of (source_name, line)
+        all_matches = []   # list of (source_label, line)
         sources_hit = set()
-        for src_dir in src_dirs:
-            for log_file in sorted(src_dir.glob("*")):
+        for src_dir, is_temp in search_pairs:
+            label = src_dir.name + (" [temp]" if is_temp else "")
+            for log_file in sorted(src_dir.glob("**/*")):
+                if not log_file.is_file():
+                    continue
                 try:
                     lines = log_file.read_text(encoding="utf-8", errors="replace").splitlines()
                     for line in lines:
@@ -2922,8 +2887,8 @@ async def _run_tool(name: str, inputs: dict, cfg_loader, store, slack_client=Non
                             ts = _parse_line_ts(line)
                             if ts and ts < cutoff:
                                 continue
-                        all_matches.append((src_dir.name, line[:300]))
-                        sources_hit.add(src_dir.name)
+                        all_matches.append((label, line[:300]))
+                        sources_hit.add(label)
                         if len(all_matches) >= max_matches:
                             break
                 except Exception:
@@ -2932,12 +2897,19 @@ async def _run_tool(name: str, inputs: dict, cfg_loader, store, slack_client=Non
                 break
 
         total = len(all_matches)
+        sources_searched = [d.name + (" [temp]" if is_temp else "") for d, is_temp in search_pairs]
         if total == 0:
+            has_temp = bool(temp_dirs)
             return json.dumps({
                 "query": query_f,
                 "total_matches": 0,
-                "sources_searched": [d.name for d in src_dirs],
-                "note": "No matches found in synced logs.",
+                "sources_searched": sources_searched,
+                "note": (
+                    "No matches found. "
+                    + ("Temp fetch results were also checked. " if has_temp else "")
+                    + "If searching for a specific log line from a new feature, use fetch_logs "
+                    "with a matching grep_filter first — the default filter only captures WARN/ERROR."
+                ),
             })
 
         # Pattern grouping: count occurrences of each error signature
@@ -2983,7 +2955,7 @@ async def _run_tool(name: str, inputs: dict, cfg_loader, store, slack_client=Non
             "query":            query_f,
             "total_matches":    total,
             "sources_hit":      sorted(sources_hit),
-            "sources_searched": [d.name for d in src_dirs],
+            "sources_searched": sources_searched,
             "top_patterns":     top_patterns,
             "sample_lines":     sample_lines,
             "time_span":        time_span,
@@ -3081,6 +3053,20 @@ async def _run_tool(name: str, inputs: dict, cfg_loader, store, slack_client=Non
         if not props_files:
             return json.dumps({"error": f"No log-config found matching '{source_filter}'"})
 
+        # When a custom grep_filter is set, route output to a temp directory
+        # so the main rolling logs are never polluted by user-requested searches.
+        # The temp dir is cleared before each custom fetch.
+        workspace_dir = Path(cfg_loader.sentinel.workspace_dir)
+        temp_base     = workspace_dir / "fetched_temp"
+        use_temp      = bool(grep_override)
+
+        if use_temp:
+            # Clear old temp results
+            import shutil
+            if temp_base.exists():
+                shutil.rmtree(temp_base)
+            temp_base.mkdir(parents=True, exist_ok=True)
+
         results = []
         for props in props_files:
             env = os.environ.copy()
@@ -3088,6 +3074,9 @@ async def _run_tool(name: str, inputs: dict, cfg_loader, store, slack_client=Non
                 env["TAIL"] = str(tail_override)
             if grep_override:
                 env["SENTINEL_GREP_FILTER_OVERRIDE"] = grep_override
+            if use_temp:
+                # Tell fetch_log.sh where to write output files
+                env["OUTPUT_DIR"] = str(temp_base)
 
             cmd = ["bash", str(script)]
             if debug:
@@ -3100,13 +3089,24 @@ async def _run_tool(name: str, inputs: dict, cfg_loader, store, slack_client=Non
                 )
                 output = (r.stdout or "").strip()
                 stderr = (r.stderr or "").strip()
+
+                # Collect lines from temp files for this source
+                temp_lines = []
+                if use_temp:
+                    for f in sorted((temp_base / props.stem).glob("*.log")) if (temp_base / props.stem).exists() else []:
+                        try:
+                            temp_lines.extend(f.read_text(encoding="utf-8", errors="replace").splitlines())
+                        except Exception:
+                            pass
+
                 results.append({
                     "source":     props.stem,
                     "returncode": r.returncode,
                     "output":     output[-2000:] if output else "",
                     "stderr":     stderr[-1000:] if stderr else "",
+                    **({"lines": temp_lines, "temp_file": str(temp_base / props.stem)} if use_temp else {}),
                 })
-                logger.info("Boss fetch_logs %s rc=%d", props.stem, r.returncode)
+                logger.info("Boss fetch_logs %s rc=%d (temp=%s)", props.stem, r.returncode, use_temp)
             except subprocess.TimeoutExpired:
                 results.append({"source": props.stem, "error": "timed out after 120s"})
             except Exception as e: