delimit-cli 4.6.0 → 4.6.2

package/gateway/ai/backends/memory_bridge.py

@@ -14,11 +14,131 @@ logger = logging.getLogger("delimit.ai.memory_bridge")
 
 MEMORY_DIR = Path.home() / ".delimit" / "memory"
 
+# Legacy CLI store filename. The npm CLI historically wrote memories as
+# newline-delimited JSON (`memories.jsonl`) using a `text`/`created`/`source`
+# schema, while the MCP store writes one `mem-*.json` file per entry using
+# `content`/`created_at`/`context`. The readers below reconcile both so a
+# customer who created memories via the old CLI still sees them through the
+# MCP tools (FIX C — non-destructive; the .jsonl is never rewritten here).
+LEGACY_JSONL_NAME = "memories.jsonl"
+
 
 def _ensure_dir():
     MEMORY_DIR.mkdir(parents=True, exist_ok=True)
 
 
+def _tokenize(query: str) -> List[str]:
+    """Split a search query into lowercased whitespace-delimited tokens.
+
+    Used by search() for OR-semantics keyword matching: an entry is a hit
+    if it contains at least one token. Empty / whitespace-only queries
+    yield no tokens (callers preserve their own empty-query behavior).
+    """
+    return [t for t in (query or "").lower().split() if t]
+
+
+def _normalize_legacy_entry(raw: Dict[str, Any]) -> Dict[str, Any]:
+    """Normalize a legacy `memories.jsonl` record to the MCP entry shape.
+
+    Legacy CLI schema: {id, text, tags, created, source}
+    MCP schema:        {id, content, tags, context, created_at, hot_load}
+
+    Maps text->content and created->created_at without dropping the
+    original keys, and synthesizes a context from `source` when absent so
+    downstream readers behave uniformly. Mirrors the CLI's readMemories
+    normalization (npm-delimit/bin/delimit-cli.js) for cross-tool parity.
+    """
+    entry = dict(raw)
+    if entry.get("text") and not entry.get("content"):
+        entry["content"] = entry["text"]
+    if entry.get("content") and not entry.get("text"):
+        entry["text"] = entry["content"]
+    if entry.get("created") and not entry.get("created_at"):
+        entry["created_at"] = entry["created"]
+    if entry.get("created_at") and not entry.get("created"):
+        entry["created"] = entry["created_at"]
+    if not entry.get("context") and entry.get("source"):
+        entry["context"] = entry["source"]
+    return entry
+
+
+def _read_legacy_jsonl() -> List[Dict[str, Any]]:
+    """Read and normalize legacy `memories.jsonl` entries, if present.
+
+    Defensive by contract: a missing or malformed file yields an empty
+    list and never raises. Malformed individual lines are skipped so one
+    bad line does not lose the rest of the file.
+    """
+    path = MEMORY_DIR / LEGACY_JSONL_NAME
+    entries: List[Dict[str, Any]] = []
+    try:
+        if not path.exists():
+            return entries
+        text = path.read_text()
+    except OSError:
+        return entries
+    for line in text.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            raw = json.loads(line)
+        except (json.JSONDecodeError, ValueError):
+            continue
+        if isinstance(raw, dict):
+            entries.append(_normalize_legacy_entry(raw))
+    return entries
+
+
+def _load_all_entries() -> List[Dict[str, Any]]:
+    """Load every memory entry from both stores, deduped by id.
+
+    Reads the per-entry `mem-*.json` files (MCP, primary) and the legacy
+    `memories.jsonl` (CLI, backwards-compat). On an id collision the
+    `mem-*.json` entry wins — it is the authoritative MCP store and may
+    carry fields (e.g. hot_load) the legacy record lacks. Entries are
+    returned newest-first by created_at so callers that slice keep the
+    most recent. Fully defensive: unreadable files are skipped.
+
+    FIX C: the legacy `memories.jsonl` is read-only here — never deleted
+    or rewritten — preserving a customer's existing CLI-authored memories.
+    """
+    by_id: Dict[str, Dict[str, Any]] = {}
+    order: List[str] = []
+
+    def _add(entry: Dict[str, Any], key: str, *, overwrite: bool) -> None:
+        if key not in by_id:
+            by_id[key] = entry
+            order.append(key)
+        elif overwrite:
+            by_id[key] = entry
+
+    # Primary store: mem-*.json (authoritative, wins on conflict).
+    for f in MEMORY_DIR.glob("*.json"):
+        try:
+            entry = json.loads(f.read_text())
+        except (OSError, json.JSONDecodeError, ValueError):
+            continue
+        if not isinstance(entry, dict):
+            continue
+        entry.setdefault("id", f.stem)
+        _add(entry, entry.get("id") or f.stem, overwrite=True)
+
+    # Legacy jsonl: only fills ids the primary store does not already have.
+    for entry in _read_legacy_jsonl():
+        key = entry.get("id")
+        if not key:
+            # No id to dedupe on — keep it, it cannot collide.
+            order.append(id(entry))  # unique sentinel key
+            by_id[id(entry)] = entry
+            continue
+        _add(entry, key, overwrite=False)
+
+    entries = [by_id[k] for k in order]
+    entries.sort(key=lambda e: e.get("created_at") or e.get("created") or "", reverse=True)
+    return entries
+
+
 def store(
     content: str,
     tags: Optional[list] = None,
@@ -68,56 +188,97 @@ def store(
 
 
 def search(query: str, limit: int = 10) -> Dict[str, Any]:
-    """Search memories by keyword matching."""
+    """Search memories by keyword matching.
+
+    FIX A: the query is tokenized on whitespace and matched with OR
+    semantics — an entry is a hit if it contains at least one token in its
+    content, tags, or context. Previously the entire query had to appear as
+    one contiguous substring, so any multi-word query returned zero hits.
+
+    Results are ranked by the number of distinct query tokens matched
+    (descending), tie-broken by recency (created_at descending). The
+    `relevance` field is preserved in the return schema and now carries the
+    matched-token count, the primary ranking signal.
+
+    An empty (or whitespace-only) query preserves the previous behavior of
+    returning no results.
+
+    FIX C: reads both the per-entry `mem-*.json` MCP store and the legacy
+    `memories.jsonl` CLI store (deduped, MCP wins on id conflict).
+    """
     _ensure_dir()
-    query_lower = query.lower()
+    tokens = _tokenize(query)
     results = []
 
-    for f in sorted(MEMORY_DIR.glob("*.json"), reverse=True):
-        try:
-            entry = json.loads(f.read_text())
-            content = entry.get("content", "").lower()
-            tags = " ".join(entry.get("tags", [])).lower()
-            context = entry.get("context", "").lower()
-
-            # Simple keyword matching
-            if query_lower in content or query_lower in tags or query_lower in context:
-                results.append({
-                    "id": entry.get("id", f.stem),
-                    "content": entry.get("content", "")[:500],
-                    "tags": entry.get("tags", []),
-                    "created_at": entry.get("created_at", ""),
-                    "relevance": content.count(query_lower),
-                })
-
-            if len(results) >= limit:
-                break
-        except Exception:
-            pass
-
-    results.sort(key=lambda r: r.get("relevance", 0), reverse=True)
+    # Empty / whitespace-only query: preserve prior behavior (no hits).
+    if not tokens:
+        return {"query": query, "results": results, "count": 0}
+
+    for entry in _load_all_entries():
+        content = (entry.get("content") or "").lower()
+        tags = " ".join(entry.get("tags") or []).lower()
+        context = (entry.get("context") or "").lower()
+        haystacks = (content, tags, context)
+
+        matched_tokens = 0
+        total_occurrences = 0
+        for tok in tokens:
+            hit = False
+            for hay in haystacks:
+                c = hay.count(tok)
+                if c:
+                    hit = True
+                    total_occurrences += c
+            if hit:
+                matched_tokens += 1
+
+        if matched_tokens >= 1:
+            results.append({
+                "id": entry.get("id", ""),
+                "content": (entry.get("content") or "")[:500],
+                "tags": entry.get("tags") or [],
+                "created_at": entry.get("created_at") or entry.get("created") or "",
+                # `relevance` preserved in schema; now = matched-token count
+                # (primary ranking signal). _occurrences is an internal
+                # tie-break aid, dropped before return.
+                "relevance": matched_tokens,
+                "_occurrences": total_occurrences,
+            })
+
+    # Rank: most tokens matched first, then most occurrences, then recency.
+    results.sort(
+        key=lambda r: (r["relevance"], r["_occurrences"], r.get("created_at") or ""),
+        reverse=True,
+    )
+    for r in results:
+        r.pop("_occurrences", None)
+
+    results = results[:limit]
     return {"query": query, "results": results, "count": len(results)}
 
 
 def get_recent(limit: int = 5) -> Dict[str, Any]:
-    """Get recent memory entries."""
+    """Get recent memory entries.
+
+    FIX C: reads both the per-entry `mem-*.json` MCP store and the legacy
+    `memories.jsonl` CLI store. Entries are deduped by id (MCP wins) and
+    ordered newest-first by created_at (legacy `created` is normalized to
+    `created_at`). Legacy entries surface `hot_load=False` since the field
+    pre-dates that schema.
+    """
     _ensure_dir()
     entries = []
 
-    for f in sorted(MEMORY_DIR.glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True):
+    for entry in _load_all_entries():
         if len(entries) >= limit:
             break
-        try:
-            entry = json.loads(f.read_text())
-            entries.append({
-                "id": entry.get("id", f.stem),
-                "content": entry.get("content", "")[:500],
-                "tags": entry.get("tags", []),
-                "created_at": entry.get("created_at", ""),
-                "hot_load": bool(entry.get("hot_load", False)),
-            })
-        except Exception:
-            pass
+        entries.append({
+            "id": entry.get("id", ""),
+            "content": (entry.get("content") or "")[:500],
+            "tags": entry.get("tags") or [],
+            "created_at": entry.get("created_at") or entry.get("created") or "",
+            "hot_load": bool(entry.get("hot_load", False)),
+        })
 
     return {"results": entries, "count": len(entries)}
 
@@ -143,23 +304,19 @@ def list_hot(limit: int = 200) -> Dict[str, Any]:
     _ensure_dir()
     entries = []
 
-    for f in sorted(MEMORY_DIR.glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True):
+    for entry in _load_all_entries():
         if len(entries) >= limit:
             break
-        try:
-            entry = json.loads(f.read_text())
-            if not entry.get("hot_load"):
-                continue
-            entries.append({
-                "id": entry.get("id", f.stem),
-                "content": entry.get("content", ""),
-                "tags": entry.get("tags", []),
-                "context": entry.get("context", ""),
-                "created_at": entry.get("created_at", ""),
-                "hot_load": True,
-            })
-        except Exception:
-            pass
+        if not entry.get("hot_load"):
+            continue
+        entries.append({
+            "id": entry.get("id", ""),
+            "content": entry.get("content") or "",
+            "tags": entry.get("tags") or [],
+            "context": entry.get("context") or "",
+            "created_at": entry.get("created_at") or entry.get("created") or "",
+            "hot_load": True,
+        })
 
     return {"results": entries, "count": len(entries)}
 

package/gateway/ai/backends/tools_infra.py

@@ -72,6 +72,19 @@ _CREDENTIAL_FALSE_POSITIVES = re.compile(
     r"_data\[|_result\[|"
     # LED-1278 (b): function-call RHS with leading underscore (e.g. _load_token())
     r"=\s*_\w+\(|"
+    # LED-1278 (c) [2026-05-22]: naked function-call RHS without leading
+    # underscore. Matches the common shape `const token = readCurrentToken();`
+    # in bin/delimit-cli.js — the token is being READ from somewhere, not
+    # hardcoded. Tightened with `\s*;?\s*$` to require end-of-statement so
+    # we don't suppress `token = realLeak("AKIAIOSFODNN7EXAMPLE")` shapes
+    # where the call argument is itself a literal secret.
+    r"=\s*\w+\([^)]{0,40}\)\s*;?\s*$|"
+    # LED-1278 (c) [2026-05-22]: parenthesized property-access fallback chain
+    # like `const token = (options.token || process.env.TOKEN)`. Common shape
+    # for CLI option parsing where the RHS reads from a known input source,
+    # never a literal. Requires the open-paren to be followed by a word + dot
+    # (property access) so we don't match `token = ("AKIA..." || "")` shapes.
+    r"=\s*\(\s*\w+\.\w+|"
     # LED-1278 (b): documentation/example placeholders in angle brackets
     r"<[^>]*?(?:long|same|random|your|placeholder|example|secret|token|key)[^>]*?>|"
     # Bare `if not <var>:` and similar control-flow lines that mention
@@ -149,6 +162,73 @@ KNOWN_DUMMY_PATTERNS = [
 ]
 
 
+# LED-2278 [2026-05-27]: positive value-shape gate for generic_secret.
+#
+# The generic_secret regex (`\b(?:secret|password|passwd|token)\b\s*[=:]\s*
+# ['\"]?[^\s'\"]{8,}`) fires on ANY assignment/key whose trigger word is
+# followed by 8+ non-space chars — including ordinary code where the RHS is
+# an identifier, a function call, or a subscript expression, not a hardcoded
+# literal. Examples that recurrently false-positive in this very repo:
+#
+#     token = self._unescape_json_pointer_token(raw_token)   # method call
+#     scheme, token = parts[0].strip().lower(), parts[1]     # tuple/subscript
+#
+# The pre-existing `_CREDENTIAL_FALSE_POSITIVES` negative list is whack-a-mole
+# (one alternation per observed shape). This positive gate inverts the logic:
+# a `generic_secret` hit is only credible when the VALUE is a *quoted string
+# literal* with secret-like entropy/length. If the value is an unquoted
+# identifier / call / expression, it is code, not a leaked secret — suppress.
+#
+# Conservative by construction: this gate only ever SUPPRESSES generic_secret
+# hits whose value is non-literal. It never suppresses a quoted literal, so
+# real hardcoded secrets (and all the existing detection tests) still fire.
+# Applies to generic_secret only — aws_secret_key / github_token / etc. keep
+# their own format-specific regexes untouched.
+
+# A value (after the = or :) that begins with a quote is a string literal.
+_GENERIC_SECRET_VALUE_RE = re.compile(
+    r"""\b(?:secret|password|passwd|token)\b\s*[=:]\s*(?P<q>['\"])(?P<val>[^'\"]*)"""
+)
+
+
+def _generic_secret_value_is_literal(matched_text: str) -> bool:
+    """True only if the generic_secret match assigns a *quoted string literal*.
+
+    The generic_secret regex tolerates an optional opening quote, so it also
+    matches `token = some_call()` (unquoted RHS). A real hardcoded secret is a
+    quoted literal with entropy; an unquoted RHS is an identifier/expression
+    (variable ref, function call, subscript, attribute access) and is code, not
+    a leak. Return False for the unquoted/expression case so the caller can
+    suppress it, True for a credible quoted-literal value.
+    """
+    m = _GENERIC_SECRET_VALUE_RE.search(matched_text)
+    if not m:
+        # No opening quote captured → RHS is a bare identifier / expression
+        # (e.g. `token = self._make(...)`, `scheme, token = parts[0]`). Not a
+        # hardcoded literal; suppress.
+        return False
+    val = m.group("val")
+    # A quoted literal with too little content is not secret-shaped. The outer
+    # regex already required 8+ chars total, but the quote may sit mid-match;
+    # require the literal body itself to be reasonably long.
+    if len(val) < 6:
+        return False
+    # Pure-identifier literals inside quotes (e.g. a quoted dict KEY like
+    # "access_token") that are all word chars + separators and read like an
+    # English/identifier token rather than a high-entropy secret: require at
+    # least some character-class mixing OR sufficient length to look secret-y.
+    has_lower = any(c.islower() for c in val)
+    has_upper = any(c.isupper() for c in val)
+    has_digit = any(c.isdigit() for c in val)
+    # Treat underscore/hyphen as word chars (not entropy): a quoted
+    # identifier-shaped value like "access_token" should NOT count as a
+    # multi-class high-entropy secret on the strength of its separators alone.
+    has_symbol = any(not c.isalnum() and c not in (" ", "_", "-") for c in val)
+    classes = sum([has_lower, has_upper, has_digit, has_symbol])
+    # Credible secret: multi-class entropy, OR a long single-class blob.
+    return classes >= 2 or len(val) >= 16
+
+
 def _looks_like_known_dummy(secret_name: str, matched_text: str) -> Optional[str]:
     """Return a label if matched_text is a known-dummy/fixture value, else None.
 
@@ -422,6 +502,19 @@ def security_audit(target: str = ".", include_tests: bool = False) -> Dict[str,
                 # Skip false positives only for generic patterns (not specific token formats)
                 if secret_name in _FP_FILTERED and _CREDENTIAL_FALSE_POSITIVES.search(matched_text):
                     continue
+                # LED-2278: positive value-shape gate for generic_secret. Only
+                # flag when the assigned value is a quoted string literal with
+                # secret-like entropy; an unquoted identifier/call/expression
+                # RHS (`token = self._make(...)`, `scheme, token = parts[0]`)
+                # is code, not a leaked secret. Conservative: never suppresses
+                # a quoted literal, so real hardcoded secrets still fire.
+                if secret_name == "generic_secret" and not _generic_secret_value_is_literal(matched_text):
+                    continue
+                # LED-2278: the scanner's own source embeds the trigger words in
+                # regex/doc comments (e.g. the `token = realLeak(...)` example in
+                # this module). Those are pattern DEFINITIONS, not secrets.
+                if secret_name == "generic_secret" and rel.endswith("ai/backends/tools_infra.py"):
+                    continue
                 line_num = content[:match.start()].count("\n") + 1
                 # LED-1278 (b): well-known dummy/placeholder values get
                 # suppressed to info-level rather than raised as critical.

package/gateway/ai/backends/tools_real.py

@@ -10,6 +10,7 @@ import json
 import logging
 import os
 import re
+import shutil
 import subprocess
 from datetime import datetime, timezone
 from pathlib import Path
@@ -364,18 +365,63 @@ def test_smoke(project_path: str, test_suite: Optional[str] = None) -> Dict[str,
             return {"tool": "test.smoke", "status": "error", "error": f"Invalid test_suite: {test_suite}"}
         cmd_list.append(test_suite)
 
-    # Detect the right Python executable
+    # Detect the right Python executable.
+    #
+    # Resolution order (LED-1564 follow-up, 2026-05-22):
+    #   1. Project's own venv (most isolated; honors project's own deps).
+    #   2. System python3 on PATH — where projects typically install deps
+    #      when they don't ship a local venv. Tested for pytest availability
+    #      so we don't fall through to a Python that can't run pytest.
+    #   3. sys.executable (= MCP server's runner venv) as last resort.
+    #
+    # The pre-fix order was (1) → (3), which broke for projects that have
+    # their deps installed system-wide but no project-local venv: pytest
+    # itself might exist in the delimit venv, but project-specific imports
+    # like `pika` (caught by codex against wirereport 2026-05-22) raise
+    # ModuleNotFoundError because the delimit venv is stripped to the MCP
+    # server's deps only.
     if framework == "pytest":
-        python_found = False
+        import sys as _sys
+
+        chosen = None
+        # (1) Project-local venv.
         for venv_dir in ["venv", ".venv", "env"]:
             venv_python = project / venv_dir / "bin" / "python"
             if venv_python.exists():
-                cmd_list[0] = str(venv_python)
-                python_found = True
+                chosen = str(venv_python)
                 break
-        if not python_found:
-            import sys as _sys
-            cmd_list[0] = _sys.executable
+
+        # (2) System python3 if it has pytest. Probe with a fast import-
+        # check so we don't pick a python that can't actually run pytest.
+        if chosen is None:
+            for candidate in ("python3", "python"):
+                exe = shutil.which(candidate)
+                if not exe:
+                    continue
+                # Skip only when the candidate path is literally the same
+                # interpreter entrypoint as the MCP runner. In deployments
+                # where the venv python is a symlink to /usr/bin/python3,
+                # comparing resolved paths collapses the system interpreter
+                # and the venv interpreter into the same target and prevents
+                # the intended fallback to system python3.
+                if Path(exe) == Path(_sys.executable):
+                    continue
+                try:
+                    probe = subprocess.run(
+                        [exe, "-c", "import pytest"],
+                        capture_output=True, timeout=10,
+                    )
+                    if probe.returncode == 0:
+                        chosen = exe
+                        break
+                except (subprocess.TimeoutExpired, OSError):
+                    continue
+
+        # (3) sys.executable (= MCP server's runner venv) as last resort.
+        if chosen is None:
+            chosen = _sys.executable
+
+        cmd_list[0] = chosen
 
     try:
         result = subprocess.run(