delimit-cli 4.6.0 → 4.6.2

package/gateway/ai/tenant_data.py

@@ -0,0 +1,339 @@
+"""LED-2268 P0 Phase 0.3 — first consumer of the tenant_data_root primitive.
+
+Provides describe_tenant_data() — the read-only view of what's on disk
+inside a given tenant's data root. Used by the /tenant/data endpoint
+and intended to power the dashboard's "your data lives here" home tile
+for browser-only operators.
+
+The describe call is deliberately minimal:
+  - data_root: absolute path string the gateway resolved for this tenant
+  - exists:    has the dir been created yet?
+  - files:     relative paths inside the dir (deepest-first, sorted)
+  - dirs:      relative paths of subdirectories
+  - total_size_bytes: sum of all file sizes (sentinel for usage display)
+  - cap_bytes: soft cap if configured (Phase 0.3 hard-codes None — no cap)
+
+Phase 0.3 ONLY reads. No write/delete API yet — that's Phase 0.4+, when
+the dashboard ships its first "create note / save memory" surface.
+
+Founder-data migration is handled by the SEPARATE manual script
+scripts/delimit_seed_tenant_data.py (also in this PR), not by an
+auto-trigger inside describe(). Keeps the read path side-effect-free.
+"""
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+from typing import Optional, TypedDict
+
+from . import tenant_paths
+
+logger = logging.getLogger("delimit.tenant_data")
+
+
+# ─────────────────────────────────────────────────────────────────────
+# Phase 0.4 — write/read/delete limits + allowlist
+# ─────────────────────────────────────────────────────────────────────
+
+# Max bytes a single tenant file may contain. Generous enough for
+# memory.jsonl / ledger.jsonl scale (typically <100KB per tenant) but
+# tight enough that a runaway client can't fill the disk. Future quota
+# enforcement will sum across files; this is per-file.
+MAX_FILE_BYTES = 1024 * 1024  # 1 MiB
+
+# Allowlist of file extensions tenants may write/read. Restrictive on
+# purpose: text-shaped data files only. Blocks .py / .sh / .so / .dll
+# / anything executable so the tenant data root can never become a
+# code-drop or LD-load source.
+_ALLOWED_EXTENSIONS = frozenset({
+    ".json",
+    ".jsonl",
+    ".md",
+    ".txt",
+    ".csv",
+    ".yaml",
+    ".yml",
+})
+
+# Max path-segment count (depth) to discourage deeply-nested layouts
+# that complicate audit + backup. Practical cap; nothing in the
+# legitimate use case needs >5 levels of subdirectory.
+_MAX_PATH_DEPTH = 5
+
+
+class TenantPathError(Exception):
+    """Raised for any tenant-data path that fails validation.
+
+    Caller pattern is `except TenantPathError as e: return 400 ...`.
+    The message is the diagnostic suitable for surfacing to the user
+    ("path_too_deep", "extension_forbidden", "path_escapes_root", etc).
+    """
+
+
+def _resolve_tenant_file(user_id: str, rel_path: str, *, create_root: bool = False) -> Path:
+    """Validate + resolve `rel_path` inside the tenant's data root.
+
+    Raises TenantPathError on any of:
+      - empty / non-string rel_path
+      - rel_path containing nul bytes
+      - rel_path with absolute prefix ('/...')
+      - rel_path with traversal segments ('..') that would escape root
+      - rel_path with > _MAX_PATH_DEPTH segments
+      - extension not in _ALLOWED_EXTENSIONS
+      - user_id unsanitisable (no resolvable tenant root)
+
+    Returns the absolute resolved Path, NEVER outside the tenant root.
+    """
+    if not isinstance(rel_path, str) or not rel_path:
+        raise TenantPathError("path_required")
+    if "\x00" in rel_path:
+        raise TenantPathError("path_invalid")
+    # Normalise separators (a tenant could send "\" on Windows-style
+    # input even if the server is Linux; treat both as separators).
+    norm = rel_path.replace("\\", "/").strip()
+    if not norm:
+        raise TenantPathError("path_required")
+    if norm.startswith("/"):
+        raise TenantPathError("path_must_be_relative")
+
+    # Split + reject any traversal segments before resolving. The
+    # post-resolve check below is a second line of defence; do this
+    # pre-check too so we don't even touch the filesystem for obvious
+    # attacks.
+    parts = [p for p in norm.split("/") if p]
+    if any(p in ("", ".", "..") for p in parts):
+        raise TenantPathError("path_traversal_forbidden")
+    if len(parts) > _MAX_PATH_DEPTH:
+        raise TenantPathError("path_too_deep")
+
+    # Extension allowlist applies to the final segment only.
+    final = parts[-1]
+    suffix = Path(final).suffix.lower()
+    if suffix not in _ALLOWED_EXTENSIONS:
+        raise TenantPathError("extension_forbidden")
+
+    root = tenant_paths.tenant_data_root(user_id, create=create_root)
+    if root is None:
+        raise TenantPathError("tenant_resolve_failed")
+
+    # Build the candidate path + verify it stays under the tenant root
+    # after path-resolution. Defence in depth against any sanitiser
+    # gap (symlinks, alternate path-separator tricks, OS-specific
+    # weirdness).
+    candidate = (root / Path(*parts)).resolve()
+    try:
+        candidate.relative_to(root.resolve())
+    except ValueError as e:
+        raise TenantPathError("path_escapes_root") from e
+    return candidate
+
+
+def write_tenant_file(user_id: str, rel_path: str, content: bytes) -> int:
+    """Atomically write `content` to `rel_path` inside the tenant's data root.
+
+    - Creates the tenant root + intermediate directories with 0o700.
+    - Enforces MAX_FILE_BYTES on `content`.
+    - Writes to a sibling `.tmp` file then renames (atomic on POSIX).
+    - File mode is 0o600 (gateway-process-owner readable only).
+
+    Returns the number of bytes written. Raises TenantPathError on
+    validation failure or OSError on filesystem failure.
+    """
+    if not isinstance(content, (bytes, bytearray, memoryview)):
+        raise TenantPathError("content_must_be_bytes")
+    if len(content) > MAX_FILE_BYTES:
+        raise TenantPathError("content_too_large")
+    target = _resolve_tenant_file(user_id, rel_path, create_root=True)
+    target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+    tmp = target.with_name(target.name + ".tmp")
+    # Use os.open so we can set the mode atomically (chmod-after-write
+    # would race with a reader that opened between create + chmod).
+    fd = os.open(str(tmp), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    try:
+        os.write(fd, bytes(content))
+    finally:
+        os.close(fd)
+    os.replace(tmp, target)
+    return len(content)
+
+
+def read_tenant_file(user_id: str, rel_path: str) -> Optional[bytes]:
+    """Read a tenant file, or None if it doesn't exist.
+
+    Raises TenantPathError on validation failure. Other filesystem
+    errors (PermissionError, IsADirectoryError) propagate — those
+    indicate a bug or hostile filesystem state, not normal client
+    input.
+    """
+    target = _resolve_tenant_file(user_id, rel_path, create_root=False)
+    if not target.is_file():
+        return None
+    if target.stat().st_size > MAX_FILE_BYTES:
+        # Defence in depth: even if a write somehow bypassed the cap,
+        # don't echo the over-large content back to a client. Return
+        # None and log — caller surfaces as "not found".
+        logger.warning(
+            "read_tenant_file refusing oversize file: user=%s path=%s size=%d",
+            user_id, rel_path, target.stat().st_size,
+        )
+        return None
+    return target.read_bytes()
+
+
+def delete_tenant_file(user_id: str, rel_path: str) -> bool:
+    """Delete a tenant file. Returns True if deleted, False if absent.
+
+    Raises TenantPathError on validation failure.
+    """
+    target = _resolve_tenant_file(user_id, rel_path, create_root=False)
+    if not target.exists():
+        return False
+    if target.is_dir():
+        # We don't currently support tenant subdirs at the API level
+        # (write creates them as a side effect of the file path).
+        # Reject directory deletes outright — tenants shouldn't be
+        # able to recursively rm their own dir tree via this API.
+        raise TenantPathError("path_is_directory")
+    target.unlink()
+    return True
+
+
+class TenantDataSummary(TypedDict):
+    """What /tenant/data returns to a caller."""
+    user_id: str
+    data_root: str
+    exists: bool
+    files: list[str]
+    dirs: list[str]
+    total_size_bytes: int
+    cap_bytes: Optional[int]
+
+
+# Conservative cap on how many entries we'll enumerate / size-sum before
+# bailing out. A tenant with 100k files shouldn't be able to make a
+# single /tenant/data call stat() every one of them on every dashboard
+# refresh. Returning truncated counts is honest enough for "how full is
+# my dir" UX; the dashboard can surface "(more — refresh to scan)".
+_MAX_ENTRIES_PER_SUMMARY = 1000
+
+
+def describe_tenant_data(user_id: str, *, create: bool = False) -> Optional[TenantDataSummary]:
+    """Read-only summary of a tenant's on-disk data.
+
+    Returns None if `user_id` is unsanitisable (same failure mode as
+    tenant_paths.tenant_data_root). Caller treats that as "unauthorised".
+
+    When `create=False` (default) and the dir doesn't exist yet, returns
+    a summary with exists=False and empty lists. This is the normal
+    first-call shape — operators see "no data yet, you're brand new."
+    When `create=True`, the dir is mkdir'd and an empty summary returned
+    (used by /tenant/setup-style flows; Phase 0.3 doesn't ship one yet).
+    """
+    root = tenant_paths.tenant_data_root(user_id, create=create)
+    if root is None:
+        return None
+
+    summary: TenantDataSummary = {
+        "user_id": user_id,
+        "data_root": str(root),
+        "exists": root.exists(),
+        "files": [],
+        "dirs": [],
+        "total_size_bytes": 0,
+        "cap_bytes": None,
+    }
+
+    if not summary["exists"]:
+        return summary
+
+    files: list[str] = []
+    dirs: list[str] = []
+    total = 0
+    count = 0
+    try:
+        for entry in sorted(root.rglob("*")):
+            count += 1
+            if count > _MAX_ENTRIES_PER_SUMMARY:
+                break
+            rel = entry.relative_to(root)
+            rel_str = str(rel)
+            if entry.is_file():
+                files.append(rel_str)
+                try:
+                    total += entry.stat().st_size
+                except OSError:
+                    # Race: file existed in glob but vanished by stat.
+                    # Treat as zero-size and continue. Not a fatal error.
+                    pass
+            elif entry.is_dir():
+                dirs.append(rel_str)
+    except (OSError, PermissionError) as e:
+        # Don't blow up the response — return what we have so the caller
+        # at least sees the root + the readability problem in the log.
+        logger.warning("describe_tenant_data partial: %s", e)
+
+    summary["files"] = files
+    summary["dirs"] = dirs
+    summary["total_size_bytes"] = total
+    return summary
+
+
+def describe_shared_data() -> dict:
+    """Read-only summary of the legacy single-tenant `~/.delimit/` view.
+
+    Used by the shared-bearer (founder/system) path on /tenant/data.
+    Returns the same shape as describe_tenant_data minus `user_id`
+    (there is no user_id for the shared-bearer caller — it's the
+    founder/system).
+    """
+    # Reuse the same _MAX_ENTRIES_PER_SUMMARY cap. Founder's `~/.delimit/`
+    # typically has hundreds of files (memory.jsonl, ledger.jsonl,
+    # evidence/, daemon/, etc), so truncation is realistic.
+    home = os.environ.get("DELIMIT_HOME")
+    root = Path(home).expanduser().resolve() if home else (Path.home() / ".delimit")
+    summary: dict = {
+        "user_id": "",  # shared-bearer: no tenant scope
+        "data_root": str(root),
+        "exists": root.is_dir(),
+        "files": [],
+        "dirs": [],
+        "total_size_bytes": 0,
+        "cap_bytes": None,
+    }
+    if not summary["exists"]:
+        return summary
+
+    files: list[str] = []
+    dirs: list[str] = []
+    total = 0
+    count = 0
+    try:
+        for entry in sorted(root.rglob("*")):
+            # Skip the tenants/ subdir from the shared view — that's the
+            # per-tenant tree, which the founder views via the dashboard's
+            # tenant-list / admin surface, not as part of her own data.
+            try:
+                if entry.relative_to(root).parts[:1] == ("tenants",):
+                    continue
+            except ValueError:
+                pass
+            count += 1
+            if count > _MAX_ENTRIES_PER_SUMMARY:
+                break
+            rel_str = str(entry.relative_to(root))
+            if entry.is_file():
+                files.append(rel_str)
+                try:
+                    total += entry.stat().st_size
+                except OSError:
+                    pass
+            elif entry.is_dir():
+                dirs.append(rel_str)
+    except (OSError, PermissionError) as e:
+        logger.warning("describe_shared_data partial: %s", e)
+
+    summary["files"] = files
+    summary["dirs"] = dirs
+    summary["total_size_bytes"] = total
+    return summary

package/gateway/ai/tenant_paths.py

@@ -0,0 +1,150 @@
+"""LED-2268 P0 Phase 0.2 — tenant-scoped filesystem layout.
+
+The gateway today stores everything under `~/.delimit/` (memory.jsonl,
+ledger.jsonl, evidence/, etc). That's correct for the single-tenant
+founder install but doesn't generalize once paying customers run their
+own tenants against a shared gateway host.
+
+This module owns the path-resolver primitive for the per-tenant layout:
+
+    ~/.delimit/                      ← legacy / shared root (unchanged)
+    ~/.delimit/tenants/
+        <safe-user-id>/              ← one dir per resolved API-key user
+            memory.jsonl
+            ledger.jsonl
+            evidence/
+            ...
+
+Phase 0.2 ONLY ships the resolver + sanitiser + base-dir creation. No
+existing storage is migrated; no endpoint is yet rerouted through here.
+Phase 0.3 will add the first endpoint that uses tenant_data_root() and
+copy the founder's existing single-tenant data into her own tenant
+folder.
+
+Security note: the user_id segment comes from Supabase
+`user_api_keys.user_id` (which itself comes from NextAuth users.id, a
+GitHub-OAuth-derived string). It's NEVER raw user input from the
+request — but we still sanitise it defensively so a malformed value in
+the DB can't escape into adjacent dirs via `..` or NUL bytes.
+"""
+from __future__ import annotations
+
+import os
+import re
+import string
+from pathlib import Path
+from typing import Optional
+
+
+# Base of the whole per-tenant tree. Lives under the existing delimit
+# home so backup/restore tooling sees it without extra wiring.
+def _delimit_home() -> Path:
+    """Resolve ~/.delimit/ — same convention as the rest of the gateway."""
+    home = os.environ.get("DELIMIT_HOME")
+    if home:
+        return Path(home).expanduser().resolve()
+    return Path.home() / ".delimit"
+
+
+_TENANTS_DIRNAME = "tenants"
+# Allowed chars in a sanitised user-id segment. Conservative: ASCII
+# alphanumerics + a small set of safe punctuation. Nothing that could
+# be interpreted by the shell, the path parser, or a downstream tool.
+_SAFE_CHARS = frozenset(string.ascii_letters + string.digits + "-_.")
+# Max chars in a single user-id segment. Filesystems generally allow
+# 255-byte basenames; we cap well below that and prefix-truncate +
+# hash-suffix any longer input so distinct over-long IDs don't collide.
+_MAX_SEGMENT_LEN = 64
+
+
+def safe_user_segment(user_id: str) -> Optional[str]:
+    """Sanitise a user_id into a filesystem-safe directory name.
+
+    Returns None for empty / suspicious input so callers MUST handle
+    the rejection rather than silently writing to a default dir. The
+    intentional asymmetry from `_hash_key` (which always produces a
+    valid hex string) is that an unauthenticated request can't land
+    here — only an already-validated identity does — so a None here
+    represents a corrupted DB row, not a normal failure mode.
+
+    Strategy:
+      - Strip whitespace, lowercase.
+      - Replace any char outside the safe set with '_'.
+      - If result is empty or only underscores, reject.
+      - If result is longer than _MAX_SEGMENT_LEN, truncate + append
+        a short hash suffix so distinct over-long IDs don't collide.
+      - Reject anything that resolves to '.' or '..' (defence in depth
+        against malformed DB rows like literally the string "..").
+    """
+    if not isinstance(user_id, str) or not user_id:
+        return None
+    s = user_id.strip().lower()
+    if not s:
+        return None
+    # Substitute unsafe chars one-for-one — preserves length / readability
+    # for the common case (NextAuth GitHub uses bare integer-ish strings).
+    safe = "".join(c if c in _SAFE_CHARS else "_" for c in s)
+    if not safe or safe.strip("_") == "":
+        return None
+    if safe in (".", ".."):
+        return None
+    if len(safe) > _MAX_SEGMENT_LEN:
+        # Truncate to (max - 9) so the suffix `-<8hex>` fits in budget.
+        import hashlib
+        digest = hashlib.sha256(s.encode("utf-8")).hexdigest()[:8]
+        safe = safe[: _MAX_SEGMENT_LEN - 9] + "-" + digest
+    return safe
+
+
+def tenants_root() -> Path:
+    """The shared parent of all per-tenant dirs. Always under DELIMIT_HOME."""
+    return _delimit_home() / _TENANTS_DIRNAME
+
+
+def tenant_data_root(user_id: str, *, create: bool = False) -> Optional[Path]:
+    """Resolve the on-disk root for a specific tenant's data.
+
+    Returns None if `user_id` doesn't sanitise to a usable segment.
+    Caller treats that as "unauthorised" — same shape as the validator.
+
+    If `create=True`, ensures the directory exists (mkdir -p, mode 0700).
+    Default is read-only resolve so this can be called on hot paths
+    without making syscalls when the dir is already present.
+    """
+    seg = safe_user_segment(user_id)
+    if seg is None:
+        return None
+    root = tenants_root() / seg
+    # Defence in depth: ensure the resolved path stays under tenants_root.
+    # Belt-and-braces against an unforeseen sanitiser bypass.
+    try:
+        if tenants_root().resolve() not in root.resolve().parents and \
+                root.resolve() != tenants_root().resolve():
+            return None
+    except (OSError, RuntimeError):
+        return None
+    if create:
+        root.mkdir(parents=True, exist_ok=True, mode=0o700)
+        # Ensure tenants_root itself has the right mode too — first-
+        # ever tenant write would otherwise inherit umask.
+        try:
+            tenants_root().chmod(0o700)
+        except OSError:
+            pass
+    return root
+
+
+def list_tenants() -> list[str]:
+    """List the segment names of all tenants currently with on-disk data.
+
+    Used by maintenance / audit / backup tooling. Returns an empty list
+    when no tenants exist yet (the directory simply doesn't exist).
+    """
+    root = tenants_root()
+    if not root.is_dir():
+        return []
+    out: list[str] = []
+    for entry in root.iterdir():
+        if entry.is_dir() and entry.name and not entry.name.startswith("."):
+            out.append(entry.name)
+    return sorted(out)