eljay-ai 1.1.0

package/agent/providers/web.py

@@ -0,0 +1,155 @@
+"""Web search provider — free, no-API-key research via DuckDuckGo Lite.
+
+Uses ``urllib.request`` (stdlib) and a tiny ``html.parser`` to extract
+search-result titles, URLs, and snippets from the DDG Lite endpoint.
+No API key required, respects robots.txt, minimal requests.
+"""
+
+from __future__ import annotations
+
+import html as html_lib
+import re
+import urllib.parse
+import urllib.request
+from html.parser import HTMLParser
+
+from .base import Provider
+
+DDG_LITE = "https://lite.duckduckgo.com/lite/"
+USER_AGENT = "ElJay-AI/1.0 (local research agent; stdlib only)"
+MAX_RESULTS = 10
+
+
+class _DDGResultParser(HTMLParser):
+    """Extract result anchors + snippets from the DDG Lite HTML table."""
+
+    def __init__(self):
+        super().__init__()
+        self.results: list[tuple[str, str, str]] = []  # (title, url, snippet)
+        self._in_link = False
+        self._in_snippet = False
+        self._current_title = ""
+        self._current_href = ""
+        self._snippet_parts: list[str] = []
+
+    def handle_starttag(self, tag, attrs):
+        attrs_d = dict(attrs)
+        if tag == "a" and attrs_d.get("class") == "result-link":
+            self._in_link = True
+            self._current_href = attrs_d.get("href", "")
+        elif tag == "td" and attrs_d.get("class") == "result-snippet":
+            self._in_snippet = True
+
+    def handle_endtag(self, tag):
+        if tag == "a" and self._in_link:
+            self._in_link = False
+            if self._current_title and self._extract_url(self._current_href):
+                self.results.append(
+                    (
+                        html_lib.unescape(self._current_title).strip(),
+                        self._extract_url(self._current_href),
+                        "",
+                    )
+                )
+        elif tag == "td" and self._in_snippet:
+            self._in_snippet = False
+            snippet = " ".join(self._snippet_parts).strip()
+            if self.results and not self.results[-1][2]:
+                title, url, _ = self.results[-1]
+                self.results[-1] = (title, url, snippet)
+        self._current_title = ""
+        self._snippet_parts = []
+
+    def handle_data(self, data):
+        if self._in_link:
+            self._current_title += data
+        elif self._in_snippet:
+            self._snippet_parts.append(data)
+
+    @staticmethod
+    def _extract_url(href: str) -> str:
+        """Extract the real destination from a DDG redirect URL."""
+        parsed = urllib.parse.urlparse(href)
+        params = urllib.parse.parse_qs(parsed.query)
+        uddg = params.get("uddg", [""])[0]
+        if uddg:
+            return html_lib.unescape(uddg)
+        return href
+
+
+def search_ddg(query: str, max_results: int = MAX_RESULTS) -> list[dict]:
+    """Search DuckDuckGo Lite and return structured results."""
+    url = DDG_LITE + "?" + urllib.parse.urlencode({"q": query, "kl": "us-en"})
+    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
+    try:
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            html_text = resp.read().decode("utf-8", errors="replace")
+    except Exception as exc:
+        return [{"error": f"Search request failed: {exc}"}]
+
+    parser = _DDGResultParser()
+    parser.feed(html_text)
+
+    out = []
+    for title, url, snippet in parser.results[:max_results]:
+        out.append(
+            {"title": title or "(no title)", "url": url, "snippet": snippet}
+        )
+    if not out:
+        out.append({"error": "No results returned for this query."})
+    return out
+
+
+def fetch_web_page(url: str, max_chars: int = 8000) -> str:
+    """Fetch *url* and return readable text (tags stripped).
+
+    Honest error messages on failure; never raises to the caller.
+    """
+    if not url.startswith(("http://", "https://")):
+        return f"ERROR: not a valid URL: {url}"
+    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
+    try:
+        with urllib.request.urlopen(req, timeout=20) as resp:
+            raw = resp.read()
+    except Exception as exc:
+        return f"ERROR: could not fetch {url}: {exc}"
+
+    try:
+        text = raw.decode("utf-8", errors="replace")
+    except Exception:
+        text = raw.decode("latin-1", errors="replace")
+
+    # Strip scripts and styles.
+    text = re.sub( r"<(script|style)[^>]*>.*?</\1>", "", text, flags=re.DOTALL | re.IGNORECASE)
+    # Strip remaining HTML tags.
+    text = re.sub(r"<[^>]+>", " ", text)
+    # Collapse whitespace.
+    text = re.sub(r"\s+", " ", text).strip()
+    if len(text) > max_chars:
+        text = text[:max_chars] + " ... [truncated]"
+    return text
+
+
+class WebSearchProvider(Provider):
+    """Free, no-API-key web search via DuckDuckGo Lite."""
+
+    def __init__(self):
+        super().__init__(
+            name="duckduckgo",
+            label="DuckDuckGo (free web)",
+            capabilities=["web-search", "web-fetch"],
+        )
+        self.query: str | None = None
+
+    def detect(self) -> bool:
+        """DDG Lite is always available in principle; we test reachability."""
+        try:
+            results = search_ddg("test connectivity", max_results=1)
+            if results and "error" not in results[0]:
+                self.available = True
+                return True
+            self.error = "Could not reach DuckDuckGo Lite."
+            return False
+        except Exception as exc:
+            self.error = str(exc)
+            return False

package/agent/router.py

@@ -0,0 +1,200 @@
+"""Agent routing and orchestration for ElJay AI.
+
+The ``AgentRouter`` bridges the user's natural-language request and the
+specialized agent definitions in ``agent/agents/registry.py``.  It decides:
+
+* Which agent persona to use (coding, research, image, video, knowledge, or
+  the main coordinator).
+* Which system-prompt additions to inject.
+* Which tools to make available.
+* Which skills and knowledge entries are relevant.
+
+This is a lightweight, deterministic layer — routing uses keyword matching
+(no LLM call) so it is fast and predictable.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Optional
+
+from .agents import AgentDefinition, get_agent, list_agents, route_request
+from .providers import PROVIDERS, get_provider_capable_of
+from .skills import Skills
+from .knowledge import KnowledgeBase
+
+
+# Capability → provider-name map (matches Provider names in PROVIDERS).
+_PROVIDER_NAMES = {
+    "text": "ollama",
+    "coding": "ollama",
+    "web-search": "duckduckgo",
+    "web-fetch": "duckduckgo",
+    "image-generation": "image-local",
+    "video-generation": "video-local",
+}
+
+
+class AgentRouter:
+    """Route user queries to the right agent persona.
+
+    Parameters
+    ----------
+    core
+        The :class:`AgentCore` instance (optional, for future multi-agent
+        delegation).
+    workspace
+        The :class:`Workspace` (optional, used for project-local context).
+    skills
+        The :class:`Skills` instance (may be empty if no ``skills/`` dir).
+    knowledge
+        The :class:`KnowledgeBase` instance (may be empty).
+    """
+
+    def __init__(
+        self,
+        core: object | None = None,
+        skills: Skills | None = None,
+        knowledge: KnowledgeBase | None = None,
+        workspace: object | None = None,
+    ) -> None:
+        self.core = core
+        self.workspace = workspace
+        self.skills = skills or _EmptySkills()
+        self.knowledge = knowledge or _EmptyKnowledge()
+        self.registry = list_agents()
+        self._providers_checked = False
+        self._provider_status: dict[str, bool] = {}
+
+    # ------------------------------------------------------------------
+    # Provider status
+    # ------------------------------------------------------------------
+
+    def provider_status(self) -> dict[str, bool]:
+        """Return ``{provider_name: is_available}`` for all providers."""
+        if not self._providers_checked:
+            self._providers_checked = True
+            for p in PROVIDERS:
+                key = p.name
+                try:
+                    self._provider_status[key] = p.is_available()
+                except Exception:
+                    self._provider_status[key] = False
+        return self._provider_status
+
+    # ------------------------------------------------------------------
+    # Routing
+    # ------------------------------------------------------------------
+
+    def route(self, query: str) -> AgentDefinition:
+        """Return the agent definition best suited to *query*."""
+        providers = self.provider_status()
+        # Translate capability names to boolean availability.
+        available = {}
+        for p in PROVIDERS:
+            available[p.name] = providers.get(p.name, False)
+        return route_request(query, available)
+
+    def __len__(self) -> int:
+        """Return the number of registered agents."""
+        return len(self.registry)
+
+    def agent_by_name(self, name: str) -> Optional[AgentDefinition]:
+        """Look up an agent by name (case-insensitive)."""
+        return get_agent(name.lower())
+
+    def all_agents(self) -> list[AgentDefinition]:
+        """Return all agent definitions, most specific first."""
+        return list_agents()
+
+    # ------------------------------------------------------------------
+    # System prompt augmentation
+    # ------------------------------------------------------------------
+
+    def system_additions(self, agent: AgentDefinition) -> str:
+        """Return the system-prompt additions for *agent*."""
+        return agent.system_additions
+
+    def available_tools(self, agent: AgentDefinition) -> list[str]:
+        """Return the list of tool names the agent may use.
+
+        The ``main`` coordinator gets all tools; specialised agents get only
+        the tools listed in their definition.
+        """
+        if agent.name == "main":
+            return []  # empty list = "all tools" (convention)
+        return list(agent.tools)
+
+    # ------------------------------------------------------------------
+    # Skills & Knowledge injection
+    # ------------------------------------------------------------------
+
+    def relevant_skills(self, query: str, limit: int = 3) -> list[str]:
+        """Return relevant skill content snippets for *query*."""
+        skills = self.skills.relevant_to(query)
+        return [s.content for s in skills[:limit]]
+
+    def relevant_knowledge(self, query: str, limit: int = 3) -> list[str]:
+        """Return relevant knowledge entries for *query*."""
+        entries = self.knowledge.search(query)
+        return [e.content for e in entries[:limit]]
+
+    def inject_context(self, query: str, agent: AgentDefinition) -> str:
+        """Build a context string to append to the system prompt.
+
+        Includes relevant skills and knowledge entries for the agent + query.
+        """
+        parts: list[str] = []
+        skills = self.relevant_skills(query)
+        if skills:
+            parts.append("## Relevant Skills\n\n" + "\n\n---\n\n".join(skills))
+        knowledge = self.relevant_knowledge(query)
+        if knowledge:
+            parts.append("## Relevant Knowledge\n\n" + "\n\n---\n\n".join(knowledge))
+        return "\n\n".join(parts)
+
+    # ------------------------------------------------------------------
+    # Orchestration (future: multi-agent delegation)
+    # ------------------------------------------------------------------
+
+    def needs_delegation(self, query: str) -> bool:
+        """Return True if this query is better handled by a specialist agent.
+
+        The Main coordinator delegates when the matched agent is not ``main``.
+        """
+        agent = self.route(query)
+        return agent.name != "main"
+
+    def delegated_agent_for(self, query: str) -> Optional[AgentDefinition]:
+        """Return the specialist agent for *query*, or None if Main is best."""
+        agent = self.route(query)
+        if agent.name == "main":
+            return None
+        return agent
+
+
+# ---------------------------------------------------------------------------
+# Lightweight empty fallbacks so the router works even when no skills/
+# knowledge directories exist yet.
+# ---------------------------------------------------------------------------
+
+class _EmptySkills:
+    def relevant_to(self, query: str) -> list:
+        return []
+
+    def all_skills(self) -> list:
+        return []
+
+    def summary(self) -> str:
+        return "no skills"
+
+
+class _EmptyKnowledge:
+    def search(self, query: str) -> list:
+        return []
+
+    def entries(self) -> list:
+        return []
+
+    def summary(self) -> str:
+        return "empty"

package/agent/rules.py

@@ -0,0 +1,52 @@
+"""Project rules: load AGENTS.md and turn it into system instructions.
+
+The agent must read the project's rules before making changes, so the rules are
+appended to the system prompt at startup.
+"""
+
+from __future__ import annotations
+
+from .workspace import Workspace
+
+RULES_FILENAMES = ("AGENTS.md", "agents.md", ".agent/AGENTS.md")
+MAX_RULES_CHARS = 8000
+
+
+def find_rules_file(ws: Workspace) -> str | None:
+    for name in RULES_FILENAMES:
+        try:
+            path = ws.resolve(name)
+        except ValueError:
+            continue
+        if path.is_file():
+            return name
+    return None
+
+
+def load_rules(ws: Workspace) -> tuple[str, str | None]:
+    """Return (rules_text, filename). rules_text is "" when there are none."""
+    name = find_rules_file(ws)
+    if name is None:
+        return "", None
+    try:
+        path = ws.resolve(name)
+        text = path.read_text(encoding="utf-8", errors="replace")
+    except (OSError, ValueError):
+        return "", None
+
+    if len(text) > MAX_RULES_CHARS:
+        text = text[:MAX_RULES_CHARS] + "\n... (rules truncated)"
+    return text.strip(), name
+
+
+def build_system_prompt(base_prompt: str, rules_text: str, source: str | None) -> str:
+    if not rules_text:
+        return base_prompt
+    return (
+        f"{base_prompt}\n\n"
+        f"PROJECT RULES (from {source}) — these are binding. Follow them, and ask "
+        "before doing anything they forbid:\n"
+        "-----\n"
+        f"{rules_text}\n"
+        "-----"
+    )

package/agent/runner.py

@@ -0,0 +1,168 @@
+"""Terminal command execution — deliberately narrow.
+
+Safety model:
+- Commands are NEVER auto-run (never `safe`). Every command needs approval.
+- A command whose program is on the allow-list is `ask`; anything else is `high`.
+- Known-dangerous patterns force `high` regardless of the program.
+- Shell operators (|, &, ;, <, >, backticks, $(...)) are refused: we run a single
+  program without a shell, so there is no shell injection surface.
+- The working directory is confined to the project root.
+- Output is captured, and the run is bounded by a timeout.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import shlex
+import shutil
+import subprocess
+
+from .tools import RISK_ASK, RISK_HIGH
+from .workspace import Workspace
+
+MAX_OUTPUT_CHARS = 20_000
+DEFAULT_TIMEOUT = 120
+MAX_TIMEOUT = 600
+
+# Programs that may run with a normal confirmation.
+ALLOWED_PROGRAMS = frozenset(
+    {
+        # python
+        "python", "python3", "py", "pip", "pip3", "pytest", "mypy", "ruff", "black",
+        # javascript / typescript
+        "node", "npm", "npx", "yarn", "pnpm", "bun", "tsc", "eslint", "vite", "jest",
+        # php
+        "php", "composer", "artisan",
+        # git (dangerous subcommands are caught below)
+        "git",
+        # other languages / build tools
+        "go", "cargo", "rustc", "dotnet", "java", "mvn", "gradle", "make",
+        "ruby", "bundle", "gem",
+        # harmless inspection
+        "echo", "ls", "dir", "cat", "type", "pwd", "where", "which", "whoami",
+        "node", "tree",
+    }
+)
+
+# Patterns that ALWAYS mean high risk, whatever the program.
+_HIGH_RISK_PATTERNS = tuple(
+    re.compile(p)
+    for p in (
+        r"\brm\b", r"\brmdir\b", r"\brd\b", r"\bdel\b", r"\berase\b",
+        r"\bformat\b", r"\bfdisk\b", r"\bmkfs\b",
+        r"\bshutdown\b", r"\breboot\b", r"\btaskkill\b", r"\bkill\b", r"\bsudo\b",
+        r"git\s+push", r"git\s+reset", r"git\s+clean", r"git\s+rebase",
+        r"git\s+checkout\s+--", r"git\s+branch\s+-d", r"git\s+filter-branch",
+        r"npm\s+uninstall", r"npm\s+install\s+(-g|--global)",
+        r"yarn\s+remove", r"pnpm\s+remove", r"composer\s+remove",
+        r"pip\s+uninstall", r"pip\s+install\s+--user",
+        r"drop\s+(database|table)", r"truncate\s+table", r"delete\s+from",
+        r"\|\s*(ba|z|)sh\b", r"\bcurl\b[^|]*\|", r"\bwget\b[^|]*\|",
+        r">\s*/dev/sd", r"\bchmod\s+777\b",
+    )
+)
+
+# Shell operators we do not support (we never use a shell).
+_SHELL_META = re.compile(r"[|&;<>]|\$\(|`")
+
+
+def analyse(command: str) -> tuple[list[str] | None, str]:
+    """Parse a command into argv. Returns (argv, "") or (None, error_message)."""
+    text = (command or "").strip()
+    if not text:
+        return None, "ERROR: command is required"
+    if _SHELL_META.search(text):
+        return None, (
+            "ERROR: shell operators (|, &, ;, <, >, backticks, $(...)) are not "
+            "allowed. Run a single command with no pipes or redirection."
+        )
+    try:
+        parts = shlex.split(text, posix=True)
+    except ValueError as exc:
+        return None, f"ERROR: could not parse the command: {exc}"
+    if not parts:
+        return None, "ERROR: command is required"
+    return parts, ""
+
+
+def _program_name(token: str) -> str:
+    name = os.path.basename(token).lower()
+    for ext in (".exe", ".cmd", ".bat", ".com"):
+        if name.endswith(ext):
+            name = name[: -len(ext)]
+    return name
+
+
+def classify_command(command: str) -> str:
+    """Return RISK_ASK for allow-listed programs, RISK_HIGH otherwise."""
+    lowered = (command or "").lower()
+    for pattern in _HIGH_RISK_PATTERNS:
+        if pattern.search(lowered):
+            return RISK_HIGH
+    argv, _err = analyse(command)
+    if not argv:
+        return RISK_HIGH
+    return RISK_ASK if _program_name(argv[0]) in ALLOWED_PROGRAMS else RISK_HIGH
+
+
+def run_command(
+    ws: Workspace,
+    command: str,
+    cwd: str = ".",
+    timeout: int = DEFAULT_TIMEOUT,
+) -> str:
+    """Run one command in the project root and return a readable report."""
+    argv, error = analyse(command)
+    if argv is None:
+        return error
+
+    try:
+        workdir = ws.resolve(cwd or ".")
+    except ValueError as exc:
+        return f"ERROR: {exc}"
+    if not workdir.is_dir():
+        return f"ERROR: working directory does not exist: {cwd}"
+
+    program = shutil.which(argv[0])
+    if program is None:
+        return f"ERROR: command not found: {argv[0]}"
+    argv[0] = program
+
+    try:
+        timeout = max(1, min(int(timeout), MAX_TIMEOUT))
+    except (TypeError, ValueError):
+        timeout = DEFAULT_TIMEOUT
+
+    try:
+        proc = subprocess.run(
+            argv,
+            cwd=str(workdir),
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+            timeout=timeout,
+        )
+    except subprocess.TimeoutExpired:
+        return (
+            f"Command: {command}\nResult: FAILED\n"
+            f"Reason: timed out after {timeout}s and was stopped."
+        )
+    except (OSError, ValueError) as exc:
+        return f"ERROR: could not run '{command}': {exc}"
+
+    output = proc.stdout or ""
+    if proc.stderr:
+        output += ("\n" if output else "") + "[stderr]\n" + proc.stderr
+    output = output.strip()
+    if len(output) > MAX_OUTPUT_CHARS:
+        output = output[:MAX_OUTPUT_CHARS] + "\n... (output truncated)"
+
+    status = "succeeded" if proc.returncode == 0 else "FAILED"
+    report = f"Command: {command}\nExit code: {proc.returncode} ({status})"
+    if output:
+        report += f"\n--- output ---\n{output}"
+    else:
+        report += "\n(no output)"
+    return report