@octocodeai/freellama-mcp-server 0.5.0 → 0.6.0

package/adapters/agent_transport.py

@@ -1,215 +0,0 @@
-"""Shared Ollama/FreeLlama chat transport for local research adapters."""
-
-from __future__ import annotations
-
-import os
-import hashlib
-import json
-import re
-from pathlib import Path
-from typing import Any
-from urllib.error import HTTPError, URLError
-from urllib.request import Request, urlopen
-
-
-def _read_identity_json(url: str) -> dict[str, Any]:
-    with urlopen(Request(url, headers=request_headers()), timeout=5) as response:
-        payload = response.read(1024 * 1024 + 1)
-    if len(payload) > 1024 * 1024:
-        raise ValueError("model identity catalog exceeds size limit")
-    return json.loads(payload)
-
-
-def resolve_model_identity(
-    endpoint: str, model: str, system_prompt: str, *, read_json=None,
-) -> dict[str, Any]:
-    """Resolve immutable manifest identity once per adapter run, without loading a model.
-
-    Managed catalogs honor exact CPU backend assignment. Failure disables persistent calibration;
-    a mutable tag or a hash of the prompt alone must never masquerade as model identity.
-    """
-    endpoint = endpoint.rstrip("/")
-    managed_suffix = "/_freellama/v1/tasks"
-    url = (endpoint[:-len(managed_suffix)] + "/_freellama/v1/models"
-           if endpoint.endswith(managed_suffix) else endpoint + "/api/tags")
-    unknown = {"identity": "", "verified": False, "scope": "current_process_only"}
-    try:
-        payload = (read_json or _read_identity_json)(url)
-        entries = payload.get("models", []) if isinstance(payload, dict) else []
-        if not isinstance(entries, list):
-            return unknown
-        names = {model}
-        if ":" not in model.rsplit("/", 1)[-1]:
-            names.add(model + ":latest")
-        entry = next((item for item in entries if isinstance(item, dict)
-                      and item.get("name", item.get("model")) in names), None)
-        digest = entry.get("digest") if entry else None
-        if not isinstance(digest, str) or not re.fullmatch(r"(?:sha256:)?[0-9a-fA-F]{64}", digest):
-            return unknown
-        digest = digest.removeprefix("sha256:").lower()
-        contract = {"endpoint": endpoint, "model": model, "digest": digest,
-                    "system_hash": hashlib.sha256(system_prompt.encode("utf-8")).hexdigest()}
-        return {"identity": hashlib.sha256(json.dumps(contract, sort_keys=True).encode()).hexdigest(),
-                "verified": True, "scope": "manifest_digest_and_system_prompt", "digest": digest}
-    except (OSError, ValueError, TypeError):
-        return unknown
-
-
-def retryable_chat_error(error: Exception) -> bool:
-    """Retry known refusals only; a timeout may leave inference running upstream."""
-    if isinstance(error, HTTPError):
-        return error.code == 503
-    return isinstance(error, URLError) and isinstance(error.reason, ConnectionRefusedError)
-
-
-def chat_error_details(error: Exception) -> tuple[str, dict[str, Any] | None]:
-    """Retain bounded application refusal receipts, excluding request/auth payloads."""
-    diagnostic = f"{type(error).__name__}: {error}"
-    if not isinstance(error, HTTPError):
-        return diagnostic, None
-    details: dict[str, Any] = {"status": error.code}
-    try:
-        body = error.read(64 * 1024 + 1)
-        if len(body) > 64 * 1024:
-            return diagnostic, details
-        payload = json.loads(body)
-        if not isinstance(payload, dict) or not isinstance(payload.get("error"), str):
-            return diagnostic, details
-        receipt_fields = {
-            "error", "code", "reason", "retry_after_seconds", "resource_admission", "lifecycle",
-            "placement", "execution", "admission", "route", "decision", "quality", "feedback",
-        }
-        payload_fields = {
-            "prompt", "system", "system_prompt", "messages", "content", "input", "images", "tools",
-            "request", "request_options", "headers", "authorization", "token", "api_key", "auth_token",
-        }
-
-        def receipt_value(value: Any) -> Any:
-            if isinstance(value, dict):
-                return {key: receipt_value(item) for key, item in value.items() if key.lower() not in payload_fields}
-            if isinstance(value, list):
-                return [receipt_value(item) for item in value]
-            return value
-
-        receipt = {key: receipt_value(value) for key, value in payload.items() if key in receipt_fields}
-        details["receipt"] = receipt
-        diagnostic = f"HTTP {error.code}: {receipt['error']}"
-    except (OSError, ValueError, RecursionError):
-        pass
-    finally:
-        error.close()
-    return diagnostic, details
-
-
-class PromptCacheUsage:
-    """Aggregate optional Ollama cache reads without treating unavailable turns as zero."""
-
-    def __init__(self) -> None:
-        self.calls = 0
-        self.reported_calls = 0
-        self.reported_tokens = 0
-        self.reported_prompt_tokens = 0
-
-    def observe(self, response: dict[str, Any]) -> None:
-        self.calls += 1
-        cached = response.get("prompt_eval_cached_count")
-        prompt = response.get("prompt_eval_count")
-        if type(cached) is int and type(prompt) is int and 0 <= cached <= prompt:
-            self.reported_calls += 1
-            self.reported_tokens += cached
-            self.reported_prompt_tokens += prompt
-
-    @property
-    def tokens(self) -> int | None:
-        return self.reported_tokens if self.calls > 0 and self.calls == self.reported_calls else None
-
-    def metadata(self) -> dict[str, Any]:
-        return {
-            "status": "reported" if self.tokens is not None else (
-                "partially_reported" if self.reported_calls else "not_reported"
-            ),
-            "calls": self.calls,
-            "reported_calls": self.reported_calls,
-            "reported_tokens": self.reported_tokens if self.reported_calls else None,
-            "hit_ratio": self.reported_tokens / self.reported_prompt_tokens
-            if self.tokens is not None and self.reported_prompt_tokens else None,
-        }
-
-
-def request_headers() -> dict[str, str]:
-    """Build transport headers, reading a bearer token from a file rather than process args."""
-    headers = {"content-type": "application/json"}
-    token_file = os.environ.get("FREELLAMA_AUTH_TOKEN_FILE", "").strip()
-    if not token_file:
-        return headers
-    token = Path(token_file).read_text(encoding="utf-8").strip()
-    if len(token) < 32 or any(character.isspace() for character in token):
-        raise ValueError("FREELLAMA_AUTH_TOKEN_FILE must contain one token of at least 32 bytes")
-    headers["authorization"] = f"Bearer {token}"
-    return headers
-
-
-def chat_request(
-    endpoint: str,
-    model: str,
-    messages: list[dict[str, Any]],
-    options: dict[str, Any],
-    think: bool,
-    keep_alive: str,
-    execution_preference: str = "auto",
-    min_placement_evidence: str = "configured",
-) -> tuple[str, dict[str, Any]]:
-    """Build one direct-Ollama or managed-FreeLlama request.
-
-    `endpoint` selects the transport: a URL ending in `/_freellama/v1/tasks` is managed. The
-    benchmark can still target raw Ollama for controlled comparisons; MCP always supplies the
-    managed URL so coding-agent turns share routing, admission, physical-placement receipts, and
-    adaptive feedback with ordinary `run_task` calls.
-    """
-    endpoint = endpoint.rstrip("/")
-    if endpoint.endswith("/_freellama/v1/tasks"):
-        managed_options = {key: value for key, value in options.items() if key != "num_ctx"}
-        return endpoint, {
-            "task": "coding",
-            "objective": "fastest",
-            "model": model,
-            "context_tokens": options["num_ctx"],
-            "execution_preference": execution_preference,
-            "min_placement_evidence": min_placement_evidence,
-            "messages": messages,
-            "keep_alive": keep_alive,
-            "request_options": {
-                "format": "json",
-                "think": think,
-                "options": managed_options,
-            },
-        }
-    return f"{endpoint}/api/chat", {
-        "model": model,
-        "messages": messages,
-        "stream": False,
-        "truncate": False,
-        "shift": False,
-        "format": "json",
-        "think": think,
-        "keep_alive": keep_alive,
-        "options": options,
-    }
-
-
-def unwrap_chat_response(
-    payload: dict[str, Any], execution_receipts: list[dict[str, Any]]
-) -> dict[str, Any]:
-    """Return the Ollama response and retain managed execution proof when present."""
-    response = payload.get("response")
-    if not isinstance(response, dict):
-        return payload
-    execution_receipts.append(
-        {
-            "model": payload.get("route", {}).get("selected_model"),
-            "execution": payload.get("execution"),
-            "admission": payload.get("admission"),
-            "feedback": payload.get("feedback"),
-        }
-    )
-    return response

package/adapters/bash_agent.py

@@ -1,345 +0,0 @@
-#!/usr/bin/env python3
-"""Agent B adapter: local Ollama model restricted to raw Linux/bash commands only.
-
-Same adapter contract and decoding settings as octocode_agent.py (see
-benchmark/harness/references/adapters.md and benchmark/local/docs/03-agent-b-bash.md) — the only
-variable under test is the tool surface: this agent has no structured tool schema, just a shell.
-"""
-
-from __future__ import annotations
-
-from contextlib import ExitStack
-import json
-import os
-import subprocess
-import time
-from pathlib import Path
-from typing import Any
-from urllib.error import HTTPError, URLError
-from urllib.request import Request, urlopen
-
-from agent_context import (
-    AgentContextManager,
-    AgentRuntimeConfig,
-    PARSE_REPAIR_NOTICE,
-    REPEAT_NOTICE,
-    ObservationStore,
-    call_signature,
-    parse_json_action,
-    write_failure_result,
-)
-import shell_sandbox
-from shell_sandbox import ReadOnlyShell
-from agent_transport import chat_request, request_headers, unwrap_chat_response, PromptCacheUsage, retryable_chat_error, resolve_model_identity, chat_error_details
-
-# Nothing is clipped any more. `calls[].result` is written to result.json on disk and read by the
-# MCP layer — it never enters the model's context, so there was never a reason to shorten it. What
-# the MODEL sees is paginated instead: one page plus an exact instruction for fetching the next.
-# See the pagination section of agent_context.py for why clipping was data loss, not economy.
-
-# Validation, restricted bash, a scrubbed environment and (where the host allows) an OS sandbox
-# live in shell_sandbox; see its module docstring for why the old regex denylist was not enough.
-assert_command_confined = shell_sandbox.assert_command_confined
-
-
-def request_json(url: str, payload: dict[str, Any], timeout_seconds: float) -> dict[str, Any]:
-    request = Request(url, data=json.dumps(payload).encode(), headers=request_headers(), method="POST")
-    with urlopen(request, timeout=timeout_seconds) as response:
-        return json.loads(response.read())
-
-
-def parse_action(content: str) -> dict[str, Any]:
-    value = parse_json_action(content)
-    action = value["action"]
-    if action == "shell" and (not isinstance(value.get("command"), str) or not value["command"].strip()):
-        raise ValueError("shell action requires a non-empty string command")
-    if action == "page" and (
-        not isinstance(value.get("step"), int)
-        or value["step"] < 1
-        or not isinstance(value.get("page"), int)
-        or value["page"] < 1
-    ):
-        raise ValueError("page action requires positive integer step and page")
-    if action == "finish" and not isinstance(value.get("answer"), str):
-        raise ValueError("finish action requires a string answer")
-    if action not in {"shell", "page", "finish"}:
-        raise ValueError(f"unsupported action for bash adapter: {action}")
-    return value
-
-
-def run_shell(root: Path, command_text: str, timeout_seconds: float, shell: ReadOnlyShell | None = None) -> str:
-    if shell is not None:
-        return shell.run(command_text, timeout_seconds)
-    with ReadOnlyShell(root) as scoped:
-        return scoped.run(command_text, timeout_seconds)
-
-
-def system_prompt() -> str:
-    return """You are a local code-research agent in a read-only workspace rooted at the current directory. Use raw POSIX shell commands without editors, special tools, or network access. Return exactly one JSON object per turn:
-
-{"action":"shell","command":"one shell command, e.g. grep -n \\"class Group\\" click/src/click/core.py"}
-{"action":"page","step":2,"page":2}
-{"action":"finish","answer":"concise final answer with repository-relative evidence"}
-
-These are the complete action schemas. Use only `shell`, `page`, or `finish`; every shown field is
-required with the shown type. An invalid shape is rejected and costs one bounded repair turn.
-
-Full command output is stored in pages; you see page 1 and the total. Context fitting can shorten
-an observation in the conversation. Request its stored page to recover the evidence without
-re-running the command. Claim "not found" only after checking the relevant search scope and pages.
-
-The shell is READ-ONLY and restricted. Available tools: ls, find, grep, rg, cat, head, tail, wc,
-sort, uniq, cut, tr, nl, sed (bounded read-only inline programs), jq, diff, xargs
-(only echo, printf, true, or false targets), tree, file, stat, and read-only git
-(log, show, grep, ls-files, diff, blame, status). There is no awk, python, network, output
-redirection, script-file loading, recursive symlink following, paths outside the workspace,
-$(...) substitution, $VARIABLES, or loops: write paths and patterns literally and
-chain tools with pipes, one pipeline per turn. A refused command costs a turn, so follow these
-rules. Be decisive: most tasks need 2-6 commands. Call finish as soon as the requested facts are
-established.
-
-SCOPE YOUR SEARCHES. A real workspace holds far more than its source, and an unscoped grep drowns
-the answer in vendored and generated files. Always exclude them:
-  grep -rn "PATTERN" . --exclude-dir={node_modules,target,.venv,venv,dist,build,.git,vendor,__pycache__,.octocode,site-packages}
-For implementation questions, prefer src/, packages/*/src/, lib/ and the repo root before fixtures,
-mocks, or examples. If the question targets tests or examples, inspect those files. Name the file
-that supplies the answer.
-If a search returns nothing, widen the pattern before concluding the thing does not exist: absence
-of a grep hit is weak evidence, and "not found" is only a real answer once you have looked in the
-source directories.
-For a production default, find its declaration before relying on test occurrences. Declarations look like
-`default=`, `unwrap_or(`, `const `, `static `, or a settings schema initializer.
-For a test-specific default or fixture, inspect its test declaration."""
-
-
-def _main(resources: ExitStack) -> int:
-    model = os.environ.get("FREELLAMA_TARGET_MODEL") or os.environ["FREELLAMA_BENCH_MODEL"]
-    workspace = Path(os.environ["FREELLAMA_BENCH_WORKSPACE"]).resolve()
-    prompt = Path(os.environ["FREELLAMA_BENCH_PROMPT"]).read_text(encoding="utf-8")
-    result_path = Path(os.environ["FREELLAMA_AGENT_RESULT"])
-    endpoint = os.environ.get("FREELLAMA_OLLAMA_ENDPOINT", "http://127.0.0.1:11434").rstrip("/")
-    managed_endpoint = os.environ.get("FREELLAMA_AGENT_MANAGED_ENDPOINT", "").rstrip("/")
-    execution_preference = os.environ.get("FREELLAMA_AGENT_EXECUTION_PREFERENCE", "auto")
-    min_placement_evidence = os.environ.get("FREELLAMA_AGENT_MIN_PLACEMENT_EVIDENCE", "configured")
-    try:
-        runtime = AgentRuntimeConfig.from_env(default_tool_timeout_seconds=30)
-    except ValueError as error:
-        answer = f"invalid research adapter configuration: {error}"
-        write_failure_result(result_path, answer)
-        print(answer)
-        return 1
-    messages: list[dict[str, Any]] = [
-        {"role": "system", "content": system_prompt()},
-        {"role": "user", "content": prompt},
-    ]
-    calls: list[dict[str, Any]] = []
-    usage = {"input_tokens": 0, "output_tokens": 0, "cache_read_tokens": None, "cache_write_tokens": None}
-    cache_usage = PromptCacheUsage()
-    metrics = {"load_ms": 0.0, "prompt_eval_ms": 0.0, "eval_ms": 0.0}
-    execution_receipts: list[dict[str, Any]] = []
-    answer = ""
-    failure: str | None = None
-    transport_error: dict[str, Any] | None = None
-    calibration_dir = os.environ.get("FREELLAMA_AGENT_TOKEN_CALIBRATION_DIR", "").strip()
-    calibration_model_identity = (
-        resolve_model_identity(managed_endpoint or endpoint, model, messages[0]["content"])
-        if calibration_dir else {"verified": False, "scope": "persistence_disabled", "identity": ""}
-    )
-    context_manager = AgentContextManager(
-        runtime,
-        model=model,
-        calibration_identity=calibration_model_identity["identity"],
-        calibration_dir=Path(calibration_dir) if calibration_dir and calibration_model_identity["verified"] else None,
-    )
-    try:
-        messages = context_manager.fit(messages)
-    except ValueError as error:
-        answer = f"research task does not fit the configured context safely: {error}"
-        write_failure_result(result_path, answer)
-        print(answer)
-        return 1
-    chat_options = {
-        "temperature": runtime.temperature,
-        "seed": runtime.seed,
-        "num_ctx": runtime.num_ctx,
-        "num_predict": runtime.num_predict,
-    }
-
-    def call_model() -> dict[str, Any]:
-        # Retry explicit overload/connection refusals only. Replaying a timeout or an uncertain
-        # transport failure can overlap a generation that is still consuming the runner.
-        last_error: Exception | None = None
-        for attempt in range(runtime.retry_attempts):
-            try:
-                transport_endpoint = (
-                    f"{managed_endpoint}/_freellama/v1/tasks" if managed_endpoint else endpoint
-                )
-                url, payload = chat_request(
-                    transport_endpoint,
-                    model,
-                    messages,
-                    chat_options,
-                    runtime.think,
-                    runtime.keep_alive,
-                    execution_preference,
-                    min_placement_evidence,
-                )
-                response = unwrap_chat_response(
-                    request_json(url, payload, runtime.request_timeout_seconds), execution_receipts
-                )
-                break
-            except (HTTPError, URLError, TimeoutError) as error:
-                if not retryable_chat_error(error):
-                    raise
-                last_error = error
-                if attempt + 1 < runtime.retry_attempts:
-                    time.sleep(runtime.retry_backoff_seconds)
-        else:
-            raise last_error  # type: ignore[misc]
-        usage["input_tokens"] += int(response.get("prompt_eval_count", 0))
-        cache_usage.observe(response)
-        usage["cache_read_tokens"] = cache_usage.tokens
-        usage["output_tokens"] += int(response.get("eval_count", 0))
-        metrics["load_ms"] += float(response.get("load_duration", 0)) / 1_000_000
-        metrics["prompt_eval_ms"] += float(response.get("prompt_eval_duration", 0)) / 1_000_000
-        metrics["eval_ms"] += float(response.get("eval_duration", 0)) / 1_000_000
-        context_manager.observe(messages, response.get("prompt_eval_count"))
-        return response
-
-    def refit() -> bool:
-        nonlocal messages, failure
-        try:
-            messages = context_manager.fit(messages)
-            return True
-        except ValueError as error:
-            failure = f"research context cannot be fitted safely: {error}"
-            return False
-
-    seen_calls: dict[str, int] = {}
-    observations = resources.enter_context(ObservationStore(runtime.context.observation_page_chars))
-    shell = resources.enter_context(ReadOnlyShell(workspace))
-    parse_failures = 0
-    for _ in range(runtime.max_turns):
-        # Transport failures are terminal (call_model already retried them). A *parse* failure is
-        # not: tell the model what was wrong with its reply and let it correct itself, keeping every
-        # tool result gathered so far.
-        try:
-            response = call_model()
-        except (HTTPError, URLError, TimeoutError) as error:
-            diagnostic, transport_error = chat_error_details(error)
-            failure = f"agent response failed: {diagnostic}"
-            break
-        raw = str(response.get("message", {}).get("content", ""))
-        try:
-            action = parse_action(raw)
-        except (ValueError, json.JSONDecodeError) as error:
-            parse_failures += 1
-            if parse_failures > runtime.max_parse_repairs:
-                failure = f"agent gave {parse_failures} unparseable replies: {type(error).__name__}: {error}"
-                break
-            messages.append({"role": "assistant", "content": raw[: runtime.parse_repair_echo_chars]})
-            messages.append({"role": "user", "content": PARSE_REPAIR_NOTICE})
-            if not refit():
-                break
-            continue
-        parse_failures = 0
-        # Paging re-serves stored output. It costs a turn but never a subprocess, and no byte of
-        # any observation is ever unreachable — which is the whole point of paginating instead of
-        # clipping.
-        if action["action"] == "page":
-            try:
-                want_step = int(action.get("step", 0))
-                want_page = int(action.get("page", 1))
-            except (TypeError, ValueError):
-                want_step, want_page = 0, 1
-            messages.append({"role": "assistant", "content": json.dumps(action, ensure_ascii=False)})
-            messages.append({"role": "user", "content": observations.message(want_step, want_page)})
-            if not refit():
-                break
-            continue
-        if action["action"] == "finish":
-            answer = str(action.get("answer", "")).strip()
-            break
-        started = time.perf_counter()
-        command_text = str(action.get("command", ""))
-        # An exact repeat costs a turn out of ten AND a second copy of an observation the model has
-        # already seen. Answer it from the prior step instead of re-running the command.
-        signature = call_signature({"command": command_text})
-        if signature in seen_calls:
-            observation = REPEAT_NOTICE
-            status = "repeat"
-        else:
-            try:
-                observation = run_shell(workspace, command_text, runtime.tool_timeout_seconds, shell)
-                status = "ok"
-            except (OSError, ValueError, subprocess.TimeoutExpired) as error:
-                observation = f"tool error: {type(error).__name__}: {error}"
-                status = "error"
-            seen_calls[signature] = len(calls) + 1
-        calls.append({
-            "name": "shell",
-            "raw_name": action.get("action", ""),
-            "arguments": {"command": command_text},
-            "status": status,
-            "duration_ms": round((time.perf_counter() - started) * 1000, 3),
-        })
-        messages.append({"role": "assistant", "content": json.dumps(action, ensure_ascii=False)})
-        remaining = runtime.max_turns - len(calls)
-        step = len(calls)
-        observations.put(step, observation, metadata={"action": "shell", "status": status, "source": command_text})
-        del observation
-        messages.append({"role": "user", "content": observations.message(step) + f"\n\nCommands remaining: {remaining}. Finish now if the task is answerable; do not repeat prior commands."})
-        if not refit():
-            break
-    else:
-        messages.append({"role": "user", "content": "Command budget is exhausted. Return exactly a finish JSON action now using the evidence collected. Do not request another command."})
-        if refit():
-            try:
-                response = call_model()
-                final_action = parse_action(str(response.get("message", {}).get("content", "")))
-                if final_action.get("action") != "finish":
-                    raise ValueError("forced final response was not finish")
-                answer = str(final_action.get("answer", "")).strip()
-            except (HTTPError, URLError, TimeoutError, ValueError, json.JSONDecodeError) as error:
-                diagnostic, transport_error = chat_error_details(error)
-                failure = f"agent exceeded {runtime.max_turns} turns and finalization failed: {diagnostic}"
-
-    if not answer:
-        answer = failure or "agent stopped without a final answer"
-    result = {
-        "final_answer": answer,
-        "tool_calls": calls,
-        "usage": usage,
-        "provider_metrics": metrics,
-        "model_metadata": {
-            "adapter": "bash_shell_agent_v1",
-            "sandbox": shell.kind,
-            "temperature": chat_options["temperature"],
-            "seed": chat_options["seed"],
-            "num_ctx": chat_options["num_ctx"],
-            "num_predict": chat_options["num_predict"],
-            "max_turns": runtime.max_turns,
-            "runtime_config": runtime.metadata(),
-            "context_management": context_manager.metadata(),
-            "context_compactions": context_manager.compactions,
-            "transport": "managed" if managed_endpoint else "direct_ollama",
-            "execution_preference": execution_preference,
-            "min_placement_evidence": min_placement_evidence,
-            "execution_receipts": execution_receipts,
-            **({"transport_error": transport_error} if transport_error else {}),
-            "cache_token_metrics": cache_usage.metadata(),
-            "calibration_model_identity": calibration_model_identity,
-        },
-    }
-    observations.write_result(result_path, result)
-    print(answer)
-    return 1 if failure else 0
-
-
-def main() -> int:
-    with ExitStack() as resources:
-        return _main(resources)
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())