@bastani/atomic 0.5.11 → 0.5.12-0

package/.agents/skills/tool-design/scripts/description_generator.py

@@ -0,0 +1,528 @@
+"""
+Tool Description Engineering -- Generation and Evaluation Utilities.
+
+Use when: building, auditing, or iterating on tool descriptions for agent
+systems.  Provides templates for structured descriptions, a scoring evaluator
+that flags vague or incomplete descriptions, error-message generators that
+produce agent-recoverable responses, and a builder that assembles complete
+tool schemas.
+
+Typical workflow:
+    1. Define a tool spec with ``ToolSchemaBuilder``.
+    2. Generate a rendered description with ``generate_tool_description``.
+    3. Score the description with ``ToolDescriptionEvaluator.evaluate``.
+    4. Generate error templates with ``ErrorMessageGenerator.generate``.
+
+Example::
+
+    builder = ToolSchemaBuilder("get_customer")
+    builder.set_description("Retrieve customer record", "Full details...")
+    builder.add_parameter("customer_id", "string", "CUST-######", required=True)
+    schema = builder.build()
+
+    desc = generate_tool_description(schema)
+    scores = ToolDescriptionEvaluator().evaluate(desc, schema)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Protocol, Sequence
+import json
+import re
+
+
+__all__ = [
+    "generate_tool_description",
+    "generate_usage_context",
+    "ToolDescriptionEvaluator",
+    "ErrorMessageGenerator",
+    "ToolSchemaBuilder",
+]
+
+
+# ---------------------------------------------------------------------------
+# Protocols -- lightweight structural typing for tool specs
+# ---------------------------------------------------------------------------
+
+class ToolSpec(Protocol):
+    """Structural interface expected by generation helpers.
+
+    Use when: passing tool metadata objects that were not built with
+    ``ToolSchemaBuilder`` (e.g., third-party specs).
+    """
+
+    name: str
+    description: str
+    triggers: Sequence[str]
+    examples: Sequence[Any]
+    parameters: Sequence[Dict[str, Any]]
+    returns: Dict[str, Any]
+    errors: Sequence[Dict[str, Any]]
+
+
+@dataclass
+class _BuiltToolSpec:
+    """Concrete implementation of ToolSpec returned by ToolSchemaBuilder.build()."""
+
+    name: str
+    description: str
+    triggers: List[str]
+    examples: List[Dict[str, str]]
+    parameters: List[Dict[str, Any]]
+    returns: Dict[str, Any]
+    errors: List[Dict[str, Any]]
+
+
+# ---------------------------------------------------------------------------
+# Description Templates
+# ---------------------------------------------------------------------------
+
+TOOL_DESCRIPTION_TEMPLATE: str = """
+## {tool_name}
+
+{detailed_description}
+
+### When to Use
+{usage_context}
+
+### Parameters
+{parameters_description}
+
+### Returns
+{returns_description}
+
+### Errors
+{errors_description}
+"""
+
+PARAM_TEMPLATE: str = """
+- **{param_name}** ({param_type}{required_label})
+
+  {param_description}
+  {default_label}
+"""
+
+
+# ---------------------------------------------------------------------------
+# Generation helpers
+# ---------------------------------------------------------------------------
+
+def generate_tool_description(tool_spec: ToolSpec) -> str:
+    """Render a complete markdown tool description from *tool_spec*.
+
+    Use when: producing human-readable or agent-injectable documentation
+    from a structured spec object.
+    """
+    description: str = TOOL_DESCRIPTION_TEMPLATE.format(
+        tool_name=tool_spec.name,
+        detailed_description=tool_spec.description,
+        usage_context=generate_usage_context(tool_spec),
+        parameters_description=_generate_parameters(tool_spec.parameters),
+        returns_description=_generate_returns(tool_spec.returns),
+        errors_description=_generate_errors(tool_spec.errors),
+    )
+    return description
+
+
+def generate_usage_context(tool_spec: ToolSpec) -> str:
+    """Build the 'When to Use' section from triggers and examples.
+
+    Use when: the caller needs only the usage-context fragment rather
+    than the full rendered description.
+    """
+    contexts: list[str] = []
+
+    for trigger in tool_spec.triggers:
+        contexts.append(f"- When {trigger}")
+
+    if tool_spec.examples:
+        contexts.append("\n**Examples**:\n")
+        for example in tool_spec.examples:
+            if isinstance(example, dict):
+                contexts.append(f"- Input: {example.get('input', '')}")
+                contexts.append(f"  Output: {example.get('tool_call', '')}")
+            else:
+                contexts.append(f"- {example}")
+
+    return "\n".join(contexts)
+
+
+def _generate_parameters(parameters: Sequence[Dict[str, Any]]) -> str:
+    """Render parameter list to markdown."""
+    parts: list[str] = []
+    for p in parameters:
+        required_label = " | required" if p.get("required") else " | optional"
+        default = p.get("default")
+        default_label = f"Default: {default}" if default is not None else ""
+        parts.append(
+            f"- **{p['name']}** ({p['type']}{required_label})\n"
+            f"  {p['description']}\n"
+            f"  {default_label}".rstrip()
+        )
+    return "\n".join(parts)
+
+
+def _generate_returns(returns: Optional[Dict[str, Any]]) -> str:
+    """Render the returns section to markdown."""
+    if not returns:
+        return "No return value documented."
+    desc = returns.get("description", "")
+    rtype = returns.get("type", "object")
+    return f"{rtype} -- {desc}"
+
+
+def _generate_errors(errors: Sequence[Dict[str, Any]]) -> str:
+    """Render error definitions to markdown."""
+    if not errors:
+        return "No error conditions documented."
+    parts: list[str] = []
+    for err in errors:
+        parts.append(f"- **{err['code']}**: {err['description']} -- {err.get('resolution', '')}")
+    return "\n".join(parts)
+
+
+# ---------------------------------------------------------------------------
+# Evaluator
+# ---------------------------------------------------------------------------
+
+class ToolDescriptionEvaluator:
+    """Score a rendered description against quality criteria.
+
+    Use when: auditing existing tool descriptions for clarity,
+    completeness, accuracy, actionability, and consistency.
+    """
+
+    CRITERIA: List[str] = [
+        "clarity",
+        "completeness",
+        "accuracy",
+        "actionability",
+        "consistency",
+    ]
+
+    def evaluate(self, description: str, tool_spec: ToolSpec) -> Dict[str, float]:
+        """Return per-criterion scores (0.0 -- 1.0) for *description*.
+
+        Use when: running automated quality checks on tool descriptions
+        before deploying them into an agent system.
+        """
+        results: Dict[str, float] = {
+            "clarity": self._check_clarity(description),
+            "completeness": self._check_completeness(description, tool_spec),
+            "accuracy": self._check_accuracy(description, tool_spec),
+            "actionability": self._check_actionability(description),
+            "consistency": self._check_consistency(description, tool_spec),
+        }
+        return results
+
+    # -- private scoring helpers ------------------------------------------
+
+    def _check_clarity(self, description: str) -> float:
+        """Score description clarity (0-1).
+
+        Use when: detecting vague or ambiguous language that would
+        confuse an agent during tool selection.
+        """
+        vague_terms: list[str] = ["help", "assist", "thing", "stuff", "handle"]
+        vague_count: int = sum(1 for term in vague_terms if term in description.lower())
+
+        ambiguous: list[str] = ["it", "this", "that"]
+        ambiguous_count: int = sum(1 for term in ambiguous if f" {term} " in description)
+
+        clarity: float = 1.0 - (vague_count * 0.1) - (ambiguous_count * 0.05)
+        return max(0.0, clarity)
+
+    def _check_completeness(self, description: str, tool_spec: ToolSpec) -> float:
+        """Score presence of required sections (0-1).
+
+        Use when: verifying a description has all mandatory sections
+        before publishing.
+        """
+        required_patterns: list[tuple[str, str]] = [
+            ("description", r"## " + re.escape(str(getattr(tool_spec, "name", "")))),
+            ("parameters", r"### Parameters"),
+            ("returns", r"### Returns"),
+            ("errors", r"### Errors"),
+        ]
+        present: int = sum(
+            1 for _, pattern in required_patterns if re.search(pattern, description)
+        )
+        return present / len(required_patterns)
+
+    def _check_accuracy(self, description: str, tool_spec: ToolSpec) -> float:
+        """Score alignment between description text and spec metadata.
+
+        Use when: detecting description rot where the text no longer
+        matches the current tool spec.
+        """
+        score = 1.0
+        # Check that tool name appears in description
+        if hasattr(tool_spec, "name") and tool_spec.name not in description:
+            score -= 0.3
+        # Check parameter names appear
+        if hasattr(tool_spec, "parameters"):
+            for param in tool_spec.parameters:
+                pname = param.get("name", "") if isinstance(param, dict) else ""
+                if pname and pname not in description:
+                    score -= 0.15
+        return max(0.0, score)
+
+    def _check_actionability(self, description: str) -> float:
+        """Score whether the description contains actionable cues.
+
+        Use when: confirming agents can determine correct usage from
+        the description alone.
+        """
+        signals: list[str] = ["Use when", "Returns", "Errors", "Args", "Parameters"]
+        found: int = sum(1 for s in signals if s in description)
+        return min(1.0, found / max(1, len(signals)))
+
+    def _check_consistency(self, description: str, tool_spec: ToolSpec) -> float:
+        """Score naming and formatting consistency.
+
+        Use when: checking that parameter and section naming follows
+        conventions across the tool collection.
+        """
+        # Penalise mixed naming styles (camelCase vs snake_case)
+        camel = len(re.findall(r"[a-z][A-Z]", description))
+        snake = len(re.findall(r"[a-z]_[a-z]", description))
+        if camel > 0 and snake > 0:
+            return 0.5
+        return 1.0
+
+
+# ---------------------------------------------------------------------------
+# Error Message Generator
+# ---------------------------------------------------------------------------
+
+class ErrorMessageGenerator:
+    """Produce structured, agent-recoverable error messages.
+
+    Use when: building error responses that tell agents what went wrong,
+    why, and how to correct the call.
+    """
+
+    TEMPLATES: Dict[str, str] = {
+        "NOT_FOUND": json.dumps({
+            "error": "{error_code}",
+            "message": "{specific_message}",
+            "resolution": "{how_to_resolve}",
+            "example": "{correct_format}",
+        }, indent=2),
+
+        "INVALID_INPUT": json.dumps({
+            "error": "{error_code}",
+            "message": "Invalid {field}: {received_value}",
+            "expected_format": "{expected_format}",
+            "resolution": "Provide value matching {expected_format}",
+        }, indent=2),
+
+        "RATE_LIMITED": json.dumps({
+            "error": "{error_code}",
+            "message": "Rate limit exceeded",
+            "retry_after": "{seconds}",
+            "resolution": "Wait {seconds} seconds before retrying",
+        }, indent=2),
+    }
+
+    def generate(self, error_type: str, context: Dict[str, str]) -> str:
+        """Render an error message for *error_type* using *context* values.
+
+        Use when: a tool needs to return a structured error that an agent
+        can parse and act on.
+        """
+        template: str = self.TEMPLATES.get(error_type, self.TEMPLATES["INVALID_INPUT"])
+        return template.format(**context)
+
+
+# ---------------------------------------------------------------------------
+# Schema Builder
+# ---------------------------------------------------------------------------
+
+class ToolSchemaBuilder:
+    """Fluent builder for complete tool schemas.
+
+    Use when: defining a new tool's schema programmatically and want
+    compile-time structure rather than hand-written dictionaries.
+    """
+
+    def __init__(self, name: str) -> None:
+        self.name: str = name
+        self.description: str = ""
+        self.detailed_description: str = ""
+        self.parameters: List[Dict[str, Any]] = []
+        self.returns: Optional[Dict[str, Any]] = None
+        self.errors: List[Dict[str, str]] = []
+        self._triggers: List[str] = []
+        self._examples: List[Dict[str, str]] = []
+
+    def set_description(self, short: str, detailed: str) -> "ToolSchemaBuilder":
+        """Set short and detailed description sections.
+
+        Use when: providing both a one-line summary and a full
+        multi-paragraph description for the tool.
+        """
+        self.description = short
+        self.detailed_description = detailed
+        return self
+
+    def add_parameter(
+        self,
+        name: str,
+        param_type: str,
+        description: str,
+        required: bool = False,
+        default: Optional[Any] = None,
+        enum: Optional[List[str]] = None,
+    ) -> "ToolSchemaBuilder":
+        """Append a parameter definition.
+
+        Use when: declaring each accepted input for the tool.
+        """
+        self.parameters.append({
+            "name": name,
+            "type": param_type,
+            "description": description,
+            "required": required,
+            "default": default,
+            "enum": enum,
+        })
+        return self
+
+    def set_returns(
+        self,
+        return_type: str,
+        description: str,
+        properties: Dict[str, Any],
+    ) -> "ToolSchemaBuilder":
+        """Define the return value schema.
+
+        Use when: documenting what the tool sends back on success.
+        """
+        self.returns = {
+            "type": return_type,
+            "description": description,
+            "properties": properties,
+        }
+        return self
+
+    def add_error(
+        self,
+        code: str,
+        description: str,
+        resolution: str,
+    ) -> "ToolSchemaBuilder":
+        """Register an error condition with recovery guidance.
+
+        Use when: enumerating known failure modes so agents can
+        handle them gracefully.
+        """
+        self.errors.append({
+            "code": code,
+            "description": description,
+            "resolution": resolution,
+        })
+        return self
+
+    def build(self) -> "_BuiltToolSpec":
+        """Assemble and return the complete tool spec.
+
+        Use when: the builder is fully configured and the schema is
+        ready for registration, serialization, or passing to
+        ``generate_tool_description``.
+
+        Returns a ``_BuiltToolSpec`` object that satisfies the ``ToolSpec``
+        protocol, so it can be used directly with ``generate_tool_description``
+        and ``ToolDescriptionEvaluator``.
+        """
+        return _BuiltToolSpec(
+            name=self.name,
+            description=self.detailed_description or self.description,
+            triggers=self._triggers,
+            examples=self._examples,
+            parameters=list(self.parameters),
+            returns=self.returns or {},
+            errors=list(self.errors),
+        )
+
+    def add_trigger(self, trigger: str) -> "ToolSchemaBuilder":
+        """Add an activation trigger for the tool.
+
+        Use when: documenting when agents should select this tool.
+        """
+        self._triggers.append(trigger)
+        return self
+
+    def add_example(
+        self, input_text: str, tool_call: str
+    ) -> "ToolSchemaBuilder":
+        """Add a usage example.
+
+        Use when: providing concrete input/output pairs that help agents
+        understand expected usage.
+        """
+        self._examples.append({"input": input_text, "tool_call": tool_call})
+        return self
+
+
+# ---------------------------------------------------------------------------
+# CLI entry point
+# ---------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    # Quick demo: build a schema, render it, and evaluate it.
+    builder = ToolSchemaBuilder("get_customer")
+    builder.set_description(
+        "Retrieve customer record by ID",
+        "Fetches a customer object from the primary datastore. "
+        "Supports concise and detailed response formats.",
+    )
+    builder.add_parameter(
+        "customer_id", "string",
+        'Customer identifier in CUST-###### format (e.g., "CUST-000001")',
+        required=True,
+    )
+    builder.add_parameter(
+        "format", "string",
+        '"concise" for key fields, "detailed" for complete record',
+        required=False,
+        default="concise",
+        enum=["concise", "detailed"],
+    )
+    builder.set_returns(
+        "object",
+        "Customer object with requested fields",
+        {"id": {"type": "string"}, "name": {"type": "string"}},
+    )
+    builder.add_error("NOT_FOUND", "Customer ID not in datastore", "Verify ID format and retry")
+    builder.add_error("INVALID_FORMAT", "ID does not match CUST-######", "Use CUST-###### pattern")
+
+    spec = builder.build()
+
+    print("=== Built Spec ===")
+    print(f"Name: {spec.name}")
+    print(f"Parameters: {[p['name'] for p in spec.parameters]}")
+    print(f"Errors: {[e['code'] for e in spec.errors]}")
+
+    # Generate and evaluate description
+    description = generate_tool_description(spec)
+    print("\n=== Generated Description ===")
+    print(description)
+
+    evaluator = ToolDescriptionEvaluator()
+    scores = evaluator.evaluate(description, spec)
+    print("\n=== Evaluation Scores ===")
+    for criterion, score in scores.items():
+        print(f"  {criterion}: {score:.2f}")
+
+    # Generate an error message example
+    err_gen = ErrorMessageGenerator()
+    err_msg = err_gen.generate("NOT_FOUND", {
+        "error_code": "NOT_FOUND",
+        "specific_message": "No customer with ID CUST-999999",
+        "how_to_resolve": "Check ID and retry",
+        "correct_format": "CUST-######",
+    })
+    print("\n=== Sample Error Message ===")
+    print(err_msg)