opencode-skills-collection 3.1.10 → 3.1.11

package/bundled-skills/weaviate/scripts/weaviate_conn.py

@@ -0,0 +1,231 @@
+"""
+Shared Weaviate connection utilities.
+
+This module handles:
+- Environment variable validation
+- API key to header mapping for all supported providers
+- Client connection with automatic header configuration
+
+Usage in scripts:
+    import sys
+    from pathlib import Path
+    sys.path.insert(0, str(Path(__file__).parent.parent.parent / "lib"))
+    from weaviate_conn import get_client, get_headers, validate_env
+"""
+
+import os
+import sys
+from contextlib import contextmanager
+from typing import Generator
+
+import weaviate
+from weaviate.classes.init import Auth
+from weaviate.client import WeaviateClient
+from weaviate.classes.init import AdditionalConfig, Timeout
+
+# Canonical environment variable to Weaviate header mapping
+API_KEY_MAP = {
+    "ANTHROPIC_API_KEY": "X-Anthropic-Api-Key",
+    "ANYSCALE_API_KEY": "X-Anyscale-Api-Key",
+    "AWS_ACCESS_KEY": "X-Aws-Access-Key",
+    "AWS_SECRET_KEY": "X-Aws-Secret-Key",
+    "COHERE_API_KEY": "X-Cohere-Api-Key",
+    "DATABRICKS_TOKEN": "X-Databricks-Token",
+    "FRIENDLI_TOKEN": "X-Friendli-Api-Key",
+    "VERTEX_API_KEY": "X-Goog-Vertex-Api-Key",
+    "STUDIO_API_KEY": "X-Goog-Studio-Api-Key",
+    "HUGGINGFACE_API_KEY": "X-HuggingFace-Api-Key",
+    "JINAAI_API_KEY": "X-JinaAI-Api-Key",
+    "MISTRAL_API_KEY": "X-Mistral-Api-Key",
+    "NVIDIA_API_KEY": "X-Nvidia-Api-Key",
+    "OPENAI_API_KEY": "X-OpenAI-Api-Key",
+    "AZURE_API_KEY": "X-Azure-Api-Key",
+    "VOYAGE_API_KEY": "X-Voyage-Api-Key",
+    "XAI_API_KEY": "X-Xai-Api-Key",
+}
+
+
+def _collect_headers_and_providers() -> tuple[dict[str, str], list[str]]:
+    """
+    Scan env once to build Weaviate headers and detected key names.
+
+    Returns:
+        Tuple of (headers, detected_env_var_names)
+    """
+    headers: dict[str, str] = {}
+    detected_providers: list[str] = []
+
+    for env_var, header_name in API_KEY_MAP.items():
+        value = os.environ.get(env_var, "").strip()
+        if not value:
+            continue
+
+        detected_providers.append(env_var)
+        headers[header_name] = value
+
+    return headers, detected_providers
+
+
+def validate_env(require_weaviate: bool = True) -> tuple[str, str]:
+    """
+    Validate required Weaviate environment variables.
+
+    Args:
+        require_weaviate: If True, exit with error if WEAVIATE_URL/API_KEY not set
+
+    Returns:
+        Tuple of (weaviate_url, weaviate_api_key)
+
+    Raises:
+        SystemExit: If required variables are missing
+    """
+    url = os.environ.get("WEAVIATE_URL", "").strip()
+    api_key = os.environ.get("WEAVIATE_API_KEY", "").strip()
+
+    if require_weaviate:
+        if not url:
+            print("Error: WEAVIATE_URL environment variable not set", file=sys.stderr)
+            sys.exit(1)
+        if not api_key:
+            print(
+                "Error: WEAVIATE_API_KEY environment variable not set", file=sys.stderr
+            )
+            sys.exit(1)
+
+    return url, api_key
+
+
+def get_headers() -> dict[str, str] | None:
+    """
+    Build headers dict from all available API keys in environment.
+
+    Scans environment for all known API key variables and builds
+    the appropriate headers dict for Weaviate client connection.
+
+    Returns:
+        Dict of headers if any API keys found, None otherwise
+    """
+    headers, _ = _collect_headers_and_providers()
+    return headers if headers else None
+
+
+def get_detected_providers() -> list[str]:
+    """
+    Get list of detected API key environment variable names.
+
+    Returns:
+        List of env var names (e.g., ["OPENAI_API_KEY", "COHERE_API_KEY"])
+    """
+    _, detected_providers = _collect_headers_and_providers()
+    return sorted(detected_providers)
+
+
+@contextmanager
+def get_client(
+    url: str | None = None,
+    api_key: str | None = None,
+    headers: dict[str, str] | None = None,
+    verbose: bool = True,
+) -> Generator[WeaviateClient, None, None]:
+    """
+    Context manager for Weaviate client connection.
+
+    Auto-detects credentials from environment if not provided.
+    Auto-builds headers from all available API keys if not provided.
+
+    Args:
+        url: Weaviate cluster URL (default: from WEAVIATE_URL env var)
+        api_key: Weaviate API key (default: from WEAVIATE_API_KEY env var)
+        headers: Custom headers dict (default: auto-detected from env vars)
+        verbose: Print connection status to stderr
+
+    Yields:
+        Connected WeaviateClient instance
+
+    Example:
+        with get_client() as client:
+            collections = client.collections.list_all()
+    """
+    # Get credentials from env if not provided
+    if url is None or api_key is None:
+        env_url, env_api_key = validate_env()
+        url = url or env_url
+        api_key = api_key or env_api_key
+
+    # Auto-detect headers if not provided
+    if headers is None:
+        headers, detected_providers = _collect_headers_and_providers()
+        headers = headers or None
+    else:
+        detected_providers = None
+
+    if verbose:
+        detected = sorted(detected_providers) if detected_providers is not None else []
+        if detected:
+            print(f"Detected providers: {', '.join(detected)}", file=sys.stderr)
+        print("Connecting to Weaviate...", file=sys.stderr)
+
+    client = weaviate.connect_to_weaviate_cloud(
+        cluster_url=url,
+        auth_credentials=Auth.api_key(api_key),
+        headers=headers,
+        additional_config=AdditionalConfig(
+            timeout=Timeout(init=30, query=60, insert=120)
+        ),
+    )
+
+    try:
+        if verbose:
+            print("Connected.", file=sys.stderr)
+        yield client
+    finally:
+        client.close()
+
+
+def connect_client(
+    url: str | None = None,
+    api_key: str | None = None,
+    headers: dict[str, str] | None = None,
+    verbose: bool = True,
+) -> WeaviateClient:
+    """
+    Get a Weaviate client connection (non-context manager version).
+
+    IMPORTANT: Caller is responsible for calling client.close()
+
+    Args:
+        url: Weaviate cluster URL (default: from WEAVIATE_URL env var)
+        api_key: Weaviate API key (default: from WEAVIATE_API_KEY env var)
+        headers: Custom headers dict (default: auto-detected from env vars)
+        verbose: Print connection status to stderr
+
+    Returns:
+        Connected WeaviateClient instance
+    """
+    if url is None or api_key is None:
+        env_url, env_api_key = validate_env()
+        url = url or env_url
+        api_key = api_key or env_api_key
+
+    if headers is None:
+        headers, detected_providers = _collect_headers_and_providers()
+        headers = headers or None
+    else:
+        detected_providers = None
+
+    if verbose:
+        detected = sorted(detected_providers) if detected_providers is not None else []
+        if detected:
+            print(f"Detected providers: {', '.join(detected)}", file=sys.stderr)
+        print("Connecting to Weaviate...", file=sys.stderr)
+
+    client = weaviate.connect_to_weaviate_cloud(
+        cluster_url=url,
+        auth_credentials=Auth.api_key(api_key),
+        headers=headers,
+    )
+
+    if verbose:
+        print("Connected.", file=sys.stderr)
+
+    return client

package/bundled-skills/weaviate-cookbooks/SKILL.md

@@ -0,0 +1,67 @@
+---
+name: weaviate-cookbooks
+description: "Build Weaviate AI apps from official cookbook blueprints for RAG, agentic RAG, data exploration, multimodal PDF search, async clients, and frontends."
+category: ai
+risk: safe
+source: community
+source_repo: weaviate/agent-skills
+source_type: official
+date_added: "2026-06-29"
+author: Weaviate
+tags: [weaviate, rag, agents, vector-database, ai-apps]
+tools: [python, weaviate, nextjs]
+license: "BSD-3-Clause"
+license_source: "https://github.com/weaviate/agent-skills/blob/main/LICENSE"
+---
+
+# Weaviate Cookbooks
+
+## Overview
+
+This skill provides an index of implementation guides and foundational requirements for building Weaviate-powered AI applications. Use the references to quickly scaffold full-stack applications with best practices for connection management, environment setup, and application architecture.
+
+## When to Use This Skill
+
+- Use when the user wants a Weaviate-backed RAG, agentic RAG, chatbot, data explorer, or multimodal document-search application.
+- Use when selecting between cookbook patterns before writing a full-stack Weaviate app.
+- Use when the project needs Weaviate environment, setup, async-client, or frontend guidance.
+- Use when the user asks for an official Weaviate blueprint rather than a generic vector database recipe.
+
+### Weaviate Cloud Instance
+
+If the user does not have an instance yet, direct them to the cloud console to register and create a free sandbox. Create a Weaviate instance via [Weaviate Cloud](https://console.weaviate.cloud/signin?utm_source=github&utm_campaign=agent_skills).
+
+## Before Building Any Cookbook
+
+Follow these shared guidelines before generating any cookbook app:
+
+- [Project Setup Contract](references/project_setup.md)
+- [Environment Requirements](references/environment_requirements.md)
+
+Then proceed to the specific cookbook reference below.
+
+## Cookbook Index
+
+- [Query Agent Chatbot](references/query_agent_chatbot.md): Build a full-stack chatbot using Weaviate Query Agent with streaming and chat history support.
+- [Data Explorer](references/data_explorer.md): Build a full-stack data explorer app including sorting, keyword search and tabular view of weaviate data.
+- [Multimodal RAG: Building Document Search](references/pdf_multimodal_rag.md): Build a multimodal Retrieval-Augmented Generation (RAG) system using Weaviate Embeddings (ModernVBERT/colmodernvbert) and Ollama with Qwen3-VL for generation.
+- [Basic RAG](references/basic_rag.md): Implement basic retrieval and generation with Weaviate. Useful for most forms of data retrieval from a Weaviate collection.
+- [Advanced RAG](references/advanced_rag.md): Improve on basic RAG by adding extra features such as re-ranking, query decomposition, query re-writing, LLM filter selection.
+- [Basic Agent](references/basic_agent.md): Build a tool-calling AI agent with structured outputs using DSPy. Covers AgentResponse signatures, RouterAgent, tool design, and sequential multi-step loops.
+- [Agentic RAG](references/agentic_rag.md): Build RAG-powered AI agents with Weaviate. Covers naive RAG tools, hierarchical RAG with LLM-created filters, vector DB memory, Weaviate Query Agent, and Elysia integration.
+
+## Interface (Optional)
+
+Use this when the user explicitly asks for a frontend for their Weaviate backend.
+
+- [Frontend Interface](references/frontend_interface.md): Build a Next.js frontend to interact with the Weaviate backend.
+
+## Client Usage
+
+- [Async Client](references/async_client.md): Guide for using the Weaviate Python async client in production applications (FastAPI, async frameworks). Covers connection patterns, lifecycle management, common pitfalls, and multi-cluster setups.
+
+## Limitations
+
+- Cookbook blueprints still need adaptation to the user's data model, embedding provider, auth model, deployment platform, and latency/cost targets.
+- This skill does not validate live Weaviate credentials, cloud quotas, or model availability unless the user provides and approves the relevant environment.
+- Generated apps should be reviewed for security, data privacy, prompt injection exposure, and production observability before launch.

package/bundled-skills/weaviate-cookbooks/references/advanced_rag.md

@@ -0,0 +1,274 @@
+# Advanced RAG Cookbook
+
+Build advanced RAG functionality with Weaviate.
+
+Read first:
+- Basic RAG cookbook, important to start from this base. MUST READ: [Basic RAG Cookbook](./basic_rag.md)
+
+Docs to reference if needed:
+- Search patterns and basics in Weaviate: https://docs.weaviate.io/weaviate/search/basics
+- Filters in Weaviate: https://docs.weaviate.io/weaviate/search/filters
+- Vector search: https://docs.weaviate.io/weaviate/search/similarity
+- Keyword search: https://docs.weaviate.io/weaviate/search/bm25
+- Hybrid search: https://docs.weaviate.io/weaviate/search/hybrid
+- Image search: https://docs.weaviate.io/weaviate/search/image
+
+
+## Core Rules
+
+First implement the basic strategy from [here](./basic_rag.md). Then modify according to this guide.
+
+- Use a virtual environment via `venv`
+- Use `uv` for Python project/dependency management.
+- Do not manually author `pyproject.toml` or `uv.lock`; let `uv` generate/update them.
+- Use this install set: `uv add weaviate-client python-dotenv dspy weaviate-agents`
+- Customise this cookbook to the users specification, ask them for details if not given. 
+
+Assume the user has data already to be used, do not create data unless asked to.
+
+Instead of following this cookbook, you first must ask the user if they would prefer to use the Weaviate Query Agent. If so, all steps in this guide can be implemented with the query agent which does advanced RAG out of the box.
+
+Query agent docs: https://docs.weaviate.io/agents/query/usage
+
+## Env Rules
+
+Mandatory:
+- `WEAVIATE_URL`
+- `WEAVIATE_API_KEY`
+
+External provider keys:
+- Fill only keys actually used by the target Weaviate collection setup.
+
+## Advanced RAG overview
+
+* Query re-writer: *Change user input text into a query text using an LLM*
+* Query decomposition: *Change query into multiple sub-queries each re-written with an LLM*
+* Filtering: *Use an LLM to define filters on the collection*
+* Re-ranking: *Score the final results by a more advanced model*
+* Prompt engineering: *Add chain of thought, Tree of thoughts, ReAct*
+
+## Query Re-writer
+
+```python
+class QueryRewriter(dspy.Signature):
+    """
+    Rewrite the user's query into a more relevant search term that is a more relevant search term for searching a database.
+    """
+    input_query: str = dspy.InputField(description="The original user query")
+    rewritten_query: str = dspy.OutputField(
+        description=(
+            "A single search term that is more relevant to the user's query. "
+            "Include only relevant information, it does not need to be a full sentence or question "
+        )
+    )
+```
+
+Modify the `query_transformation` function:
+
+```python
+def query_transformation(query: str) -> list[str]:
+    lm = dspy.LM(subtask_model_name)
+    answer = dspy.Predict(QueryRewriter)
+    pred = answer(input_query=query, lm=lm)
+    return [pred.rewritten_query]
+```
+
+## Query Decomposition
+
+```python
+class QueryRewriter(dspy.Signature):
+    """
+    Rewrite the user's query into a more relevant search terms that are more relevant search term for searching a database.
+    """
+    input_query: str = dspy.InputField(description="The original user query")
+    rewritten_queries: list[str] = dspy.OutputField(
+        description=(
+            "A list of search terms that are more relevant to the user's query. "
+            "Each entry should include only relevant information, it does not need to be a full sentence or question "
+            "Split independent searches into different entries "
+            "Each entry should be relevant independently that capture a different required search aspect "
+            "Do not repeat similar search terms, each one should have a unique meaning "
+            "Be sparse, do not duplicate search terms "
+        )
+    )
+
+def query_transformation(query: str) -> list[str]:
+    lm = dspy.LM(subtask_model_name)
+    answer = dspy.Predict(QueryRewriter)
+    pred = answer(input_query=query, lm=lm)
+    return pred.rewritten_queries
+```
+
+## LLM-created Filters
+
+Filters can be specified by the user (for specific use-cases, perhaps), or you can get an LLM to write the filters also. Writing filters requires knowledge of the collection schema. This can be retrieved by advanced methods or a simple version can be used.
+
+Simple version:
+
+1. First create structured responses to format filters
+
+```python
+from pydantic import BaseModel, Field
+from typing import Literal, Any
+
+class SearchFilter(BaseModel):
+    field: str = Field(description="The field to be filtered on.")
+    operator: Literal["=", "!=", ">", "<"] = Field(description="The operator to be used in conjunction with the value. These are strict operators.")
+    value: Any = Field(description="The value to be used in conjunction with the operator.")
+
+class Search(BaseModel):
+    filters: list[SearchFilter] = Field(description="The filters to be used in the vector database. This is an AND operation.")
+
+class SearchCreation(dspy.Signature):
+    """
+    Create filters and search parameters for a search query in a database.
+    """
+    query: str = dspy.InputField()
+    schema: list[dict] = dspy.InputField(desc="Schema of the collection to be searched.")
+    data_sample: list[dict] = dspy.InputField(desc="A sample of the data in the collection to be searched.")
+    search: Search = dspy.OutputField(
+        desc=(
+            "Your filters and search parameters, this should be a valid JSON object. "
+            "This should be constructed so that it matches the goal of the user prompt."
+        )
+    )
+```
+This requires `schema` and `data_sample` as an input field to the LLM call `SearchCreation`.
+
+2. Helper function to turn structured response into weaviate filter
+
+```python
+def _format_filters(search_filters: list[SearchFilter]):
+    filters = []
+    for search_filter in search_filters:
+        base_filter = Filter.by_property(search_filter.field)
+        if search_filter.operator == "=":
+            filter = base_filter.equal(search_filter.value)
+        elif search_filter.operator == "!=":
+            filter = base_filter.not_equal(search_filter.value)
+        elif search_filter.operator == ">":
+            filter = base_filter.greater_than(search_filter.value)
+        elif search_filter.operator == "<":
+            filter = base_filter.less_than(search_filter.value)
+        filters.append(filter)
+    return Filter.all_of(filters) if filters else None
+```
+
+3. Combine
+
+```python
+def create_filters(query: str):
+    
+    # import client here
+
+    collection = client.collections.use("<collection_name>")
+
+    # Get collection schema (for field names etc.). can replace this with more advanced configuration (like aggregating for unique groups)
+    config = collection.config.get()
+    schema = [{"name": p.name, "type": p.data_type[:]} for p in config.properties]
+
+    # Get a sample of the data in the collection to be searched
+    data_sample = collection.query.fetch_objects(limit=5)
+
+    # Create search parameters
+    search_parameters = dspy.ChainOfThought(SearchCreation)
+    search_parameters_output = search_parameters(query=query, schema=schema, data_sample=data_sample, lm=dspy.LM(subtask_model_name))
+    
+    return _format_filters(search_parameters_output.search.filters)
+```
+
+These filters can be passed into the `collection.query.near_text` (or equivalent search function).
+
+## Re-ranking
+
+Do not modify the user's collection unless requested to do so. Re-ranking requires configuring the collection with a re-ranker, for example:
+
+```python
+collection = client.collections.use("<collection_name>")
+collection.config.update(
+    reranker_config=Reconfigure.Reranker.cohere()  
+)
+```
+(this would require a Cohere API key).
+
+Modify the `retrieve` function
+
+```python
+from weaviate.classes.query import Rerank
+
+def retrieve(query: str, limit: int | None = None, filters = []) -> list[dict]:
+
+    # ...existing code
+    
+    response = collection.query.hybrid(
+        query=query,
+        limit=limit,
+        rerank=Rerank(
+            prop="content", # what field to re-rank on
+            query=query # what the search term for the re-ranker should be (same as original in this case)
+        ),
+        filters=filters if filters else None
+    )
+    
+    # ...existing code
+```
+
+## Prompt Engineering
+
+This step depends on the LLM framework used. You can manually ask the LLM to include reasoning before giving its final answer, adding a reasoning sub-field to be completed before giving the final answer in structured response, or specify in DSPy to use chain-of-thought.
+
+```python
+class Generator(dspy.Signature):
+    """
+    Answer the question based on the context.
+    Do not include any information from external sources, only use the information provided in the context.
+    If you cannot answer the question based on the information provided, say "I don't know".
+    """
+    context: str | list[dict] =  dspy.InputField(desc="The context to answer the question.")
+    query: str = dspy.InputField(desc="The question to answer.")
+    answer: str = dspy.OutputField(desc="The single answer to the question with no additional communication")
+```
+
+Modify the `generate` function:
+
+```python
+def generate(query: str, context: list[dict]) -> str:
+    lm = dspy.LM(generation_model_name)
+    answer = dspy.Predict(Generator)
+    pred = answer(context=context, query=query, lm=lm)
+    return pred.answer
+```
+
+Consider other prompt engineering techniques like ReAct (if necessary but likely overkill), few-shot learning (requires advanced specification), or otherwise.
+
+## Query Agent
+
+Skip this guide altogether and use the Weaviate Query Agent.
+
+```python
+from weaviate.agents.query import QueryAgent
+
+# import client here
+
+qa = QueryAgent(
+    client=client, collections=["Example_Communications_Raw"]
+)
+response = qa.search("<user query here>") # just search with no text response
+response = qa.ask("<user query here>") # search with text response accessible via response.final_answer
+```
+
+## Customisation Points
+
+**LLM framework**
+
+This guide used DSPy. Follow the guidelines in [here](./basic_rag.md), but most likely you will need an LLM framework involving structured responses.
+
+## Troubleshooting
+
+- Weaviate startup host errors: ensure `WEAVIATE_URL` is full `https://...` URL.
+- For any other issues, refer to the official library/package documentation and use web search extensively for troubleshooting.
+
+## Done Criteria
+
+- Create test scripts to check each function works independently with test data. Tear down tests after completion, or create a proper test suite with pytest (requires install)
+- User has completed specification of the app.