opencode-skills-collection 4.1.14 → 4.1.16

package/bundled-skills/azure-ai-projects-py/references/datasets-indexes.md

@@ -0,0 +1,177 @@
+# Datasets and Indexes Reference
+
+## Datasets
+
+### Upload File
+
+```python
+from azure.ai.projects.models import DatasetVersion
+
+dataset = project_client.datasets.upload_file(
+    name="my-dataset",
+    version="1.0",
+    file_path="./data/training_data.csv",
+    connection_name="my-storage-connection",
+)
+print(f"Dataset uploaded: {dataset.name} v{dataset.version}")
+```
+
+### Upload Folder
+
+```python
+import re
+from azure.ai.projects.models import DatasetVersion
+
+dataset = project_client.datasets.upload_folder(
+    name="document-collection",
+    version="2.0",
+    folder="./data/documents/",
+    connection_name="my-storage-connection",
+    file_pattern=re.compile(r"\.(txt|csv|md|json)$", re.IGNORECASE),
+)
+print(f"Folder uploaded: {dataset.name} v{dataset.version}")
+```
+
+### Get Dataset
+
+```python
+dataset = project_client.datasets.get(name="my-dataset", version="1.0")
+print(f"Name: {dataset.name}")
+print(f"Version: {dataset.version}")
+```
+
+### Get Dataset Credentials
+
+```python
+credentials = project_client.datasets.get_credentials(
+    name="my-dataset",
+    version="1.0",
+)
+# Use credentials to access dataset storage
+```
+
+### List Datasets
+
+```python
+# List all datasets
+for dataset in project_client.datasets.list():
+    print(f"{dataset.name}: {dataset.version}")
+
+# List versions of a specific dataset
+for dataset in project_client.datasets.list_versions(name="my-dataset"):
+    print(f"Version: {dataset.version}")
+```
+
+### Delete Dataset
+
+```python
+project_client.datasets.delete(name="my-dataset", version="1.0")
+```
+
+## Indexes
+
+### Create or Update Index
+
+```python
+from azure.ai.projects.models import AzureAISearchIndex
+
+index = project_client.indexes.create_or_update(
+    name="my-index",
+    version="1.0",
+    index=AzureAISearchIndex(
+        connection_name="my-ai-search-connection",
+        index_name="products-index",
+    ),
+)
+print(f"Index created: {index.name} v{index.version}")
+```
+
+### Get Index
+
+```python
+index = project_client.indexes.get(name="my-index", version="1.0")
+print(f"Name: {index.name}")
+print(f"Version: {index.version}")
+```
+
+### List Indexes
+
+```python
+# List all indexes
+for index in project_client.indexes.list():
+    print(f"{index.name}: {index.version}")
+
+# List versions of a specific index
+for index in project_client.indexes.list_versions(name="my-index"):
+    print(f"Version: {index.version}")
+```
+
+### Delete Index
+
+```python
+project_client.indexes.delete(name="my-index", version="1.0")
+```
+
+## Using Indexes with Agents
+
+```python
+from azure.ai.projects.models import (
+    AzureAISearchAgentTool,
+    AzureAISearchToolResource,
+    AISearchIndexResource,
+    AzureAISearchQueryType,
+    PromptAgentDefinition,
+)
+
+# Create index reference
+index = project_client.indexes.get(name="products-index", version="1.0")
+
+# Get connection for the index
+search_connection = project_client.connections.get("my-ai-search-connection")
+
+# Create agent with index
+agent = project_client.agents.create_version(
+    agent_name="search-agent",
+    definition=PromptAgentDefinition(
+        model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+        instructions="Search the product catalog to answer questions.",
+        tools=[
+            AzureAISearchAgentTool(
+                azure_ai_search=AzureAISearchToolResource(
+                    indexes=[
+                        AISearchIndexResource(
+                            project_connection_id=search_connection.id,
+                            index_name="products-index",
+                            query_type=AzureAISearchQueryType.SEMANTIC,
+                        )
+                    ]
+                )
+            )
+        ],
+    ),
+)
+```
+
+## Version Management Pattern
+
+```python
+# Semantic versioning for datasets
+dataset_v1 = project_client.datasets.upload_file(
+    name="training-data",
+    version="1.0.0",
+    file_path="./v1/data.csv",
+    connection_name="storage",
+)
+
+# Update with new version
+dataset_v2 = project_client.datasets.upload_file(
+    name="training-data",
+    version="1.1.0",  # Minor version bump
+    file_path="./v2/data.csv",
+    connection_name="storage",
+)
+
+# List all versions
+versions = list(project_client.datasets.list_versions(name="training-data"))
+print(f"Available versions: {[v.version for v in versions]}")
+```

package/bundled-skills/azure-ai-projects-py/references/deployments.md

@@ -0,0 +1,134 @@
+# Deployments Operations Reference
+
+## Overview
+
+Deployments represent AI model deployments in your Azure AI Foundry project.
+
+## List Deployments
+
+### List All Deployments
+
+```python
+deployments = project_client.deployments.list()
+for deployment in deployments:
+    print(f"Name: {deployment.name}")
+    print(f"Model: {deployment.model_name}")
+    print(f"Publisher: {deployment.model_publisher}")
+    print("---")
+```
+
+### Filter by Publisher
+
+```python
+# List only OpenAI model deployments
+for deployment in project_client.deployments.list(model_publisher="OpenAI"):
+    print(f"{deployment.name}: {deployment.model_name}")
+```
+
+### Filter by Model Name
+
+```python
+# List deployments of a specific model
+for deployment in project_client.deployments.list(model_name="gpt-4o"):
+    print(f"{deployment.name}: {deployment.model_version}")
+```
+
+## Get Deployment
+
+```python
+from azure.ai.projects.models import ModelDeployment
+
+deployment = project_client.deployments.get("my-deployment-name")
+
+if isinstance(deployment, ModelDeployment):
+    print(f"Type: {deployment.type}")
+    print(f"Name: {deployment.name}")
+    print(f"Model Name: {deployment.model_name}")
+    print(f"Model Version: {deployment.model_version}")
+    print(f"Model Publisher: {deployment.model_publisher}")
+    print(f"Capabilities: {deployment.capabilities}")
+```
+
+## Deployment Properties
+
+```python
+deployment = project_client.deployments.get("gpt-4o-mini")
+
+# Available properties
+print(f"Name: {deployment.name}")           # Deployment name
+print(f"Model: {deployment.model_name}")    # e.g., "gpt-4o-mini"
+print(f"Version: {deployment.model_version}")  # e.g., "2024-07-18"
+print(f"Publisher: {deployment.model_publisher}")  # e.g., "OpenAI"
+print(f"Type: {deployment.type}")           # Deployment type
+print(f"Capabilities: {deployment.capabilities}")  # Model capabilities
+```
+
+## Using Deployments
+
+### Dynamic Model Selection
+
+```python
+# Find available GPT-4 deployments
+gpt4_deployments = [
+    d for d in project_client.deployments.list()
+    if "gpt-4" in d.model_name.lower()
+]
+
+if gpt4_deployments:
+    deployment_name = gpt4_deployments[0].name
+
+    agent = project_client.agents.create_agent(
+        model=deployment_name,
+        name="dynamic-agent",
+        instructions="You are helpful.",
+    )
+```
+
+### Capability Checking
+
+```python
+deployment = project_client.deployments.get("my-deployment")
+
+# Check if deployment supports certain capabilities
+if deployment.capabilities:
+    supports_vision = deployment.capabilities.get("vision", False)
+    supports_functions = deployment.capabilities.get("function_calling", False)
+
+    print(f"Vision: {supports_vision}")
+    print(f"Function Calling: {supports_functions}")
+```
+
+## Environment Variables Pattern
+
+```bash
+# Store deployment name in environment
+AZURE_AI_MODEL_DEPLOYMENT_NAME=gpt-4o-mini
+```
+
+```python
+import os
+
+# Use deployment from environment
+agent = project_client.agents.create_agent(
+    model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+    name="my-agent",
+    instructions="You are helpful.",
+)
+```
+
+## List Available Models
+
+```python
+# Print all available models grouped by publisher
+from collections import defaultdict
+
+deployments_by_publisher = defaultdict(list)
+
+for deployment in project_client.deployments.list():
+    deployments_by_publisher[deployment.model_publisher].append(deployment)
+
+for publisher, deployments in deployments_by_publisher.items():
+    print(f"\n{publisher}:")
+    for d in deployments:
+        print(f"  - {d.name} ({d.model_name} v{d.model_version})")
+```

package/bundled-skills/azure-ai-projects-py/references/evaluation.md

@@ -0,0 +1,349 @@
+# Evaluation Operations Reference
+
+Evaluate AI agents and models using Microsoft Foundry's cloud evaluation service.
+
+## Setup
+
+```python
+import os
+from azure.ai.projects import AIProjectClient
+from azure.identity import DefaultAzureCredential
+
+endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"]
+deployment = os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o-mini")
+
+with (
+    DefaultAzureCredential() as credential,
+    AIProjectClient(endpoint=endpoint, credential=credential) as project_client,
+):
+    openai_client = project_client.get_openai_client()
+    # Use openai_client.evals.*
+```
+
+## Quick Start: Run a Basic Evaluation
+
+```python
+from openai.types.evals.create_eval_jsonl_run_data_source_param import (
+    CreateEvalJSONLRunDataSourceParam,
+    SourceFileContent,
+    SourceFileContentContent,
+)
+from openai.types.eval_create_params import DataSourceConfigCustom
+
+# 1. Prepare test data
+data = [
+    {"query": "What is Azure?", "response": "Azure is Microsoft's cloud platform."},
+    {"query": "What is AI?", "response": "AI is artificial intelligence."},
+]
+
+# 2. Create data source
+data_source = CreateEvalJSONLRunDataSourceParam(
+    type="jsonl",
+    source=SourceFileContent(
+        type="file_content",
+        content=[SourceFileContentContent(item=item, sample={}) for item in data],
+    ),
+)
+
+# 3. Configure schema
+data_source_config = DataSourceConfigCustom(
+    type="custom",
+    item_schema={
+        "type": "object",
+        "properties": {
+            "query": {"type": "string"},
+            "response": {"type": "string"},
+        },
+        "required": ["query", "response"],
+    },
+    include_sample_schema=False,
+)
+
+# 4. Define evaluators
+testing_criteria = [
+    {
+        "type": "azure_ai_evaluator",
+        "name": "coherence",
+        "evaluator_name": "builtin.coherence",
+        "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"},
+        "initialization_parameters": {"deployment_name": deployment},
+    },
+    {
+        "type": "azure_ai_evaluator",
+        "name": "relevance",
+        "evaluator_name": "builtin.relevance",
+        "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"},
+        "initialization_parameters": {"deployment_name": deployment},
+    },
+]
+
+# 5. Create and run evaluation
+eval_object = openai_client.evals.create(
+    name="Quality Evaluation",
+    data_source_config=data_source_config,
+    testing_criteria=testing_criteria,
+)
+
+run = openai_client.evals.runs.create(
+    eval_id=eval_object.id,
+    name="Run 1",
+    data_source=data_source,
+)
+
+# 6. Poll for completion
+import time
+while run.status not in ["completed", "failed", "cancelled"]:
+    time.sleep(5)
+    run = openai_client.evals.runs.retrieve(eval_id=eval_object.id, run_id=run.id)
+    print(f"Status: {run.status}")
+
+# 7. Retrieve results
+output_items = list(openai_client.evals.runs.output_items.list(
+    eval_id=eval_object.id, run_id=run.id
+))
+
+for item in output_items:
+    for result in item.results:
+        print(f"{result.name}: {result.score}")
+```
+
+## Built-in Evaluators
+
+Use the `builtin.` prefix for all built-in evaluators:
+
+### Quality Evaluators
+
+| Evaluator | Data Mapping | Use Case |
+|-----------|--------------|----------|
+| `builtin.coherence` | query, response | Logical flow and consistency |
+| `builtin.relevance` | query, response | Response addresses the query |
+| `builtin.fluency` | query, response | Language quality and readability |
+| `builtin.groundedness` | query, context, response | Factual alignment with context |
+
+### Safety Evaluators
+
+| Evaluator | Data Mapping | Use Case |
+|-----------|--------------|----------|
+| `builtin.violence` | query, response | Violent content detection |
+| `builtin.sexual` | query, response | Sexual content detection |
+| `builtin.self_harm` | query, response | Self-harm content detection |
+| `builtin.hate_unfairness` | query, response | Hate/bias detection |
+
+### Agent Evaluators
+
+| Evaluator | Data Mapping | Use Case |
+|-----------|--------------|----------|
+| `builtin.intent_resolution` | query, response | Did agent understand intent? |
+| `builtin.response_completeness` | query, response | Did agent answer fully? |
+| `builtin.task_adherence` | query, response | Did agent follow instructions? |
+| `builtin.tool_call_accuracy` | query, response (JSON) | Were tool calls correct? |
+
+See [built-in-evaluators.md](built-in-evaluators.md) for complete evaluator reference.
+
+## Agent Evaluation
+
+For evaluating AI agents with tool calls, use `sample` mapping:
+
+```python
+# Data with agent outputs
+data_source = CreateEvalJSONLRunDataSourceParam(
+    type="jsonl",
+    source=SourceFileContent(
+        type="file_content",
+        content=[
+            SourceFileContentContent(
+                item={"query": "Weather in Seattle?"},
+                sample={
+                    "output_text": "It's 55°F and cloudy in Seattle.",
+                    "output_items": [
+                        {
+                            "type": "tool_call",
+                            "name": "get_weather",
+                            "arguments": {"location": "Seattle"},
+                            "result": {"temp": "55", "condition": "cloudy"},
+                        }
+                    ],
+                },
+            )
+        ],
+    ),
+)
+
+data_source_config = DataSourceConfigCustom(
+    type="custom",
+    item_schema={"type": "object", "properties": {"query": {"type": "string"}}},
+    include_sample_schema=True,  # Required for agent evaluations
+)
+
+testing_criteria = [
+    {
+        "type": "azure_ai_evaluator",
+        "name": "intent_resolution",
+        "evaluator_name": "builtin.intent_resolution",
+        "data_mapping": {
+            "query": "{{item.query}}",
+            "response": "{{sample.output_text}}",  # Use sample for agent outputs
+        },
+        "initialization_parameters": {"deployment_name": deployment},
+    },
+    {
+        "type": "azure_ai_evaluator",
+        "name": "tool_call_accuracy",
+        "evaluator_name": "builtin.tool_call_accuracy",
+        "data_mapping": {
+            "query": "{{item.query}}",
+            "response": "{{sample.output_items}}",  # JSON with tool calls
+        },
+        "initialization_parameters": {"deployment_name": deployment},
+    },
+]
+```
+
+## OpenAI Graders
+
+For simpler evaluation patterns, use OpenAI graders:
+
+```python
+testing_criteria = [
+    # Label grader (classification)
+    {
+        "type": "label_model",
+        "name": "sentiment",
+        "model": deployment,
+        "input": [{"role": "user", "content": "Classify sentiment: {{item.response}}"}],
+        "labels": ["positive", "negative", "neutral"],
+        "passing_labels": ["positive", "neutral"],
+    },
+    # String check grader
+    {
+        "type": "string_check",
+        "name": "has_disclaimer",
+        "input": "{{item.response}}",
+        "operation": "contains",
+        "reference": "Please consult",
+    },
+    # Text similarity grader
+    {
+        "type": "text_similarity",
+        "name": "matches_expected",
+        "input": "{{item.response}}",
+        "reference": "{{item.expected}}",
+        "evaluation_metric": "fuzzy_match",
+        "pass_threshold": 0.8,
+    },
+]
+```
+
+## Custom Evaluators
+
+Create custom evaluators for domain-specific needs.
+
+### Code-Based Evaluator
+
+```python
+from azure.ai.projects.models import (
+    EvaluatorVersion, EvaluatorCategory, EvaluatorType,
+    CodeBasedEvaluatorDefinition, EvaluatorMetric, EvaluatorMetricType,
+)
+
+evaluator = project_client.evaluators.create_version(
+    name="word_count",
+    evaluator_version=EvaluatorVersion(
+        evaluator_type=EvaluatorType.CUSTOM,
+        categories=[EvaluatorCategory.QUALITY],
+        display_name="Word Count",
+        definition=CodeBasedEvaluatorDefinition(
+            code_text='''
+def grade(sample, item) -> dict:
+    return {"word_count": len(item.get("response", "").split())}
+''',
+            data_schema={
+                "type": "object",
+                "properties": {"response": {"type": "string"}},
+                "required": ["response"],
+            },
+            metrics={
+                "word_count": EvaluatorMetric(type=EvaluatorMetricType.ORDINAL),
+            },
+        ),
+    ),
+)
+```
+
+### Prompt-Based Evaluator
+
+```python
+from azure.ai.projects.models import PromptBasedEvaluatorDefinition
+
+evaluator = project_client.evaluators.create_version(
+    name="helpfulness",
+    evaluator_version=EvaluatorVersion(
+        evaluator_type=EvaluatorType.CUSTOM,
+        categories=[EvaluatorCategory.QUALITY],
+        display_name="Helpfulness",
+        definition=PromptBasedEvaluatorDefinition(
+            prompt_text='''
+Rate the helpfulness of the response (1-5):
+Query: {query}
+Response: {response}
+Return JSON: {"score": <1-5>, "reason": "<explanation>"}
+''',
+            init_parameters={
+                "type": "object",
+                "properties": {"deployment_name": {"type": "string"}},
+                "required": ["deployment_name"],
+            },
+            data_schema={
+                "type": "object",
+                "properties": {"query": {"type": "string"}, "response": {"type": "string"}},
+                "required": ["query", "response"],
+            },
+            metrics={"score": EvaluatorMetric(type=EvaluatorMetricType.ORDINAL)},
+        ),
+    ),
+)
+```
+
+See [custom-evaluators.md](custom-evaluators.md) for complete custom evaluator reference.
+
+## Discover Available Evaluators
+
+```python
+# List built-in evaluators
+evaluators = project_client.evaluators.list_latest_versions(type="builtin")
+for e in evaluators:
+    print(f"builtin.{e.name}: {e.description}")
+
+# List custom evaluators
+custom = project_client.evaluators.list_latest_versions(type="custom")
+for e in custom:
+    print(f"{e.name}: {e.description}")
+```
+
+## Data Mapping Reference
+
+| Pattern | Source | Use Case |
+|---------|--------|----------|
+| `{{item.field}}` | Your JSONL data | Standard evaluation data |
+| `{{sample.output_text}}` | Agent response (text) | Agent text outputs |
+| `{{sample.output_items}}` | Agent response (JSON) | Tool calls, structured data |
+
+## CLI Tool
+
+A batch evaluation script is available at `scripts/run_batch_evaluation.py`:
+
+```bash
+python run_batch_evaluation.py --data test_data.jsonl --evaluators coherence relevance
+python run_batch_evaluation.py --data test_data.jsonl --safety
+python run_batch_evaluation.py --data test_data.jsonl --agent --evaluators intent_resolution
+```
+
+## Related Reference Files
+
+- [built-in-evaluators.md](built-in-evaluators.md): Complete built-in evaluator reference
+- [custom-evaluators.md](custom-evaluators.md): Code and prompt-based evaluator patterns
+
+## Related Documentation
+
+- [Azure AI Projects Evaluation Samples](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/ai/azure-ai-projects/samples/evaluations)
+- [Cloud Evaluation Documentation](https://learn.microsoft.com/en-us/azure/ai-foundry/how-to/develop/cloud-evaluation)