@salesforce/afv-skills 1.65.1 → 1.67.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@salesforce/afv-skills",
-  "version": "1.65.1",
+  "version": "1.67.0",
   "description": "Salesforce's curated collection of agent skills for building applications. Optimized for Agentforce Vibes, compatible with all AI tools.",
   "license": "Apache-2.0",
   "files": [
@@ -12,9 +12,7 @@
   },
   "devDependencies": {
     "@salesforce/ui-bundle-template-app-react-sample-b2e": "^12.4.4",
-    "@salesforce/ui-bundle-template-app-react-sample-b2x": "^12.4.4",
-    "@salesforce/webapp-template-app-react-sample-b2e-experimental": "^1.117.1",
-    "@salesforce/webapp-template-app-react-sample-b2x-experimental": "^1.117.1"
+    "@salesforce/ui-bundle-template-app-react-sample-b2x": "^12.4.4"
   },
   "scripts": {
     "sync-react-b2e-sample": "node scripts/sync-react-b2e-sample.js",

package/skills/agentforce-offline-experimentation-run/SKILL.md

@@ -4,20 +4,20 @@ description: "Use this skill when users want to execute this workflow when a use
 metadata:
   relatedSkills:
   - dx-apexguru-scan
-  version: '1.0'
+  version: "1.0"
   cliTools:
   - tool:
-    - curl
-    semver: '>=7.0'
+    - "curl"
+    semver: ">=7.0"
   - tool:
-    - jq
-    semver: '>=1.6'
+    - "jq"
+    semver: ">=1.6"
   - tool:
-    - python3
-    semver: '>=3.8'
+    - "python3"
+    semver: ">=3.8"
   - tool:
-    - sf
-    semver: '>=2.0.0'
+    - "sf"
+    semver: ">=2.0.0"
 ---
 
 # Managing Agentforce Offline Experimentation
@@ -28,7 +28,7 @@ Execute this workflow when a user wants to test how a different LLM model would
 
 ## Workflow
 
-discover-data-space → verify-stdm-availability → discover-agents → resolve-agent-label → list-available-models (if user needs help choosing) → find-sessions → get-conversation-details → get-llm-step-details → obtain-sfap-token → replay-turn-prompts → [score-exact-matches, judge-function-args, judge-text-responses] → generate-csv-report → generate-html-report.
+discover-data-space → verify-stdm-availability → discover-agents → list-available-models (if user needs help choosing) → find-sessions → get-conversation-details → get-llm-step-details → obtain-sfap-token → replay-turn-prompts → [score-exact-matches, judge-function-args, judge-text-responses] → generate-csv-report → generate-html-report.
 WORKFLOW GATE: Do not proceed past Phase 0 until org alias (auto-discover from authenticated orgs), agent API name (auto-discover from STDM), and target model ID (ask user) are all confirmed.
 SFAP JWT expires after ~2 hours. For 20+ session replays, refresh the token between batches. Each SFAP replay call takes ~1-3 seconds; keep under 20 sessions for reasonable wall-clock time (~10-30 minutes).
 The scoring and judging steps (Phase 3) can run in parallel once replay completes. Exact-match scoring is instant; LLM judge calls add ~1-2 seconds per turn.
@@ -38,7 +38,6 @@ The scoring and judging steps (Phase 3) can run in parallel once replay complete
 **Preconditions:**
 
 - Data Cloud is provisioned and enabled in the org (check: `DataCloudProvisioned`)
-- Agentforce Activity data streaming is active in Data Cloud (check: `AgentforceActivityStream`)
 - Einstein LLM / GenAI Gateway is configured in the org (check: `EinsteinLlmEnabled`)
 - User has an authenticated session to the target org (check: `OrgAuthenticated`)
 
@@ -53,18 +52,17 @@ Operations grouped by purpose. Use these as the building blocks for the workflow
 | `discover-data-space` | read | implemented | `getDataSpaces` | — |
 | `verify-stdm-availability` | validate | implemented | `queryCdpAnsiSqlV2` | `discover-data-space` |
 | `discover-agents` | read | implemented | `queryCdpAnsiSqlV2` | `verify-stdm-availability` |
-| `resolve-agent-label` | read | implemented | `queryGenAiPlannerDefinition` | `discover-agents` |
 | `list-available-models` | read | implemented | `queryAiModel` | — |
 | `find-sessions` | read | implemented | `queryCdpAnsiSqlV2` | `verify-stdm-availability`, `discover-agents` |
 | `get-conversation-details` | read | proposed | `queryCdpAnsiSqlV2` | `find-sessions` |
 | `get-llm-step-details` | read | proposed | `queryCdpAnsiSqlV2` | `get-conversation-details` |
-| `obtain-sfap-token` | validate | proposed | `obtainSfapJwt` | — |
-| `replay-turn-prompts` | read | proposed | `sfapChatGenerations` | `get-llm-step-details`, `obtain-sfap-token` |
-| `score-exact-matches` | read | proposed | `computeExactMatchScores` | `replay-turn-prompts` |
-| `judge-function-args` | read | proposed | `sfapChatGenerations` | `score-exact-matches`, `obtain-sfap-token` |
-| `judge-text-responses` | read | proposed | `sfapChatGenerations` | `replay-turn-prompts`, `obtain-sfap-token` |
-| `generate-csv-report` | write | proposed | `generateCsvReport` | `score-exact-matches`, `judge-function-args`, `judge-text-responses` |
-| `generate-html-report` | write | proposed | `generateHtmlReport` | `generate-csv-report` |
+| `obtain-sfap-token` | validate | proposed | — | — |
+| `replay-turn-prompts` | read | proposed | — | `get-llm-step-details`, `obtain-sfap-token` |
+| `score-exact-matches` | read | proposed | — | `replay-turn-prompts` |
+| `judge-function-args` | read | proposed | — | `score-exact-matches`, `obtain-sfap-token` |
+| `judge-text-responses` | read | proposed | — | `replay-turn-prompts`, `obtain-sfap-token` |
+| `generate-csv-report` | write | proposed | — | `score-exact-matches`, `judge-function-args`, `judge-text-responses` |
+| `generate-html-report` | write | proposed | — | `generate-csv-report` |
 
 ### Dependency graph
 
@@ -73,7 +71,6 @@ graph TD
   discover_data_space["discover-data-space (read)"]
   verify_stdm_availability["verify-stdm-availability (validate)"]
   discover_agents["discover-agents (read)"]
-  resolve_agent_label["resolve-agent-label (read)"]
   list_available_models["list-available-models (read)"]
   find_sessions["find-sessions (read)"]
   get_conversation_details["get-conversation-details (read)"]
@@ -87,7 +84,6 @@ graph TD
   generate_html_report["generate-html-report (write)"]
   discover_data_space --> verify_stdm_availability
   verify_stdm_availability --> discover_agents
-  discover_agents --> resolve_agent_label
   verify_stdm_availability --> find_sessions
   discover_agents --> find_sessions
   find_sessions --> get_conversation_details
@@ -115,24 +111,12 @@ Discover active Data Cloud data spaces in the org
 
 #### `discover-agents`
 
-Query STDM for distinct Agentforce agent API names that have session traces
+Query STDM for distinct agent + version combinations that have session traces
 
 **Call:** `queryCdpAnsiSqlV2`   — status: `implemented`
 
 **Depends on:** `verify-stdm-availability`
 
-#### `resolve-agent-label`
-
-Enrich the selected agent API name with a human-readable MasterLabel
-
-**Call:** `queryGenAiPlannerDefinition`   — status: `implemented`
-
-**Inputs:**
-
-- `agent_api_name` *(`String`)* — Agent API name selected in discover-agents
-
-**Depends on:** `discover-agents`
-
 #### `list-available-models`
 
 List Einstein LLM models configured in the org to help the user choose a target model
@@ -141,13 +125,14 @@ List Einstein LLM models configured in the org to help the user choose a target
 
 #### `find-sessions`
 
-Find recent Agentforce sessions in STDM filtered by agent API name and date range
+Find recent Agentforce sessions in STDM filtered by agent API name, version, and date range
 
 **Call:** `queryCdpAnsiSqlV2`   — status: `implemented`
 
 **Inputs:**
 
 - `agent_api_name` *(`String`)* — Agent API name selected in discover-agents
+- `agent_version_api_name` *(`String`)* — Agent version API name selected in discover-agents
 - `start_date` *(`String`)*
 - `end_date` *(`String`)*
 - `max_sessions` *(`Integer`)*
@@ -172,9 +157,10 @@ Fetch interaction steps including LLM inputs and outputs for each interaction
 
 #### `replay-turn-prompts`
 
-Replay each turn's LLM prompt through the target model via SFAP REST API with structured tool calling
+Extract structured messages (system/assistant/user) from STDM LLM step inputs, convert to OpenAI-format messages array preserving role positions, extract tool definitions dynamically from gen_ai.request.tools, classify each step as router (topic) or action based on tool signatures, and replay via SFAP REST API POST /ai/gpt/v1/chat/generations with tool_choice=required
+
 
-**Call:** `sfapChatGenerations`   — status: `proposed`
+**Status:** `proposed`
 
 **Inputs:**
 
@@ -186,7 +172,7 @@ Replay each turn's LLM prompt through the target model via SFAP REST API with st
 
 Compute exact-match scores for topic agreement and function name accuracy
 
-**Call:** `computeExactMatchScores`   — status: `proposed`
+**Status:** `proposed`
 
 **Depends on:** `replay-turn-prompts`
 
@@ -194,7 +180,7 @@ Compute exact-match scores for topic agreement and function name accuracy
 
 LLM-as-judge scoring of function arguments when function names match (exact or lenient)
 
-**Call:** `sfapChatGenerations`   — status: `proposed`
+**Status:** `proposed`
 
 **Depends on:** `score-exact-matches`, `obtain-sfap-token`
 
@@ -202,7 +188,7 @@ LLM-as-judge scoring of function arguments when function names match (exact or l
 
 LLM-as-judge scoring of text responses for turns that end with text output
 
-**Call:** `sfapChatGenerations`   — status: `proposed`
+**Status:** `proposed`
 
 **Depends on:** `replay-turn-prompts`, `obtain-sfap-token`
 
@@ -212,7 +198,7 @@ LLM-as-judge scoring of text responses for turns that end with text output
 
 Generate CSV report with per-turn replay results and aggregate summary scores
 
-**Call:** `generateCsvReport`   — status: `proposed`
+**Status:** `proposed`
 
 **Inputs:**
 
@@ -224,7 +210,7 @@ Generate CSV report with per-turn replay results and aggregate summary scores
 
 Build a self-contained HTML analysis report with verdict badges, scorecards, and failure breakdown
 
-**Call:** `generateHtmlReport`   — status: `proposed`
+**Status:** `proposed`
 
 **Inputs:**
 
@@ -246,7 +232,7 @@ Verify STDM (Session Trace Data Model) is queryable in Data Cloud
 
 Obtain an SFAP JWT token via org session authentication and the /ide/auth endpoint
 
-**Call:** `obtainSfapJwt`   — status: `proposed`
+**Status:** `proposed`
 
 **Inputs:**
 

package/skills/agentforce-offline-experimentation-run/scripts/sfap_replay.py

@@ -22,206 +22,10 @@ STDM_DATA_FILE = os.environ.get("STDM_DATA_FILE", "/tmp/details_result.json")
 LLM_STEPS_FILES = os.environ.get("LLM_STEPS_FILES", "/tmp/llm_steps_1.json,/tmp/llm_steps_2.json").split(",")
 
 # ---------------------------------------------------------------------------
-# Dynamic tool extraction from STDM data
-# ---------------------------------------------------------------------------
-# Instead of hardcoded tools, we extract actual tool definitions from the
-# original LLM responses in the STDM data. This handles both routing tools
-# (go_to_*) and domain action tools (SearchProducts, AddToCart, etc.)
+# Tool extraction is dynamic-only: definitions come from STDM step inputs
+# (gen_ai.request.tools) at runtime. No hardcoded tool schemas.
 # ---------------------------------------------------------------------------
 
-# Well-known tool schemas observed from agent responses. Keyed by tool name.
-# These are built from observed argument patterns in STDM LLM responses.
-KNOWN_TOOL_SCHEMAS = {
-    # --- Routing tools (topic router layer) ---
-    "go_to_product_discovery": {
-        "type": "function",
-        "function": {
-            "name": "go_to_product_discovery",
-            "description": "Route to product discovery topic for searching, browsing, product questions, recommendations.",
-            "parameters": {"type": "object", "properties": {}}
-        }
-    },
-    "go_to_cart_and_checkout": {
-        "type": "function",
-        "function": {
-            "name": "go_to_cart_and_checkout",
-            "description": "Route to cart and checkout topic for adding items, viewing cart, checkout, payments.",
-            "parameters": {"type": "object", "properties": {}}
-        }
-    },
-    "go_to_store_help": {
-        "type": "function",
-        "function": {
-            "name": "go_to_store_help",
-            "description": "Route to store help topic for shipping, returns, store policies, greetings.",
-            "parameters": {"type": "object", "properties": {}}
-        }
-    },
-    "go_to_off_topic": {
-        "type": "function",
-        "function": {
-            "name": "go_to_off_topic",
-            "description": "Route to off-topic handler for queries unrelated to shopping.",
-            "parameters": {"type": "object", "properties": {}}
-        }
-    },
-    # --- Domain action tools (sub-topic agents) ---
-    "SearchProducts": {
-        "type": "function",
-        "function": {
-            "name": "SearchProducts",
-            "description": "Search the product catalog with optional filters.",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "searchQuery": {"type": "string", "description": "Search terms or keywords"},
-                    "category": {"type": "string", "description": "Product category filter"},
-                    "color": {"type": "string", "description": "Color filter"},
-                    "department": {"type": "string", "description": "Department filter (e.g. Men, Women, Unisex)"},
-                    "maxPrice": {"type": "string", "description": "Maximum price filter"},
-                    "minPrice": {"type": "string", "description": "Minimum price filter"},
-                    "sortBy": {"type": "string", "description": "Sort order (e.g. relevance, price_asc, price_desc, rating)"}
-                },
-                "required": []
-            }
-        }
-    },
-    "GetProductDetails": {
-        "type": "function",
-        "function": {
-            "name": "GetProductDetails",
-            "description": "Get detailed information about a specific product.",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "productCode": {"type": "string", "description": "The product code (e.g. SHOE-M-088)"}
-                },
-                "required": ["productCode"]
-            }
-        }
-    },
-    "GetRecommendations": {
-        "type": "function",
-        "function": {
-            "name": "GetRecommendations",
-            "description": "Get product recommendations based on a product.",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "productCode": {"type": "string", "description": "Product code to base recommendations on"}
-                },
-                "required": ["productCode"]
-            }
-        }
-    },
-    "AddToCart": {
-        "type": "function",
-        "function": {
-            "name": "AddToCart",
-            "description": "Add a product to the shopping cart.",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "productCode": {"type": "string", "description": "Product code to add"},
-                    "quantity": {"type": "string", "description": "Quantity to add"},
-                    "size": {"type": "string", "description": "Size selection"},
-                    "color": {"type": "string", "description": "Color selection"}
-                },
-                "required": ["productCode", "quantity"]
-            }
-        }
-    },
-    "ViewCart": {
-        "type": "function",
-        "function": {
-            "name": "ViewCart",
-            "description": "View the current shopping cart contents.",
-            "parameters": {"type": "object", "properties": {}}
-        }
-    },
-    "UpdateCart": {
-        "type": "function",
-        "function": {
-            "name": "UpdateCart",
-            "description": "Update cart item quantity, size, or color. Set quantity to 0 to remove.",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "productCode": {"type": "string", "description": "Product code to update (or 'all' to clear cart)"},
-                    "quantity": {"type": "string", "description": "New quantity (0 to remove)"},
-                    "size": {"type": "string", "description": "New size"},
-                    "color": {"type": "string", "description": "New color"}
-                },
-                "required": ["productCode", "quantity"]
-            }
-        }
-    },
-    "Checkout": {
-        "type": "function",
-        "function": {
-            "name": "Checkout",
-            "description": "Place an order with the current cart.",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "shippingMethod": {"type": "string", "description": "Shipping method (Standard or Express)"},
-                    "shopperName": {"type": "string", "description": "Name for the order"}
-                },
-                "required": ["shippingMethod"]
-            }
-        }
-    },
-    # --- Generic CRM tools (fallback for non-shopper agents) ---
-    "QueryRecords": {
-        "type": "function",
-        "function": {
-            "name": "QueryRecords",
-            "description": "Query records using natural language.",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "query": {"type": "string", "description": "Natural language query"}
-                },
-                "required": ["query"]
-            }
-        }
-    },
-    "IdentifyRecordByName": {
-        "type": "function",
-        "function": {
-            "name": "IdentifyRecordByName",
-            "description": "Identify a record by its name to get the record ID.",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "name": {"type": "string", "description": "The name of the record"},
-                    "objectType": {"type": "string", "description": "The Salesforce object type"}
-                },
-                "required": ["name"]
-            }
-        }
-    },
-    "GetRecordDetails": {
-        "type": "function",
-        "function": {
-            "name": "GetRecordDetails",
-            "description": "Get detailed information about a specific record by its ID.",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "recordId": {"type": "string", "description": "The Salesforce record ID"}
-                },
-                "required": ["recordId"]
-            }
-        }
-    },
-}
-
-ROUTING_TOOLS = ['go_to_product_discovery', 'go_to_cart_and_checkout', 'go_to_store_help', 'go_to_off_topic']
-ACTION_TOOLS_SHOPPER = ['SearchProducts', 'GetProductDetails', 'GetRecommendations',
-                        'AddToCart', 'ViewCart', 'UpdateCart', 'Checkout']
-GENERIC_CRM_TOOLS = ['QueryRecords', 'IdentifyRecordByName', 'GetRecordDetails']
-
 
 def _is_routing_tools(tools):
     """True when ALL tools in the list are routing functions (go_to_* / __end_session_action__).
@@ -270,27 +74,6 @@ def sanitize_tools_for_sfap(tools):
     return sanitized
 
 
-def build_tools_for_step(original_tool_name, prompt_text):
-    """Select the right tool set based on the original tool call and prompt content."""
-    # If original called a routing tool, offer routing tools
-    if original_tool_name and original_tool_name.startswith('go_to_'):
-        return [KNOWN_TOOL_SCHEMAS[t] for t in ROUTING_TOOLS if t in KNOWN_TOOL_SCHEMAS]
-
-    # If original called a shopper action tool, offer shopper tools
-    if original_tool_name and original_tool_name in ACTION_TOOLS_SHOPPER:
-        return [KNOWN_TOOL_SCHEMAS[t] for t in ACTION_TOOLS_SHOPPER if t in KNOWN_TOOL_SCHEMAS]
-
-    # Detect from prompt content
-    if prompt_text:
-        if 'go_to_product_discovery' in prompt_text or 'go_to_cart' in prompt_text:
-            return [KNOWN_TOOL_SCHEMAS[t] for t in ROUTING_TOOLS if t in KNOWN_TOOL_SCHEMAS]
-        if 'SearchProducts' in prompt_text or 'AddToCart' in prompt_text:
-            return [KNOWN_TOOL_SCHEMAS[t] for t in ACTION_TOOLS_SHOPPER if t in KNOWN_TOOL_SCHEMAS]
-
-    # Fallback to generic CRM tools
-    return [KNOWN_TOOL_SCHEMAS[t] for t in GENERIC_CRM_TOOLS if t in KNOWN_TOOL_SCHEMAS]
-
-
 def extract_tools_from_llm_responses(llm_step_map):
     """Scan all LLM responses to discover the actual tool names used by the agent."""
     seen = set()
@@ -309,10 +92,6 @@ def extract_tools_from_llm_responses(llm_step_map):
     return seen
 
 
-# Legacy alias for backward compatibility
-AGENT_TOOLS = [KNOWN_TOOL_SCHEMAS[t] for t in GENERIC_CRM_TOOLS if t in KNOWN_TOOL_SCHEMAS]
-
-
 def read_token():
     if SFAP_TOKEN_FILE and os.path.exists(SFAP_TOKEN_FILE):
         with open(SFAP_TOKEN_FILE) as f:
@@ -415,24 +194,6 @@ def convert_stdm_messages_for_sfap(stdm_messages):
     return out
 
 
-def prompt_has_tools(prompt_text):
-    """Check if this prompt includes tool/function calling instructions."""
-    indicators = [
-        'data retrieval functions',
-        'QueryRecords',
-        'IdentifyRecordByName',
-        'GetRecordDetails',
-        'TOOL USAGE PROTOCOL',
-        'go_to_product_discovery',
-        'go_to_cart_and_checkout',
-        'go_to_store_help',
-        'SearchProducts',
-        'AddToCart',
-        'Checkout',
-    ]
-    return any(ind in prompt_text for ind in indicators)
-
-
 def call_sfap(messages, model, tools=None, max_tokens=2048):
     """Call SFAP chat generations endpoint."""
     token = read_token()
@@ -880,13 +641,10 @@ def main():
             print(f"    SKIP: Could not parse prompt into messages")
             continue
 
-        # Use tool definitions from STDM step input if available, else fall back
+        # Use tool definitions from STDM step input (dynamic-only, no hardcoded fallback)
         tools = target.get('step_tools')
         if tools:
             tools = sanitize_tools_for_sfap(tools)
-        else:
-            has_tools = prompt_has_tools(target['prompt'])
-            tools = build_tools_for_step(target.get('original_tool_name'), target['prompt']) if has_tools else None
 
         print(f"    Messages: {len(messages)}, Tools: {'yes' if tools else 'no'}")