agentic-qe 3.7.8 → 3.7.9

package/assets/skills/wms-testing-patterns/evals/wms-testing-patterns.yaml

@@ -0,0 +1,165 @@
+# =============================================================================
+# AQE WMS Testing Patterns Skill Evaluation Test Suite v1.0.0
+# Per ADR-056 - Trust Tier 3 Validation
+# =============================================================================
+
+skill: wms-testing-patterns
+version: 1.0.0
+description: >
+  Evaluation suite for WMS testing patterns skill. Tests inventory operations,
+  pick/pack/ship workflows, EDI compliance, RF scanning, and WMS-ERP integration.
+
+models_to_test:
+  - claude-sonnet-4
+
+mcp_integration:
+  enabled: true
+  namespace: skill-validation
+  query_patterns: true
+  track_outcomes: true
+
+learning:
+  store_success_patterns: true
+  store_failure_patterns: true
+  pattern_ttl_days: 90
+  min_confidence_to_store: 0.7
+
+result_format:
+  json_output: true
+  markdown_report: true
+  include_timing: true
+
+setup:
+  required_tools:
+    - jq
+
+test_cases:
+  - id: tc001_inventory_accuracy
+    description: "Skill generates inventory accuracy test scenarios"
+    category: inventory
+    priority: critical
+    input:
+      prompt: |
+        Generate test scenarios for warehouse inventory accuracy:
+        - Cycle count verification
+        - Receipt putaway validation
+        - Adjustment reconciliation
+        Target: 99.5% inventory accuracy
+    expected_output:
+      must_contain:
+        - "inventory"
+        - "accuracy"
+        - "cycle count"
+      must_not_contain:
+        - "TODO"
+        - "error"
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.8
+
+  - id: tc002_pick_pack_ship
+    description: "Skill covers pick/pack/ship workflow testing"
+    category: workflow
+    priority: critical
+    input:
+      prompt: |
+        Create end-to-end test plan for pick/pack/ship workflow:
+        - Wave release triggers picks
+        - Pick confirmation updates inventory
+        - Pack station validates items
+        - Shipment confirm generates ASN (EDI 856)
+    expected_output:
+      must_contain:
+        - "pick"
+        - "pack"
+        - "ship"
+        - "wave"
+      must_match_regex:
+        - "(?i)(workflow|end.to.end|sequence)"
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.8
+
+  - id: tc003_edi_compliance
+    description: "Skill validates EDI message testing"
+    category: edi
+    priority: high
+    input:
+      prompt: |
+        Test EDI X12 message processing for WMS:
+        - 940 Warehouse Shipping Order
+        - 945 Warehouse Shipping Advice
+        - 856 ASN (Advance Ship Notice)
+        - 943/944 Warehouse Stock Transfer
+    expected_output:
+      must_contain:
+        - "EDI"
+        - "940"
+        - "945"
+        - "856"
+      must_match_regex:
+        - "(?i)(x12|edi|transaction.set)"
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.7
+
+  - id: tc004_rf_scanning
+    description: "Skill tests RF/barcode scanning flows"
+    category: rf-scanning
+    priority: high
+    input:
+      prompt: |
+        Test RF scanning workflows:
+        - Scan location barcode -> validate location
+        - Scan item barcode -> verify item matches pick
+        - Scan LPN -> confirm container
+        - Handle scan errors and re-scans
+    expected_output:
+      must_contain:
+        - "scan"
+        - "barcode"
+        - "location"
+      must_match_regex:
+        - "(?i)(rf|scan|barcode|validate)"
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.7
+
+  - id: tc005_wms_erp_integration
+    description: "Skill covers WMS-ERP integration testing"
+    category: integration
+    priority: critical
+    input:
+      prompt: |
+        Test WMS-ERP integration points:
+        - Inventory sync (real-time vs batch)
+        - Order status updates
+        - Goods receipt posting
+        - Master data synchronization
+    expected_output:
+      must_contain:
+        - "integration"
+        - "ERP"
+        - "inventory"
+        - "sync"
+      must_match_regex:
+        - "(?i)(integration|sync|interface|api)"
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.8
+
+success_criteria:
+  pass_rate: 0.80
+  critical_pass_rate: 1.0
+  avg_reasoning_quality: 0.7
+  max_execution_time_ms: 300000
+
+metadata:
+  author: "@agentic-qe"
+  created: "2026-02-11"
+  last_updated: "2026-02-11"
+  coverage_target: >
+    WMS testing patterns including inventory accuracy, pick/pack/ship workflows,
+    EDI X12 compliance, RF/barcode scanning, and WMS-ERP integration.
+  adr_reference: "ADR-056"
+  trust_tier: 3

package/assets/skills/wms-testing-patterns/schemas/output.json

@@ -0,0 +1,150 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agentic-qe.dev/schemas/skills/wms-testing-patterns/output.json",
+  "title": "WMS Testing Patterns Skill Output Schema",
+  "description": "Schema for WMS testing patterns skill output. Covers inventory operations, pick/pack/ship workflows, EDI compliance, RF scanning, and WMS-ERP integration testing.",
+  "allOf": [
+    {
+      "$comment": "Base schema structure from skill-output.template.json",
+      "type": "object",
+      "required": ["skillName", "version", "timestamp", "status", "trustTier", "output"],
+      "properties": {
+        "skillName": {
+          "type": "string",
+          "const": "wms-testing-patterns"
+        },
+        "version": {
+          "type": "string",
+          "pattern": "^\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9]+)?$"
+        },
+        "timestamp": {
+          "type": "string",
+          "format": "date-time"
+        },
+        "status": {
+          "type": "string",
+          "enum": ["success", "partial", "failed", "skipped"]
+        },
+        "trustTier": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 3
+        }
+      }
+    }
+  ],
+  "properties": {
+    "output": {
+      "type": "object",
+      "required": ["summary", "testingDomain", "workflowCoverage"],
+      "properties": {
+        "summary": {
+          "type": "string",
+          "minLength": 10,
+          "maxLength": 2000,
+          "description": "Human-readable summary of WMS testing analysis"
+        },
+        "testingDomain": {
+          "type": "string",
+          "enum": ["inventory", "pick-pack-ship", "edi", "rf-scanning", "allocation", "integration", "mixed"],
+          "description": "Primary WMS testing domain"
+        },
+        "wmsPlatform": {
+          "type": "string",
+          "enum": ["blue-yonder", "manhattan", "sap-ewm", "oracle-wms", "generic", "custom"],
+          "description": "Target WMS platform"
+        },
+        "workflowCoverage": {
+          "type": "object",
+          "required": ["totalWorkflows", "coveredWorkflows"],
+          "properties": {
+            "totalWorkflows": { "type": "integer", "minimum": 0 },
+            "coveredWorkflows": { "type": "integer", "minimum": 0 },
+            "coveragePercentage": { "type": "number", "minimum": 0, "maximum": 100 }
+          }
+        },
+        "testRecommendations": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["id", "testType", "description"],
+            "properties": {
+              "id": { "type": "string", "pattern": "^WMS-\\d{3,6}$" },
+              "testType": { "type": "string", "enum": ["inventory", "workflow", "edi", "rf", "integration", "performance"] },
+              "description": { "type": "string", "maxLength": 500 },
+              "priority": { "type": "string", "enum": ["critical", "high", "medium", "low"] }
+            }
+          },
+          "maxItems": 50
+        },
+        "findings": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["id", "title", "severity", "category"],
+            "properties": {
+              "id": { "type": "string", "pattern": "^WMS-\\d{3,6}$" },
+              "title": { "type": "string", "minLength": 5, "maxLength": 200 },
+              "description": { "type": "string", "maxLength": 2000 },
+              "severity": { "type": "string", "enum": ["critical", "high", "medium", "low", "info"] },
+              "category": { "type": "string", "enum": ["inventory-accuracy", "workflow-gap", "edi-compliance", "rf-scanning", "integration-issue", "performance-risk"] },
+              "remediation": { "type": "string" }
+            }
+          },
+          "maxItems": 200
+        },
+        "recommendations": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["id", "title", "priority"],
+            "properties": {
+              "id": { "type": "string", "pattern": "^REC-\\d{3,6}$" },
+              "title": { "type": "string", "maxLength": 200 },
+              "description": { "type": "string", "maxLength": 2000 },
+              "priority": { "type": "string", "enum": ["critical", "high", "medium", "low"] },
+              "effort": { "type": "string", "enum": ["trivial", "low", "medium", "high", "major"] }
+            }
+          },
+          "maxItems": 50
+        },
+        "metrics": {
+          "type": "object",
+          "properties": {
+            "inventoryAccuracyTests": { "type": "integer", "minimum": 0 },
+            "workflowTests": { "type": "integer", "minimum": 0 },
+            "ediComplianceTests": { "type": "integer", "minimum": 0 },
+            "rfScanningTests": { "type": "integer", "minimum": 0 },
+            "integrationTests": { "type": "integer", "minimum": 0 },
+            "coveragePercentage": { "type": "number", "minimum": 0, "maximum": 100 }
+          }
+        }
+      }
+    },
+    "metadata": {
+      "type": "object",
+      "properties": {
+        "executionTimeMs": { "type": "integer", "minimum": 0 },
+        "toolsUsed": { "type": "array", "items": { "type": "string" } },
+        "agentId": { "type": "string", "pattern": "^qe-[a-z][a-z0-9-]*$" },
+        "wmsPlatform": { "type": "string" },
+        "ediStandards": { "type": "array", "items": { "type": "string" } }
+      }
+    },
+    "validation": {
+      "type": "object",
+      "properties": {
+        "schemaValid": { "type": "boolean" },
+        "contentValid": { "type": "boolean" },
+        "confidence": { "type": "number", "minimum": 0, "maximum": 1 }
+      }
+    },
+    "learning": {
+      "type": "object",
+      "properties": {
+        "patternsDetected": { "type": "array", "items": { "type": "string" } },
+        "reward": { "type": "number", "minimum": 0, "maximum": 1 }
+      }
+    }
+  }
+}

package/assets/skills/wms-testing-patterns/scripts/validate-config.json

@@ -0,0 +1,51 @@
+{
+  "skillName": "wms-testing-patterns",
+  "skillVersion": "1.0.0",
+  "requiredTools": [
+    "jq"
+  ],
+  "optionalTools": [
+    "node",
+    "python3"
+  ],
+  "schemaPath": "schemas/output.json",
+  "requiredFields": [
+    "skillName",
+    "status",
+    "output",
+    "output.testingDomain",
+    "output.workflowCoverage"
+  ],
+  "requiredNonEmptyFields": [
+    "output.summary",
+    "output.testingDomain"
+  ],
+  "mustContainTerms": [
+    "warehouse",
+    "inventory"
+  ],
+  "mustNotContainTerms": [
+    "TODO",
+    "FIXME",
+    "placeholder",
+    "example.com",
+    "lorem ipsum"
+  ],
+  "enumValidations": {
+    ".status": [
+      "success",
+      "partial",
+      "failed",
+      "skipped"
+    ],
+    ".output.testingDomain": [
+      "inventory",
+      "pick-pack-ship",
+      "edi",
+      "rf-scanning",
+      "allocation",
+      "integration",
+      "mixed"
+    ]
+  }
+}

package/dist/benchmarks/performance-benchmarks.js

@@ -160,7 +160,7 @@ export async function benchmarkHNSWIndex() {
     try {
         // Dynamic import to handle optional dependency
         const { HNSWIndex, DEFAULT_HNSW_CONFIG } = await import('../domains/coverage-analysis/services/hnsw-index.js');
-        const dimensions = 768;
+        const dimensions = 384;
         const config = {
             ...DEFAULT_HNSW_CONFIG,
             dimensions,