@fabricorg/databricks-testkit 0.7.2 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -2495,6 +2495,7 @@ function createLakeflowJobsTargetPack() {
2495
2495
  }
2496
2496
 
2497
2497
  // src/advanced-workload-checks.ts
2498
+ init_workspace_client();
2498
2499
  init_workspace_context();
2499
2500
  var DatabricksAdvancedWorkloadsAdapter = class {
2500
2501
  constructor(client) {
@@ -2568,6 +2569,15 @@ var DatabricksAdvancedWorkloadsAdapter = class {
2568
2569
  warehouse(id) {
2569
2570
  return this.client.get(`/api/2.0/sql/warehouses/${segment(id)}`);
2570
2571
  }
2572
+ app(name) {
2573
+ return this.client.get(`/api/2.0/apps/${segment(name)}`);
2574
+ }
2575
+ agentService(fullName) {
2576
+ return this.client.get(`/api/2.1/unity-catalog/agent-services/${segment(fullName)}`);
2577
+ }
2578
+ agentServicePermissions(fullName) {
2579
+ return this.client.get(`/api/2.1/unity-catalog/permissions/AGENT_SERVICE/${segment(fullName)}`);
2580
+ }
2571
2581
  };
2572
2582
  function segment(value) {
2573
2583
  return encodeURIComponent(value);
@@ -2614,6 +2624,27 @@ async function booleanSql(label, sql, query) {
2614
2624
  throw new Error(`${label} assertion did not return a truthy first column`);
2615
2625
  }
2616
2626
  }
2627
+ async function runCertificationJob(label, jobIdValue, env, client, options = {}) {
2628
+ const jobId = Number(jobIdValue);
2629
+ if (!Number.isSafeInteger(jobId) || jobId <= 0) {
2630
+ throw new Error(`${label} job id must be a positive integer`);
2631
+ }
2632
+ const submitted = await client.post("/api/2.1/jobs/run-now", {
2633
+ job_id: jobId
2634
+ });
2635
+ if (!Number.isSafeInteger(submitted.run_id)) {
2636
+ throw new Error(`${label} certification job returned no run_id`);
2637
+ }
2638
+ const run = await waitForDatabricksRun(client, submitted.run_id, {
2639
+ timeoutMs: options.timeoutMs ?? Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
2640
+ pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
2641
+ });
2642
+ const result = run.state?.result_state;
2643
+ if (result !== "SUCCESS") {
2644
+ throw new Error(`${label} certification job ${jobId} ended in ${result ?? "UNKNOWN"}`);
2645
+ }
2646
+ return { jobId, runId: submitted.run_id };
2647
+ }
2617
2648
  function advancedStreamingCdcCheck() {
2618
2649
  const names = [
2619
2650
  "DBX_TEST_STREAMING_PIPELINE_ID",
@@ -2993,6 +3024,201 @@ function securityPostureCostCheck() {
2993
3024
  }
2994
3025
  };
2995
3026
  }
3027
+ function mlflow3GenAiQualityCheck() {
3028
+ const names = [
3029
+ "DBX_TEST_MLFLOW3_JOB_ID",
3030
+ "DBX_TEST_MLFLOW3_TRACE_ASSERTION_SQL",
3031
+ "DBX_TEST_MLFLOW3_EVAL_ASSERTION_SQL",
3032
+ "DBX_TEST_MLFLOW3_SCORER_ASSERTION_SQL",
3033
+ "DBX_TEST_MLFLOW3_MONITOR_ASSERTION_SQL",
3034
+ "DBX_TEST_AI_SEARCH_EVAL_ASSERTION_SQL"
3035
+ ];
3036
+ return {
3037
+ id: "mlflow3-genai-quality",
3038
+ description: "MLflow 3 tracing, offline agent evaluation and production scorer monitoring produce durable evidence",
3039
+ configured: (env) => required2(env, names).length === 0,
3040
+ async run({ env, client, warehouse: warehouse2 }) {
3041
+ requireEnv(env, names);
3042
+ const job = await runCertificationJob(
3043
+ "MLflow 3 GenAI",
3044
+ env.DBX_TEST_MLFLOW3_JOB_ID,
3045
+ env,
3046
+ client,
3047
+ { timeoutMs: Number(env.DBX_TEST_MLFLOW3_TIMEOUT_MS ?? 72e5) }
3048
+ );
3049
+ await booleanSql("MLflow 3 trace", env.DBX_TEST_MLFLOW3_TRACE_ASSERTION_SQL, warehouse2.query);
3050
+ await booleanSql(
3051
+ "MLflow 3 offline evaluation",
3052
+ env.DBX_TEST_MLFLOW3_EVAL_ASSERTION_SQL,
3053
+ warehouse2.query
3054
+ );
3055
+ await booleanSql(
3056
+ "MLflow 3 production scorer lifecycle",
3057
+ env.DBX_TEST_MLFLOW3_SCORER_ASSERTION_SQL,
3058
+ warehouse2.query
3059
+ );
3060
+ await booleanSql(
3061
+ "MLflow 3 production monitoring",
3062
+ env.DBX_TEST_MLFLOW3_MONITOR_ASSERTION_SQL,
3063
+ warehouse2.query
3064
+ );
3065
+ await booleanSql(
3066
+ "AI Search retrieval quality",
3067
+ env.DBX_TEST_AI_SEARCH_EVAL_ASSERTION_SQL,
3068
+ warehouse2.query
3069
+ );
3070
+ return {
3071
+ ...job,
3072
+ traces: true,
3073
+ offlineEvaluation: true,
3074
+ scorerLifecycle: true,
3075
+ monitor: true,
3076
+ aiSearchEvaluation: true
3077
+ };
3078
+ }
3079
+ };
3080
+ }
3081
+ function agentEvaluationPipelineCheck() {
3082
+ const names = ["DBX_TEST_AGENT_EVAL_E2E_JOB_ID", "DBX_TEST_AGENT_EVAL_ASSERTION_SQL"];
3083
+ return {
3084
+ id: "agent-evaluation-pipeline",
3085
+ description: "Experiments API, outbox, Temporal worker, Model Serving judge, SQL score sink and Quality completion execute end to end",
3086
+ configured: (env) => required2(env, names).length === 0,
3087
+ async run({ env, client, warehouse: warehouse2 }) {
3088
+ requireEnv(env, names);
3089
+ const job = await runCertificationJob(
3090
+ "Agent evaluation end-to-end",
3091
+ env.DBX_TEST_AGENT_EVAL_E2E_JOB_ID,
3092
+ env,
3093
+ client
3094
+ );
3095
+ await booleanSql(
3096
+ "Agent evaluation terminal Quality evidence",
3097
+ env.DBX_TEST_AGENT_EVAL_ASSERTION_SQL,
3098
+ warehouse2.query
3099
+ );
3100
+ return { ...job, api: true, outbox: true, temporal: true, judged: true, persisted: true };
3101
+ }
3102
+ };
3103
+ }
3104
+ function managedMcpAgentsCheck() {
3105
+ const names = [
3106
+ "DBX_TEST_MCP_JOB_ID",
3107
+ "DBX_TEST_MCP_MANAGED_ASSERTION_SQL",
3108
+ "DBX_TEST_MCP_SERVICE_ASSERTION_SQL",
3109
+ "DBX_TEST_MCP_DENY_ASSERTION_SQL"
3110
+ ];
3111
+ return {
3112
+ id: "managed-mcp-agents",
3113
+ description: "Managed MCP and Unity Catalog MCP Services support list, call and deny behavior",
3114
+ configured: (env) => required2(env, names).length === 0,
3115
+ async run({ env, client, warehouse: warehouse2 }) {
3116
+ requireEnv(env, names);
3117
+ const job = await runCertificationJob("Databricks MCP", env.DBX_TEST_MCP_JOB_ID, env, client);
3118
+ await booleanSql(
3119
+ "Managed MCP list/call",
3120
+ env.DBX_TEST_MCP_MANAGED_ASSERTION_SQL,
3121
+ warehouse2.query
3122
+ );
3123
+ await booleanSql(
3124
+ "Unity Catalog MCP Service list/call",
3125
+ env.DBX_TEST_MCP_SERVICE_ASSERTION_SQL,
3126
+ warehouse2.query
3127
+ );
3128
+ await booleanSql(
3129
+ "MCP excluded-tool denial",
3130
+ env.DBX_TEST_MCP_DENY_ASSERTION_SQL,
3131
+ warehouse2.query
3132
+ );
3133
+ return { ...job, managedServer: true, mcpService: true, denyControl: true };
3134
+ }
3135
+ };
3136
+ }
3137
+ function dataQualityAppTelemetryCheck() {
3138
+ const names = [
3139
+ "DBX_TEST_TELEMETRY_APP_NAME",
3140
+ "DBX_TEST_DQM_ASSERTION_SQL",
3141
+ "DBX_TEST_APP_LOGS_ASSERTION_SQL",
3142
+ "DBX_TEST_APP_SPANS_ASSERTION_SQL",
3143
+ "DBX_TEST_APP_METRICS_ASSERTION_SQL"
3144
+ ];
3145
+ return {
3146
+ id: "data-quality-app-telemetry",
3147
+ description: "Unity Catalog data quality monitoring and Databricks Apps logs, spans and metrics are current",
3148
+ configured: (env) => required2(env, names).length === 0,
3149
+ async run({ env, client, warehouse: warehouse2 }) {
3150
+ requireEnv(env, names);
3151
+ const appName = env.DBX_TEST_TELEMETRY_APP_NAME;
3152
+ const app = await adapter(client).app(appName);
3153
+ const destinations = Array.isArray(app.telemetry_export_destinations) ? app.telemetry_export_destinations : [];
3154
+ const unityCatalog = destinations.find(
3155
+ (destination) => isObject(destination) && isObject(destination.unity_catalog)
3156
+ );
3157
+ if (!isObject(unityCatalog) || !isObject(unityCatalog.unity_catalog)) {
3158
+ throw new Error(`Databricks App ${appName} has no Unity Catalog telemetry destination`);
3159
+ }
3160
+ const tables = unityCatalog.unity_catalog;
3161
+ for (const field of ["logs_table", "traces_table", "metrics_table"]) {
3162
+ if (typeof tables[field] !== "string" || !tables[field]) {
3163
+ throw new Error(`Databricks App ${appName} telemetry destination has no ${field}`);
3164
+ }
3165
+ }
3166
+ await booleanSql("Data Quality Monitoring", env.DBX_TEST_DQM_ASSERTION_SQL, warehouse2.query);
3167
+ await booleanSql("App logs", env.DBX_TEST_APP_LOGS_ASSERTION_SQL, warehouse2.query);
3168
+ await booleanSql("App spans", env.DBX_TEST_APP_SPANS_ASSERTION_SQL, warehouse2.query);
3169
+ await booleanSql("App metrics", env.DBX_TEST_APP_METRICS_ASSERTION_SQL, warehouse2.query);
3170
+ return { app: appName, unityCatalogTelemetry: true, dataQualityMonitoring: true };
3171
+ }
3172
+ };
3173
+ }
3174
+ function agentServicesEnrollmentCheck() {
3175
+ const names = [
3176
+ "DBX_TEST_AGENT_SERVICE_FULL_NAME",
3177
+ "DBX_TEST_AGENT_SERVICE_CONNECTION",
3178
+ "DBX_TEST_AGENT_SERVICE_EXECUTE_PRINCIPAL"
3179
+ ];
3180
+ return {
3181
+ id: "agent-services-enrollment",
3182
+ description: "A customer-owned Harness agent is registered as an external Unity Catalog Agent Service with governed access",
3183
+ configured: (env) => required2(env, names).length === 0,
3184
+ async run({ env, client }) {
3185
+ requireEnv(env, names);
3186
+ const fullName = env.DBX_TEST_AGENT_SERVICE_FULL_NAME;
3187
+ const api = adapter(client);
3188
+ const service = await api.agentService(fullName);
3189
+ if (service.agent_service_type !== "AGENT_SERVICE_TYPE_EXTERNAL") {
3190
+ throw new Error(`${fullName} is not an external Agent Service`);
3191
+ }
3192
+ const expectedConnection = env.DBX_TEST_AGENT_SERVICE_CONNECTION;
3193
+ const actualConnection = service.config?.connection?.name ?? "";
3194
+ if (actualConnection !== expectedConnection && actualConnection.split("/").at(-1) !== expectedConnection) {
3195
+ throw new Error(`${fullName} uses unexpected connection ${actualConnection || "UNKNOWN"}`);
3196
+ }
3197
+ if (!service.config?.base_path?.startsWith("/")) {
3198
+ throw new Error(`${fullName} has no absolute external base path`);
3199
+ }
3200
+ const permissions = await api.agentServicePermissions(fullName);
3201
+ const assignments = Array.isArray(permissions.privilege_assignments) ? permissions.privilege_assignments : [];
3202
+ const expectedPrincipal = env.DBX_TEST_AGENT_SERVICE_EXECUTE_PRINCIPAL;
3203
+ const assignment = assignments.find(
3204
+ (entry) => isObject(entry) && entry.principal === expectedPrincipal
3205
+ );
3206
+ const privileges = isObject(assignment) && Array.isArray(assignment.privileges) ? assignment.privileges : [];
3207
+ const canExecute = privileges.some(
3208
+ (privilege) => isObject(privilege) && privilege.privilege === "EXECUTE"
3209
+ );
3210
+ if (!canExecute) {
3211
+ throw new Error(`${expectedPrincipal} does not have EXECUTE on ${fullName}`);
3212
+ }
3213
+ return {
3214
+ fullName: service.full_name ?? service.name ?? fullName,
3215
+ connection: actualConnection,
3216
+ executePrincipal: expectedPrincipal,
3217
+ runtimeInvocationAvailable: false
3218
+ };
3219
+ }
3220
+ };
3221
+ }
2996
3222
  function regionalDrCheck(options = {}) {
2997
3223
  const names = [
2998
3224
  "DBX_TEST_DR_HOST",
@@ -3077,7 +3303,7 @@ function createAdvancedStreamingTargetPack() {
3077
3303
  "lakeflow-connect.replication"
3078
3304
  ],
3079
3305
  checks: () => [restrictedPrincipalCheck(), advancedStreamingCdcCheck(), lakeflowConnectCheck()],
3080
- requiredChecks: ["streaming-cdc", "lakeflow-connect"],
3306
+ requiredChecks: ["restricted-principal", "streaming-cdc", "lakeflow-connect"],
3081
3307
  requirements: [
3082
3308
  workspace,
3083
3309
  auth,
@@ -3116,7 +3342,7 @@ function createAiBiGenieTargetPack() {
3116
3342
  maturity: "live-gated",
3117
3343
  capabilities: ["aibi.dashboard", "genie.space", "genie.conversation"],
3118
3344
  checks: () => [restrictedPrincipalCheck(), aiBiDashboardCheck(), genieCheck()],
3119
- requiredChecks: ["aibi-dashboard", "genie"],
3345
+ requiredChecks: ["restricted-principal", "aibi-dashboard", "genie"],
3120
3346
  requirements: [
3121
3347
  workspace,
3122
3348
  auth,
@@ -3148,7 +3374,7 @@ function createMlflowLifecycleTargetPack() {
3148
3374
  "models.lifecycle"
3149
3375
  ],
3150
3376
  checks: () => [restrictedPrincipalCheck(), mlflowLifecycleCheck()],
3151
- requiredChecks: ["mlflow-lifecycle"],
3377
+ requiredChecks: ["restricted-principal", "mlflow-lifecycle"],
3152
3378
  requirements: [
3153
3379
  workspace,
3154
3380
  auth,
@@ -3180,7 +3406,7 @@ function createFeatureEngineeringTargetPack() {
3180
3406
  "features.serving"
3181
3407
  ],
3182
3408
  checks: () => [restrictedPrincipalCheck(), featureEngineeringServingCheck()],
3183
- requiredChecks: ["feature-engineering"],
3409
+ requiredChecks: ["restricted-principal", "feature-engineering"],
3184
3410
  requirements: [
3185
3411
  workspace,
3186
3412
  auth,
@@ -3217,7 +3443,7 @@ function createModelServingTargetPack() {
3217
3443
  "ai-gateway.configuration"
3218
3444
  ],
3219
3445
  checks: () => [restrictedPrincipalCheck(), modelServingCheck()],
3220
- requiredChecks: ["model-serving"],
3446
+ requiredChecks: ["restricted-principal", "model-serving"],
3221
3447
  requirements: [
3222
3448
  workspace,
3223
3449
  auth,
@@ -3253,7 +3479,7 @@ function createVectorRagAgentsTargetPack() {
3253
3479
  "agents.serving"
3254
3480
  ],
3255
3481
  checks: () => [restrictedPrincipalCheck(), vectorSearchCheck(), ragAgentCheck()],
3256
- requiredChecks: ["vector-search", "rag-agent"],
3482
+ requiredChecks: ["restricted-principal", "vector-search", "rag-agent"],
3257
3483
  requirements: [
3258
3484
  workspace,
3259
3485
  auth,
@@ -3283,7 +3509,7 @@ function createFederationSharingTargetPack() {
3283
3509
  "clean-rooms.access"
3284
3510
  ],
3285
3511
  checks: () => [restrictedPrincipalCheck(), federationSharingCleanRoomsCheck()],
3286
- requiredChecks: ["federation-sharing-cleanrooms"],
3512
+ requiredChecks: ["restricted-principal", "federation-sharing-cleanrooms"],
3287
3513
  requirements: [
3288
3514
  workspace,
3289
3515
  auth,
@@ -3317,7 +3543,7 @@ function createSecurityCostDrTargetPack() {
3317
3543
  "dr.secondary-region"
3318
3544
  ],
3319
3545
  checks: () => [restrictedPrincipalCheck(), securityPostureCostCheck(), regionalDrCheck()],
3320
- requiredChecks: ["security-cost", "regional-dr"],
3546
+ requiredChecks: ["restricted-principal", "security-cost", "regional-dr"],
3321
3547
  requirements: [
3322
3548
  workspace,
3323
3549
  auth,
@@ -3339,6 +3565,234 @@ function createSecurityCostDrTargetPack() {
3339
3565
  ]
3340
3566
  });
3341
3567
  }
3568
+ function createMlflow3AgentQualityTargetPack() {
3569
+ return defineTargetPack({
3570
+ id: "mlflow3-agent-quality",
3571
+ name: "MLflow 3 agent quality and monitoring",
3572
+ description: "Native MLflow 3 traces, offline agent evaluation, production scorer lifecycle and online monitoring.",
3573
+ version: "1.0.0",
3574
+ maturity: "shipped",
3575
+ capabilities: [
3576
+ "mlflow3.tracing",
3577
+ "mlflow3.agent-evaluation",
3578
+ "mlflow3.production-scorers",
3579
+ "mlflow3.production-monitoring",
3580
+ "ai-search.retrieval-evaluation"
3581
+ ],
3582
+ checks: () => [restrictedPrincipalCheck(), mlflow3GenAiQualityCheck()],
3583
+ requiredChecks: ["restricted-principal", "mlflow3-genai-quality"],
3584
+ requirements: [
3585
+ workspace,
3586
+ auth,
3587
+ warehouse,
3588
+ requirement("mlflow3-job", "Native MLflow 3 certification Job", "DBX_TEST_MLFLOW3_JOB_ID"),
3589
+ requirement(
3590
+ "mlflow3-traces",
3591
+ "Boolean SQL proving a fresh MLflow 3 trace",
3592
+ "DBX_TEST_MLFLOW3_TRACE_ASSERTION_SQL"
3593
+ ),
3594
+ requirement(
3595
+ "mlflow3-evaluation",
3596
+ "Boolean SQL proving offline agent evaluation results",
3597
+ "DBX_TEST_MLFLOW3_EVAL_ASSERTION_SQL"
3598
+ ),
3599
+ requirement(
3600
+ "mlflow3-scorers",
3601
+ "Boolean SQL proving production scorer lifecycle evidence",
3602
+ "DBX_TEST_MLFLOW3_SCORER_ASSERTION_SQL"
3603
+ ),
3604
+ requirement(
3605
+ "mlflow3-monitoring",
3606
+ "Boolean SQL proving production monitoring output",
3607
+ "DBX_TEST_MLFLOW3_MONITOR_ASSERTION_SQL"
3608
+ ),
3609
+ requirement(
3610
+ "ai-search-evaluation",
3611
+ "Boolean SQL proving AI Search retrieval-quality thresholds",
3612
+ "DBX_TEST_AI_SEARCH_EVAL_ASSERTION_SQL"
3613
+ )
3614
+ ],
3615
+ docsUrl: `${DOCS2}#mlflow-3-agent-quality-and-monitoring`,
3616
+ certificationScopes: [
3617
+ `${AZURE_M2M_SCOPE}; Python-native MLflow 3 trace, evaluation, complete scorer lifecycle and monitor evidence`
3618
+ ]
3619
+ });
3620
+ }
3621
+ function createAgentEvaluationPipelineTargetPack() {
3622
+ return defineTargetPack({
3623
+ id: "agent-evaluation-pipeline",
3624
+ name: "Agent evaluation delivery pipeline",
3625
+ description: "Exact-candidate API submission, outbox dispatch, Temporal execution, Databricks judge inference, SQL score persistence and Quality completion.",
3626
+ version: "1.0.0",
3627
+ maturity: "live-gated",
3628
+ capabilities: [
3629
+ "agent-evals.api-submission",
3630
+ "agent-evals.temporal-dispatch",
3631
+ "agent-evals.model-serving-judge",
3632
+ "agent-evals.sql-persistence",
3633
+ "agent-evals.quality-evidence"
3634
+ ],
3635
+ checks: () => [
3636
+ restrictedPrincipalCheck(),
3637
+ evalJudgeModelServingCheck(),
3638
+ agentEvaluationPipelineCheck()
3639
+ ],
3640
+ requiredChecks: ["restricted-principal", "model-eval-judge", "agent-evaluation-pipeline"],
3641
+ requirements: [
3642
+ workspace,
3643
+ auth,
3644
+ warehouse,
3645
+ requirement(
3646
+ "eval-serving-endpoint",
3647
+ "Databricks Model Serving endpoint satisfying the evaluator chat contract",
3648
+ "DBX_TEST_EVAL_SERVING_ENDPOINT"
3649
+ ),
3650
+ requirement(
3651
+ "agent-eval-job",
3652
+ "Exact-candidate end-to-end agent evaluation certification Job",
3653
+ "DBX_TEST_AGENT_EVAL_E2E_JOB_ID"
3654
+ ),
3655
+ requirement(
3656
+ "agent-eval-evidence",
3657
+ "Boolean SQL proving terminal scores and Quality evidence",
3658
+ "DBX_TEST_AGENT_EVAL_ASSERTION_SQL"
3659
+ )
3660
+ ],
3661
+ docsUrl: `${DOCS2}#agent-evaluation-delivery-pipeline`,
3662
+ certificationScopes: [
3663
+ `${AZURE_M2M_SCOPE}; Experiments API through customer worker and Databricks judge to SQL/Quality`
3664
+ ]
3665
+ });
3666
+ }
3667
+ function createManagedMcpAgentsTargetPack() {
3668
+ return defineTargetPack({
3669
+ id: "managed-mcp-agents",
3670
+ name: "Managed MCP and MCP Services",
3671
+ description: "Streamable HTTP tool discovery and calls for Databricks managed MCP servers and governed Unity Catalog MCP Services, including a deny control.",
3672
+ version: "1.0.0",
3673
+ maturity: "live-gated",
3674
+ capabilities: [
3675
+ "mcp.managed.list-call",
3676
+ "mcp.services.list-call",
3677
+ "mcp.unity-catalog-governance",
3678
+ "mcp.denial"
3679
+ ],
3680
+ checks: () => [restrictedPrincipalCheck(), managedMcpAgentsCheck()],
3681
+ requiredChecks: ["restricted-principal", "managed-mcp-agents"],
3682
+ requirements: [
3683
+ workspace,
3684
+ auth,
3685
+ warehouse,
3686
+ requirement("mcp-job", "Native Streamable HTTP MCP certification Job", "DBX_TEST_MCP_JOB_ID"),
3687
+ requirement(
3688
+ "managed-mcp",
3689
+ "Boolean SQL proving managed MCP list/call behavior",
3690
+ "DBX_TEST_MCP_MANAGED_ASSERTION_SQL"
3691
+ ),
3692
+ requirement(
3693
+ "mcp-service",
3694
+ "Boolean SQL proving UC MCP Service list/call behavior",
3695
+ "DBX_TEST_MCP_SERVICE_ASSERTION_SQL"
3696
+ ),
3697
+ requirement(
3698
+ "mcp-denial",
3699
+ "Boolean SQL proving an MCP tool excluded by UC selectors was denied",
3700
+ "DBX_TEST_MCP_DENY_ASSERTION_SQL"
3701
+ )
3702
+ ],
3703
+ docsUrl: `${DOCS2}#managed-mcp-and-mcp-services`,
3704
+ certificationScopes: [
3705
+ `${AZURE_M2M_SCOPE}; managed AI Search or SQL MCP plus a UC MCP Service over Streamable HTTP`
3706
+ ]
3707
+ });
3708
+ }
3709
+ function createDataQualityObservabilityTargetPack() {
3710
+ return defineTargetPack({
3711
+ id: "data-quality-observability",
3712
+ name: "Data quality and Databricks Apps observability",
3713
+ description: "Unity Catalog Data Quality Monitoring plus Databricks Apps OpenTelemetry logs, spans and metrics.",
3714
+ version: "1.0.0",
3715
+ maturity: "shipped",
3716
+ capabilities: [
3717
+ "data-quality.freshness-completeness",
3718
+ "apps.telemetry.logs",
3719
+ "apps.telemetry.traces",
3720
+ "apps.telemetry.metrics"
3721
+ ],
3722
+ checks: () => [restrictedPrincipalCheck(), dataQualityAppTelemetryCheck()],
3723
+ requiredChecks: ["restricted-principal", "data-quality-app-telemetry"],
3724
+ requirements: [
3725
+ workspace,
3726
+ auth,
3727
+ warehouse,
3728
+ requirement("telemetry-app", "App with UC telemetry export", "DBX_TEST_TELEMETRY_APP_NAME"),
3729
+ requirement(
3730
+ "data-quality",
3731
+ "Boolean SQL over current Data Quality Monitoring results",
3732
+ "DBX_TEST_DQM_ASSERTION_SQL"
3733
+ ),
3734
+ requirement(
3735
+ "app-logs",
3736
+ "Boolean SQL proving current app logs",
3737
+ "DBX_TEST_APP_LOGS_ASSERTION_SQL"
3738
+ ),
3739
+ requirement(
3740
+ "app-spans",
3741
+ "Boolean SQL proving current app spans",
3742
+ "DBX_TEST_APP_SPANS_ASSERTION_SQL"
3743
+ ),
3744
+ requirement(
3745
+ "app-metrics",
3746
+ "Boolean SQL proving current app metrics",
3747
+ "DBX_TEST_APP_METRICS_ASSERTION_SQL"
3748
+ )
3749
+ ],
3750
+ docsUrl: `${DOCS2}#data-quality-and-databricks-apps-observability`,
3751
+ certificationScopes: [
3752
+ `${AZURE_M2M_SCOPE}; system.data_quality_monitoring.table_results and UC OTel tables`
3753
+ ]
3754
+ });
3755
+ }
3756
+ function createAgentServicesEnrollmentTargetPack() {
3757
+ return defineTargetPack({
3758
+ id: "agent-services-enrollment",
3759
+ name: "Unity Catalog Agent Services enrollment",
3760
+ description: "Customer-owned Fabric Harness agents registered as external Unity Catalog Agent Services with explicit EXECUTE grants.",
3761
+ version: "1.0.0",
3762
+ maturity: "shipped",
3763
+ capabilities: [
3764
+ "agent-services.external-registration",
3765
+ "agent-services.discovery",
3766
+ "agent-services.permissions",
3767
+ "harness.customer-worker-enrollment"
3768
+ ],
3769
+ checks: () => [restrictedPrincipalCheck(), agentServicesEnrollmentCheck()],
3770
+ requiredChecks: ["restricted-principal", "agent-services-enrollment"],
3771
+ requirements: [
3772
+ workspace,
3773
+ auth,
3774
+ requirement(
3775
+ "agent-service",
3776
+ "External Unity Catalog Agent Service full name",
3777
+ "DBX_TEST_AGENT_SERVICE_FULL_NAME"
3778
+ ),
3779
+ requirement(
3780
+ "agent-connection",
3781
+ "Expected Unity Catalog HTTP connection",
3782
+ "DBX_TEST_AGENT_SERVICE_CONNECTION"
3783
+ ),
3784
+ requirement(
3785
+ "agent-execute-principal",
3786
+ "Principal expected to have EXECUTE",
3787
+ "DBX_TEST_AGENT_SERVICE_EXECUTE_PRINCIPAL"
3788
+ )
3789
+ ],
3790
+ docsUrl: `${DOCS2}#unity-catalog-agent-services-enrollment`,
3791
+ certificationScopes: [
3792
+ `${AZURE_M2M_SCOPE}; Agent Services Beta registration/discovery/ACL only; runtime invocation is not claimed`
3793
+ ]
3794
+ });
3795
+ }
3342
3796
  function createAdvancedTargetPacks() {
3343
3797
  return [
3344
3798
  createAdvancedStreamingTargetPack(),
@@ -3348,7 +3802,12 @@ function createAdvancedTargetPacks() {
3348
3802
  createModelServingTargetPack(),
3349
3803
  createVectorRagAgentsTargetPack(),
3350
3804
  createFederationSharingTargetPack(),
3351
- createSecurityCostDrTargetPack()
3805
+ createSecurityCostDrTargetPack(),
3806
+ createMlflow3AgentQualityTargetPack(),
3807
+ createManagedMcpAgentsTargetPack(),
3808
+ createDataQualityObservabilityTargetPack(),
3809
+ createAgentServicesEnrollmentTargetPack(),
3810
+ createAgentEvaluationPipelineTargetPack()
3352
3811
  ];
3353
3812
  }
3354
3813
 
@@ -3368,6 +3827,8 @@ init_workspace_client();
3368
3827
  exports.DatabricksAdvancedWorkloadsAdapter = DatabricksAdvancedWorkloadsAdapter;
3369
3828
  exports.DatabricksJobsAdapter = DatabricksJobsAdapter;
3370
3829
  exports.advancedStreamingCdcCheck = advancedStreamingCdcCheck;
3830
+ exports.agentEvaluationPipelineCheck = agentEvaluationPipelineCheck;
3831
+ exports.agentServicesEnrollmentCheck = agentServicesEnrollmentCheck;
3371
3832
  exports.aiBiDashboardCheck = aiBiDashboardCheck;
3372
3833
  exports.appHealthCheck = appHealthCheck;
3373
3834
  exports.assertDeltaContract = assertDeltaContract;
@@ -3378,10 +3839,13 @@ exports.classicComputeCheck = classicComputeCheck;
3378
3839
  exports.compareToGolden = compareToGolden;
3379
3840
  exports.createAdvancedStreamingTargetPack = createAdvancedStreamingTargetPack;
3380
3841
  exports.createAdvancedTargetPacks = createAdvancedTargetPacks;
3842
+ exports.createAgentEvaluationPipelineTargetPack = createAgentEvaluationPipelineTargetPack;
3843
+ exports.createAgentServicesEnrollmentTargetPack = createAgentServicesEnrollmentTargetPack;
3381
3844
  exports.createAiBiGenieTargetPack = createAiBiGenieTargetPack;
3382
3845
  exports.createAppsOperationalFoundationPack = createAppsOperationalFoundationPack;
3383
3846
  exports.createBuiltinTargetPacks = createBuiltinTargetPacks;
3384
3847
  exports.createClientFromEnv = createClientFromEnv;
3848
+ exports.createDataQualityObservabilityTargetPack = createDataQualityObservabilityTargetPack;
3385
3849
  exports.createDuckDbContext = createDuckDbContext;
3386
3850
  exports.createEphemeralLakebaseBranch = createEphemeralLakebaseBranch;
3387
3851
  exports.createExecutionContext = createExecutionContext;
@@ -3392,6 +3856,8 @@ exports.createJobsCodeFoundationPack = createJobsCodeFoundationPack;
3392
3856
  exports.createLakebaseTestProvider = createLakebaseTestProvider;
3393
3857
  exports.createLakeflowFoundationPack = createLakeflowFoundationPack;
3394
3858
  exports.createLakeflowJobsTargetPack = createLakeflowJobsTargetPack;
3859
+ exports.createManagedMcpAgentsTargetPack = createManagedMcpAgentsTargetPack;
3860
+ exports.createMlflow3AgentQualityTargetPack = createMlflow3AgentQualityTargetPack;
3395
3861
  exports.createMlflowLifecycleTargetPack = createMlflowLifecycleTargetPack;
3396
3862
  exports.createMockDatabricksClient = createMockDatabricksClient;
3397
3863
  exports.createModelServingTargetPack = createModelServingTargetPack;
@@ -3402,6 +3868,7 @@ exports.createUnityStorageFoundationPack = createUnityStorageFoundationPack;
3402
3868
  exports.createVectorRagAgentsTargetPack = createVectorRagAgentsTargetPack;
3403
3869
  exports.createWorkspaceContext = createWorkspaceContext;
3404
3870
  exports.customLiveCheck = customLiveCheck;
3871
+ exports.dataQualityAppTelemetryCheck = dataQualityAppTelemetryCheck;
3405
3872
  exports.dbtBuildCheck = dbtBuildCheck;
3406
3873
  exports.defineLiveSuite = defineLiveSuite;
3407
3874
  exports.defineTargetPack = defineTargetPack;
@@ -3425,6 +3892,8 @@ exports.jobsOrchestrationCheck = jobsOrchestrationCheck;
3425
3892
  exports.lakebaseCredentialCheck = lakebaseCredentialCheck;
3426
3893
  exports.lakebaseRoundTripCheck = lakebaseRoundTripCheck;
3427
3894
  exports.lakeflowConnectCheck = lakeflowConnectCheck;
3895
+ exports.managedMcpAgentsCheck = managedMcpAgentsCheck;
3896
+ exports.mlflow3GenAiQualityCheck = mlflow3GenAiQualityCheck;
3428
3897
  exports.mlflowLifecycleCheck = mlflowLifecycleCheck;
3429
3898
  exports.modelServingCheck = modelServingCheck;
3430
3899
  exports.normalizeJobRun = normalizeJobRun;