@fabricorg/databricks-testkit 0.7.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +478 -9
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +37 -1
- package/dist/index.d.ts +37 -1
- package/dist/index.js +468 -10
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -2495,6 +2495,7 @@ function createLakeflowJobsTargetPack() {
|
|
|
2495
2495
|
}
|
|
2496
2496
|
|
|
2497
2497
|
// src/advanced-workload-checks.ts
|
|
2498
|
+
init_workspace_client();
|
|
2498
2499
|
init_workspace_context();
|
|
2499
2500
|
var DatabricksAdvancedWorkloadsAdapter = class {
|
|
2500
2501
|
constructor(client) {
|
|
@@ -2568,6 +2569,15 @@ var DatabricksAdvancedWorkloadsAdapter = class {
|
|
|
2568
2569
|
warehouse(id) {
|
|
2569
2570
|
return this.client.get(`/api/2.0/sql/warehouses/${segment(id)}`);
|
|
2570
2571
|
}
|
|
2572
|
+
app(name) {
|
|
2573
|
+
return this.client.get(`/api/2.0/apps/${segment(name)}`);
|
|
2574
|
+
}
|
|
2575
|
+
agentService(fullName) {
|
|
2576
|
+
return this.client.get(`/api/2.1/unity-catalog/agent-services/${segment(fullName)}`);
|
|
2577
|
+
}
|
|
2578
|
+
agentServicePermissions(fullName) {
|
|
2579
|
+
return this.client.get(`/api/2.1/unity-catalog/permissions/AGENT_SERVICE/${segment(fullName)}`);
|
|
2580
|
+
}
|
|
2571
2581
|
};
|
|
2572
2582
|
function segment(value) {
|
|
2573
2583
|
return encodeURIComponent(value);
|
|
@@ -2614,6 +2624,27 @@ async function booleanSql(label, sql, query) {
|
|
|
2614
2624
|
throw new Error(`${label} assertion did not return a truthy first column`);
|
|
2615
2625
|
}
|
|
2616
2626
|
}
|
|
2627
|
+
async function runCertificationJob(label, jobIdValue, env, client, options = {}) {
|
|
2628
|
+
const jobId = Number(jobIdValue);
|
|
2629
|
+
if (!Number.isSafeInteger(jobId) || jobId <= 0) {
|
|
2630
|
+
throw new Error(`${label} job id must be a positive integer`);
|
|
2631
|
+
}
|
|
2632
|
+
const submitted = await client.post("/api/2.1/jobs/run-now", {
|
|
2633
|
+
job_id: jobId
|
|
2634
|
+
});
|
|
2635
|
+
if (!Number.isSafeInteger(submitted.run_id)) {
|
|
2636
|
+
throw new Error(`${label} certification job returned no run_id`);
|
|
2637
|
+
}
|
|
2638
|
+
const run = await waitForDatabricksRun(client, submitted.run_id, {
|
|
2639
|
+
timeoutMs: options.timeoutMs ?? Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
|
|
2640
|
+
pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
|
|
2641
|
+
});
|
|
2642
|
+
const result = run.state?.result_state;
|
|
2643
|
+
if (result !== "SUCCESS") {
|
|
2644
|
+
throw new Error(`${label} certification job ${jobId} ended in ${result ?? "UNKNOWN"}`);
|
|
2645
|
+
}
|
|
2646
|
+
return { jobId, runId: submitted.run_id };
|
|
2647
|
+
}
|
|
2617
2648
|
function advancedStreamingCdcCheck() {
|
|
2618
2649
|
const names = [
|
|
2619
2650
|
"DBX_TEST_STREAMING_PIPELINE_ID",
|
|
@@ -2993,6 +3024,201 @@ function securityPostureCostCheck() {
|
|
|
2993
3024
|
}
|
|
2994
3025
|
};
|
|
2995
3026
|
}
|
|
3027
|
+
function mlflow3GenAiQualityCheck() {
|
|
3028
|
+
const names = [
|
|
3029
|
+
"DBX_TEST_MLFLOW3_JOB_ID",
|
|
3030
|
+
"DBX_TEST_MLFLOW3_TRACE_ASSERTION_SQL",
|
|
3031
|
+
"DBX_TEST_MLFLOW3_EVAL_ASSERTION_SQL",
|
|
3032
|
+
"DBX_TEST_MLFLOW3_SCORER_ASSERTION_SQL",
|
|
3033
|
+
"DBX_TEST_MLFLOW3_MONITOR_ASSERTION_SQL",
|
|
3034
|
+
"DBX_TEST_AI_SEARCH_EVAL_ASSERTION_SQL"
|
|
3035
|
+
];
|
|
3036
|
+
return {
|
|
3037
|
+
id: "mlflow3-genai-quality",
|
|
3038
|
+
description: "MLflow 3 tracing, offline agent evaluation and production scorer monitoring produce durable evidence",
|
|
3039
|
+
configured: (env) => required2(env, names).length === 0,
|
|
3040
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
3041
|
+
requireEnv(env, names);
|
|
3042
|
+
const job = await runCertificationJob(
|
|
3043
|
+
"MLflow 3 GenAI",
|
|
3044
|
+
env.DBX_TEST_MLFLOW3_JOB_ID,
|
|
3045
|
+
env,
|
|
3046
|
+
client,
|
|
3047
|
+
{ timeoutMs: Number(env.DBX_TEST_MLFLOW3_TIMEOUT_MS ?? 72e5) }
|
|
3048
|
+
);
|
|
3049
|
+
await booleanSql("MLflow 3 trace", env.DBX_TEST_MLFLOW3_TRACE_ASSERTION_SQL, warehouse2.query);
|
|
3050
|
+
await booleanSql(
|
|
3051
|
+
"MLflow 3 offline evaluation",
|
|
3052
|
+
env.DBX_TEST_MLFLOW3_EVAL_ASSERTION_SQL,
|
|
3053
|
+
warehouse2.query
|
|
3054
|
+
);
|
|
3055
|
+
await booleanSql(
|
|
3056
|
+
"MLflow 3 production scorer lifecycle",
|
|
3057
|
+
env.DBX_TEST_MLFLOW3_SCORER_ASSERTION_SQL,
|
|
3058
|
+
warehouse2.query
|
|
3059
|
+
);
|
|
3060
|
+
await booleanSql(
|
|
3061
|
+
"MLflow 3 production monitoring",
|
|
3062
|
+
env.DBX_TEST_MLFLOW3_MONITOR_ASSERTION_SQL,
|
|
3063
|
+
warehouse2.query
|
|
3064
|
+
);
|
|
3065
|
+
await booleanSql(
|
|
3066
|
+
"AI Search retrieval quality",
|
|
3067
|
+
env.DBX_TEST_AI_SEARCH_EVAL_ASSERTION_SQL,
|
|
3068
|
+
warehouse2.query
|
|
3069
|
+
);
|
|
3070
|
+
return {
|
|
3071
|
+
...job,
|
|
3072
|
+
traces: true,
|
|
3073
|
+
offlineEvaluation: true,
|
|
3074
|
+
scorerLifecycle: true,
|
|
3075
|
+
monitor: true,
|
|
3076
|
+
aiSearchEvaluation: true
|
|
3077
|
+
};
|
|
3078
|
+
}
|
|
3079
|
+
};
|
|
3080
|
+
}
|
|
3081
|
+
function agentEvaluationPipelineCheck() {
|
|
3082
|
+
const names = ["DBX_TEST_AGENT_EVAL_E2E_JOB_ID", "DBX_TEST_AGENT_EVAL_ASSERTION_SQL"];
|
|
3083
|
+
return {
|
|
3084
|
+
id: "agent-evaluation-pipeline",
|
|
3085
|
+
description: "Experiments API, outbox, Temporal worker, Model Serving judge, SQL score sink and Quality completion execute end to end",
|
|
3086
|
+
configured: (env) => required2(env, names).length === 0,
|
|
3087
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
3088
|
+
requireEnv(env, names);
|
|
3089
|
+
const job = await runCertificationJob(
|
|
3090
|
+
"Agent evaluation end-to-end",
|
|
3091
|
+
env.DBX_TEST_AGENT_EVAL_E2E_JOB_ID,
|
|
3092
|
+
env,
|
|
3093
|
+
client
|
|
3094
|
+
);
|
|
3095
|
+
await booleanSql(
|
|
3096
|
+
"Agent evaluation terminal Quality evidence",
|
|
3097
|
+
env.DBX_TEST_AGENT_EVAL_ASSERTION_SQL,
|
|
3098
|
+
warehouse2.query
|
|
3099
|
+
);
|
|
3100
|
+
return { ...job, api: true, outbox: true, temporal: true, judged: true, persisted: true };
|
|
3101
|
+
}
|
|
3102
|
+
};
|
|
3103
|
+
}
|
|
3104
|
+
function managedMcpAgentsCheck() {
|
|
3105
|
+
const names = [
|
|
3106
|
+
"DBX_TEST_MCP_JOB_ID",
|
|
3107
|
+
"DBX_TEST_MCP_MANAGED_ASSERTION_SQL",
|
|
3108
|
+
"DBX_TEST_MCP_SERVICE_ASSERTION_SQL",
|
|
3109
|
+
"DBX_TEST_MCP_DENY_ASSERTION_SQL"
|
|
3110
|
+
];
|
|
3111
|
+
return {
|
|
3112
|
+
id: "managed-mcp-agents",
|
|
3113
|
+
description: "Managed MCP and Unity Catalog MCP Services support list, call and deny behavior",
|
|
3114
|
+
configured: (env) => required2(env, names).length === 0,
|
|
3115
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
3116
|
+
requireEnv(env, names);
|
|
3117
|
+
const job = await runCertificationJob("Databricks MCP", env.DBX_TEST_MCP_JOB_ID, env, client);
|
|
3118
|
+
await booleanSql(
|
|
3119
|
+
"Managed MCP list/call",
|
|
3120
|
+
env.DBX_TEST_MCP_MANAGED_ASSERTION_SQL,
|
|
3121
|
+
warehouse2.query
|
|
3122
|
+
);
|
|
3123
|
+
await booleanSql(
|
|
3124
|
+
"Unity Catalog MCP Service list/call",
|
|
3125
|
+
env.DBX_TEST_MCP_SERVICE_ASSERTION_SQL,
|
|
3126
|
+
warehouse2.query
|
|
3127
|
+
);
|
|
3128
|
+
await booleanSql(
|
|
3129
|
+
"MCP excluded-tool denial",
|
|
3130
|
+
env.DBX_TEST_MCP_DENY_ASSERTION_SQL,
|
|
3131
|
+
warehouse2.query
|
|
3132
|
+
);
|
|
3133
|
+
return { ...job, managedServer: true, mcpService: true, denyControl: true };
|
|
3134
|
+
}
|
|
3135
|
+
};
|
|
3136
|
+
}
|
|
3137
|
+
function dataQualityAppTelemetryCheck() {
|
|
3138
|
+
const names = [
|
|
3139
|
+
"DBX_TEST_TELEMETRY_APP_NAME",
|
|
3140
|
+
"DBX_TEST_DQM_ASSERTION_SQL",
|
|
3141
|
+
"DBX_TEST_APP_LOGS_ASSERTION_SQL",
|
|
3142
|
+
"DBX_TEST_APP_SPANS_ASSERTION_SQL",
|
|
3143
|
+
"DBX_TEST_APP_METRICS_ASSERTION_SQL"
|
|
3144
|
+
];
|
|
3145
|
+
return {
|
|
3146
|
+
id: "data-quality-app-telemetry",
|
|
3147
|
+
description: "Unity Catalog data quality monitoring and Databricks Apps logs, spans and metrics are current",
|
|
3148
|
+
configured: (env) => required2(env, names).length === 0,
|
|
3149
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
3150
|
+
requireEnv(env, names);
|
|
3151
|
+
const appName = env.DBX_TEST_TELEMETRY_APP_NAME;
|
|
3152
|
+
const app = await adapter(client).app(appName);
|
|
3153
|
+
const destinations = Array.isArray(app.telemetry_export_destinations) ? app.telemetry_export_destinations : [];
|
|
3154
|
+
const unityCatalog = destinations.find(
|
|
3155
|
+
(destination) => isObject(destination) && isObject(destination.unity_catalog)
|
|
3156
|
+
);
|
|
3157
|
+
if (!isObject(unityCatalog) || !isObject(unityCatalog.unity_catalog)) {
|
|
3158
|
+
throw new Error(`Databricks App ${appName} has no Unity Catalog telemetry destination`);
|
|
3159
|
+
}
|
|
3160
|
+
const tables = unityCatalog.unity_catalog;
|
|
3161
|
+
for (const field of ["logs_table", "traces_table", "metrics_table"]) {
|
|
3162
|
+
if (typeof tables[field] !== "string" || !tables[field]) {
|
|
3163
|
+
throw new Error(`Databricks App ${appName} telemetry destination has no ${field}`);
|
|
3164
|
+
}
|
|
3165
|
+
}
|
|
3166
|
+
await booleanSql("Data Quality Monitoring", env.DBX_TEST_DQM_ASSERTION_SQL, warehouse2.query);
|
|
3167
|
+
await booleanSql("App logs", env.DBX_TEST_APP_LOGS_ASSERTION_SQL, warehouse2.query);
|
|
3168
|
+
await booleanSql("App spans", env.DBX_TEST_APP_SPANS_ASSERTION_SQL, warehouse2.query);
|
|
3169
|
+
await booleanSql("App metrics", env.DBX_TEST_APP_METRICS_ASSERTION_SQL, warehouse2.query);
|
|
3170
|
+
return { app: appName, unityCatalogTelemetry: true, dataQualityMonitoring: true };
|
|
3171
|
+
}
|
|
3172
|
+
};
|
|
3173
|
+
}
|
|
3174
|
+
function agentServicesEnrollmentCheck() {
|
|
3175
|
+
const names = [
|
|
3176
|
+
"DBX_TEST_AGENT_SERVICE_FULL_NAME",
|
|
3177
|
+
"DBX_TEST_AGENT_SERVICE_CONNECTION",
|
|
3178
|
+
"DBX_TEST_AGENT_SERVICE_EXECUTE_PRINCIPAL"
|
|
3179
|
+
];
|
|
3180
|
+
return {
|
|
3181
|
+
id: "agent-services-enrollment",
|
|
3182
|
+
description: "A customer-owned Harness agent is registered as an external Unity Catalog Agent Service with governed access",
|
|
3183
|
+
configured: (env) => required2(env, names).length === 0,
|
|
3184
|
+
async run({ env, client }) {
|
|
3185
|
+
requireEnv(env, names);
|
|
3186
|
+
const fullName = env.DBX_TEST_AGENT_SERVICE_FULL_NAME;
|
|
3187
|
+
const api = adapter(client);
|
|
3188
|
+
const service = await api.agentService(fullName);
|
|
3189
|
+
if (service.agent_service_type !== "AGENT_SERVICE_TYPE_EXTERNAL") {
|
|
3190
|
+
throw new Error(`${fullName} is not an external Agent Service`);
|
|
3191
|
+
}
|
|
3192
|
+
const expectedConnection = env.DBX_TEST_AGENT_SERVICE_CONNECTION;
|
|
3193
|
+
const actualConnection = service.config?.connection?.name ?? "";
|
|
3194
|
+
if (actualConnection !== expectedConnection && actualConnection.split("/").at(-1) !== expectedConnection) {
|
|
3195
|
+
throw new Error(`${fullName} uses unexpected connection ${actualConnection || "UNKNOWN"}`);
|
|
3196
|
+
}
|
|
3197
|
+
if (!service.config?.base_path?.startsWith("/")) {
|
|
3198
|
+
throw new Error(`${fullName} has no absolute external base path`);
|
|
3199
|
+
}
|
|
3200
|
+
const permissions = await api.agentServicePermissions(fullName);
|
|
3201
|
+
const assignments = Array.isArray(permissions.privilege_assignments) ? permissions.privilege_assignments : [];
|
|
3202
|
+
const expectedPrincipal = env.DBX_TEST_AGENT_SERVICE_EXECUTE_PRINCIPAL;
|
|
3203
|
+
const assignment = assignments.find(
|
|
3204
|
+
(entry) => isObject(entry) && entry.principal === expectedPrincipal
|
|
3205
|
+
);
|
|
3206
|
+
const privileges = isObject(assignment) && Array.isArray(assignment.privileges) ? assignment.privileges : [];
|
|
3207
|
+
const canExecute = privileges.some(
|
|
3208
|
+
(privilege) => isObject(privilege) && privilege.privilege === "EXECUTE"
|
|
3209
|
+
);
|
|
3210
|
+
if (!canExecute) {
|
|
3211
|
+
throw new Error(`${expectedPrincipal} does not have EXECUTE on ${fullName}`);
|
|
3212
|
+
}
|
|
3213
|
+
return {
|
|
3214
|
+
fullName: service.full_name ?? service.name ?? fullName,
|
|
3215
|
+
connection: actualConnection,
|
|
3216
|
+
executePrincipal: expectedPrincipal,
|
|
3217
|
+
runtimeInvocationAvailable: false
|
|
3218
|
+
};
|
|
3219
|
+
}
|
|
3220
|
+
};
|
|
3221
|
+
}
|
|
2996
3222
|
function regionalDrCheck(options = {}) {
|
|
2997
3223
|
const names = [
|
|
2998
3224
|
"DBX_TEST_DR_HOST",
|
|
@@ -3077,7 +3303,7 @@ function createAdvancedStreamingTargetPack() {
|
|
|
3077
3303
|
"lakeflow-connect.replication"
|
|
3078
3304
|
],
|
|
3079
3305
|
checks: () => [restrictedPrincipalCheck(), advancedStreamingCdcCheck(), lakeflowConnectCheck()],
|
|
3080
|
-
requiredChecks: ["streaming-cdc", "lakeflow-connect"],
|
|
3306
|
+
requiredChecks: ["restricted-principal", "streaming-cdc", "lakeflow-connect"],
|
|
3081
3307
|
requirements: [
|
|
3082
3308
|
workspace,
|
|
3083
3309
|
auth,
|
|
@@ -3116,7 +3342,7 @@ function createAiBiGenieTargetPack() {
|
|
|
3116
3342
|
maturity: "live-gated",
|
|
3117
3343
|
capabilities: ["aibi.dashboard", "genie.space", "genie.conversation"],
|
|
3118
3344
|
checks: () => [restrictedPrincipalCheck(), aiBiDashboardCheck(), genieCheck()],
|
|
3119
|
-
requiredChecks: ["aibi-dashboard", "genie"],
|
|
3345
|
+
requiredChecks: ["restricted-principal", "aibi-dashboard", "genie"],
|
|
3120
3346
|
requirements: [
|
|
3121
3347
|
workspace,
|
|
3122
3348
|
auth,
|
|
@@ -3148,7 +3374,7 @@ function createMlflowLifecycleTargetPack() {
|
|
|
3148
3374
|
"models.lifecycle"
|
|
3149
3375
|
],
|
|
3150
3376
|
checks: () => [restrictedPrincipalCheck(), mlflowLifecycleCheck()],
|
|
3151
|
-
requiredChecks: ["mlflow-lifecycle"],
|
|
3377
|
+
requiredChecks: ["restricted-principal", "mlflow-lifecycle"],
|
|
3152
3378
|
requirements: [
|
|
3153
3379
|
workspace,
|
|
3154
3380
|
auth,
|
|
@@ -3180,7 +3406,7 @@ function createFeatureEngineeringTargetPack() {
|
|
|
3180
3406
|
"features.serving"
|
|
3181
3407
|
],
|
|
3182
3408
|
checks: () => [restrictedPrincipalCheck(), featureEngineeringServingCheck()],
|
|
3183
|
-
requiredChecks: ["feature-engineering"],
|
|
3409
|
+
requiredChecks: ["restricted-principal", "feature-engineering"],
|
|
3184
3410
|
requirements: [
|
|
3185
3411
|
workspace,
|
|
3186
3412
|
auth,
|
|
@@ -3217,7 +3443,7 @@ function createModelServingTargetPack() {
|
|
|
3217
3443
|
"ai-gateway.configuration"
|
|
3218
3444
|
],
|
|
3219
3445
|
checks: () => [restrictedPrincipalCheck(), modelServingCheck()],
|
|
3220
|
-
requiredChecks: ["model-serving"],
|
|
3446
|
+
requiredChecks: ["restricted-principal", "model-serving"],
|
|
3221
3447
|
requirements: [
|
|
3222
3448
|
workspace,
|
|
3223
3449
|
auth,
|
|
@@ -3253,7 +3479,7 @@ function createVectorRagAgentsTargetPack() {
|
|
|
3253
3479
|
"agents.serving"
|
|
3254
3480
|
],
|
|
3255
3481
|
checks: () => [restrictedPrincipalCheck(), vectorSearchCheck(), ragAgentCheck()],
|
|
3256
|
-
requiredChecks: ["vector-search", "rag-agent"],
|
|
3482
|
+
requiredChecks: ["restricted-principal", "vector-search", "rag-agent"],
|
|
3257
3483
|
requirements: [
|
|
3258
3484
|
workspace,
|
|
3259
3485
|
auth,
|
|
@@ -3283,7 +3509,7 @@ function createFederationSharingTargetPack() {
|
|
|
3283
3509
|
"clean-rooms.access"
|
|
3284
3510
|
],
|
|
3285
3511
|
checks: () => [restrictedPrincipalCheck(), federationSharingCleanRoomsCheck()],
|
|
3286
|
-
requiredChecks: ["federation-sharing-cleanrooms"],
|
|
3512
|
+
requiredChecks: ["restricted-principal", "federation-sharing-cleanrooms"],
|
|
3287
3513
|
requirements: [
|
|
3288
3514
|
workspace,
|
|
3289
3515
|
auth,
|
|
@@ -3317,7 +3543,7 @@ function createSecurityCostDrTargetPack() {
|
|
|
3317
3543
|
"dr.secondary-region"
|
|
3318
3544
|
],
|
|
3319
3545
|
checks: () => [restrictedPrincipalCheck(), securityPostureCostCheck(), regionalDrCheck()],
|
|
3320
|
-
requiredChecks: ["security-cost", "regional-dr"],
|
|
3546
|
+
requiredChecks: ["restricted-principal", "security-cost", "regional-dr"],
|
|
3321
3547
|
requirements: [
|
|
3322
3548
|
workspace,
|
|
3323
3549
|
auth,
|
|
@@ -3339,6 +3565,234 @@ function createSecurityCostDrTargetPack() {
|
|
|
3339
3565
|
]
|
|
3340
3566
|
});
|
|
3341
3567
|
}
|
|
3568
|
+
function createMlflow3AgentQualityTargetPack() {
|
|
3569
|
+
return defineTargetPack({
|
|
3570
|
+
id: "mlflow3-agent-quality",
|
|
3571
|
+
name: "MLflow 3 agent quality and monitoring",
|
|
3572
|
+
description: "Native MLflow 3 traces, offline agent evaluation, production scorer lifecycle and online monitoring.",
|
|
3573
|
+
version: "1.0.0",
|
|
3574
|
+
maturity: "shipped",
|
|
3575
|
+
capabilities: [
|
|
3576
|
+
"mlflow3.tracing",
|
|
3577
|
+
"mlflow3.agent-evaluation",
|
|
3578
|
+
"mlflow3.production-scorers",
|
|
3579
|
+
"mlflow3.production-monitoring",
|
|
3580
|
+
"ai-search.retrieval-evaluation"
|
|
3581
|
+
],
|
|
3582
|
+
checks: () => [restrictedPrincipalCheck(), mlflow3GenAiQualityCheck()],
|
|
3583
|
+
requiredChecks: ["restricted-principal", "mlflow3-genai-quality"],
|
|
3584
|
+
requirements: [
|
|
3585
|
+
workspace,
|
|
3586
|
+
auth,
|
|
3587
|
+
warehouse,
|
|
3588
|
+
requirement("mlflow3-job", "Native MLflow 3 certification Job", "DBX_TEST_MLFLOW3_JOB_ID"),
|
|
3589
|
+
requirement(
|
|
3590
|
+
"mlflow3-traces",
|
|
3591
|
+
"Boolean SQL proving a fresh MLflow 3 trace",
|
|
3592
|
+
"DBX_TEST_MLFLOW3_TRACE_ASSERTION_SQL"
|
|
3593
|
+
),
|
|
3594
|
+
requirement(
|
|
3595
|
+
"mlflow3-evaluation",
|
|
3596
|
+
"Boolean SQL proving offline agent evaluation results",
|
|
3597
|
+
"DBX_TEST_MLFLOW3_EVAL_ASSERTION_SQL"
|
|
3598
|
+
),
|
|
3599
|
+
requirement(
|
|
3600
|
+
"mlflow3-scorers",
|
|
3601
|
+
"Boolean SQL proving production scorer lifecycle evidence",
|
|
3602
|
+
"DBX_TEST_MLFLOW3_SCORER_ASSERTION_SQL"
|
|
3603
|
+
),
|
|
3604
|
+
requirement(
|
|
3605
|
+
"mlflow3-monitoring",
|
|
3606
|
+
"Boolean SQL proving production monitoring output",
|
|
3607
|
+
"DBX_TEST_MLFLOW3_MONITOR_ASSERTION_SQL"
|
|
3608
|
+
),
|
|
3609
|
+
requirement(
|
|
3610
|
+
"ai-search-evaluation",
|
|
3611
|
+
"Boolean SQL proving AI Search retrieval-quality thresholds",
|
|
3612
|
+
"DBX_TEST_AI_SEARCH_EVAL_ASSERTION_SQL"
|
|
3613
|
+
)
|
|
3614
|
+
],
|
|
3615
|
+
docsUrl: `${DOCS2}#mlflow-3-agent-quality-and-monitoring`,
|
|
3616
|
+
certificationScopes: [
|
|
3617
|
+
`${AZURE_M2M_SCOPE}; Python-native MLflow 3 trace, evaluation, complete scorer lifecycle and monitor evidence`
|
|
3618
|
+
]
|
|
3619
|
+
});
|
|
3620
|
+
}
|
|
3621
|
+
function createAgentEvaluationPipelineTargetPack() {
|
|
3622
|
+
return defineTargetPack({
|
|
3623
|
+
id: "agent-evaluation-pipeline",
|
|
3624
|
+
name: "Agent evaluation delivery pipeline",
|
|
3625
|
+
description: "Exact-candidate API submission, outbox dispatch, Temporal execution, Databricks judge inference, SQL score persistence and Quality completion.",
|
|
3626
|
+
version: "1.0.0",
|
|
3627
|
+
maturity: "live-gated",
|
|
3628
|
+
capabilities: [
|
|
3629
|
+
"agent-evals.api-submission",
|
|
3630
|
+
"agent-evals.temporal-dispatch",
|
|
3631
|
+
"agent-evals.model-serving-judge",
|
|
3632
|
+
"agent-evals.sql-persistence",
|
|
3633
|
+
"agent-evals.quality-evidence"
|
|
3634
|
+
],
|
|
3635
|
+
checks: () => [
|
|
3636
|
+
restrictedPrincipalCheck(),
|
|
3637
|
+
evalJudgeModelServingCheck(),
|
|
3638
|
+
agentEvaluationPipelineCheck()
|
|
3639
|
+
],
|
|
3640
|
+
requiredChecks: ["restricted-principal", "model-eval-judge", "agent-evaluation-pipeline"],
|
|
3641
|
+
requirements: [
|
|
3642
|
+
workspace,
|
|
3643
|
+
auth,
|
|
3644
|
+
warehouse,
|
|
3645
|
+
requirement(
|
|
3646
|
+
"eval-serving-endpoint",
|
|
3647
|
+
"Databricks Model Serving endpoint satisfying the evaluator chat contract",
|
|
3648
|
+
"DBX_TEST_EVAL_SERVING_ENDPOINT"
|
|
3649
|
+
),
|
|
3650
|
+
requirement(
|
|
3651
|
+
"agent-eval-job",
|
|
3652
|
+
"Exact-candidate end-to-end agent evaluation certification Job",
|
|
3653
|
+
"DBX_TEST_AGENT_EVAL_E2E_JOB_ID"
|
|
3654
|
+
),
|
|
3655
|
+
requirement(
|
|
3656
|
+
"agent-eval-evidence",
|
|
3657
|
+
"Boolean SQL proving terminal scores and Quality evidence",
|
|
3658
|
+
"DBX_TEST_AGENT_EVAL_ASSERTION_SQL"
|
|
3659
|
+
)
|
|
3660
|
+
],
|
|
3661
|
+
docsUrl: `${DOCS2}#agent-evaluation-delivery-pipeline`,
|
|
3662
|
+
certificationScopes: [
|
|
3663
|
+
`${AZURE_M2M_SCOPE}; Experiments API through customer worker and Databricks judge to SQL/Quality`
|
|
3664
|
+
]
|
|
3665
|
+
});
|
|
3666
|
+
}
|
|
3667
|
+
function createManagedMcpAgentsTargetPack() {
|
|
3668
|
+
return defineTargetPack({
|
|
3669
|
+
id: "managed-mcp-agents",
|
|
3670
|
+
name: "Managed MCP and MCP Services",
|
|
3671
|
+
description: "Streamable HTTP tool discovery and calls for Databricks managed MCP servers and governed Unity Catalog MCP Services, including a deny control.",
|
|
3672
|
+
version: "1.0.0",
|
|
3673
|
+
maturity: "live-gated",
|
|
3674
|
+
capabilities: [
|
|
3675
|
+
"mcp.managed.list-call",
|
|
3676
|
+
"mcp.services.list-call",
|
|
3677
|
+
"mcp.unity-catalog-governance",
|
|
3678
|
+
"mcp.denial"
|
|
3679
|
+
],
|
|
3680
|
+
checks: () => [restrictedPrincipalCheck(), managedMcpAgentsCheck()],
|
|
3681
|
+
requiredChecks: ["restricted-principal", "managed-mcp-agents"],
|
|
3682
|
+
requirements: [
|
|
3683
|
+
workspace,
|
|
3684
|
+
auth,
|
|
3685
|
+
warehouse,
|
|
3686
|
+
requirement("mcp-job", "Native Streamable HTTP MCP certification Job", "DBX_TEST_MCP_JOB_ID"),
|
|
3687
|
+
requirement(
|
|
3688
|
+
"managed-mcp",
|
|
3689
|
+
"Boolean SQL proving managed MCP list/call behavior",
|
|
3690
|
+
"DBX_TEST_MCP_MANAGED_ASSERTION_SQL"
|
|
3691
|
+
),
|
|
3692
|
+
requirement(
|
|
3693
|
+
"mcp-service",
|
|
3694
|
+
"Boolean SQL proving UC MCP Service list/call behavior",
|
|
3695
|
+
"DBX_TEST_MCP_SERVICE_ASSERTION_SQL"
|
|
3696
|
+
),
|
|
3697
|
+
requirement(
|
|
3698
|
+
"mcp-denial",
|
|
3699
|
+
"Boolean SQL proving an MCP tool excluded by UC selectors was denied",
|
|
3700
|
+
"DBX_TEST_MCP_DENY_ASSERTION_SQL"
|
|
3701
|
+
)
|
|
3702
|
+
],
|
|
3703
|
+
docsUrl: `${DOCS2}#managed-mcp-and-mcp-services`,
|
|
3704
|
+
certificationScopes: [
|
|
3705
|
+
`${AZURE_M2M_SCOPE}; managed AI Search or SQL MCP plus a UC MCP Service over Streamable HTTP`
|
|
3706
|
+
]
|
|
3707
|
+
});
|
|
3708
|
+
}
|
|
3709
|
+
function createDataQualityObservabilityTargetPack() {
|
|
3710
|
+
return defineTargetPack({
|
|
3711
|
+
id: "data-quality-observability",
|
|
3712
|
+
name: "Data quality and Databricks Apps observability",
|
|
3713
|
+
description: "Unity Catalog Data Quality Monitoring plus Databricks Apps OpenTelemetry logs, spans and metrics.",
|
|
3714
|
+
version: "1.0.0",
|
|
3715
|
+
maturity: "shipped",
|
|
3716
|
+
capabilities: [
|
|
3717
|
+
"data-quality.freshness-completeness",
|
|
3718
|
+
"apps.telemetry.logs",
|
|
3719
|
+
"apps.telemetry.traces",
|
|
3720
|
+
"apps.telemetry.metrics"
|
|
3721
|
+
],
|
|
3722
|
+
checks: () => [restrictedPrincipalCheck(), dataQualityAppTelemetryCheck()],
|
|
3723
|
+
requiredChecks: ["restricted-principal", "data-quality-app-telemetry"],
|
|
3724
|
+
requirements: [
|
|
3725
|
+
workspace,
|
|
3726
|
+
auth,
|
|
3727
|
+
warehouse,
|
|
3728
|
+
requirement("telemetry-app", "App with UC telemetry export", "DBX_TEST_TELEMETRY_APP_NAME"),
|
|
3729
|
+
requirement(
|
|
3730
|
+
"data-quality",
|
|
3731
|
+
"Boolean SQL over current Data Quality Monitoring results",
|
|
3732
|
+
"DBX_TEST_DQM_ASSERTION_SQL"
|
|
3733
|
+
),
|
|
3734
|
+
requirement(
|
|
3735
|
+
"app-logs",
|
|
3736
|
+
"Boolean SQL proving current app logs",
|
|
3737
|
+
"DBX_TEST_APP_LOGS_ASSERTION_SQL"
|
|
3738
|
+
),
|
|
3739
|
+
requirement(
|
|
3740
|
+
"app-spans",
|
|
3741
|
+
"Boolean SQL proving current app spans",
|
|
3742
|
+
"DBX_TEST_APP_SPANS_ASSERTION_SQL"
|
|
3743
|
+
),
|
|
3744
|
+
requirement(
|
|
3745
|
+
"app-metrics",
|
|
3746
|
+
"Boolean SQL proving current app metrics",
|
|
3747
|
+
"DBX_TEST_APP_METRICS_ASSERTION_SQL"
|
|
3748
|
+
)
|
|
3749
|
+
],
|
|
3750
|
+
docsUrl: `${DOCS2}#data-quality-and-databricks-apps-observability`,
|
|
3751
|
+
certificationScopes: [
|
|
3752
|
+
`${AZURE_M2M_SCOPE}; system.data_quality_monitoring.table_results and UC OTel tables`
|
|
3753
|
+
]
|
|
3754
|
+
});
|
|
3755
|
+
}
|
|
3756
|
+
function createAgentServicesEnrollmentTargetPack() {
|
|
3757
|
+
return defineTargetPack({
|
|
3758
|
+
id: "agent-services-enrollment",
|
|
3759
|
+
name: "Unity Catalog Agent Services enrollment",
|
|
3760
|
+
description: "Customer-owned Fabric Harness agents registered as external Unity Catalog Agent Services with explicit EXECUTE grants.",
|
|
3761
|
+
version: "1.0.0",
|
|
3762
|
+
maturity: "shipped",
|
|
3763
|
+
capabilities: [
|
|
3764
|
+
"agent-services.external-registration",
|
|
3765
|
+
"agent-services.discovery",
|
|
3766
|
+
"agent-services.permissions",
|
|
3767
|
+
"harness.customer-worker-enrollment"
|
|
3768
|
+
],
|
|
3769
|
+
checks: () => [restrictedPrincipalCheck(), agentServicesEnrollmentCheck()],
|
|
3770
|
+
requiredChecks: ["restricted-principal", "agent-services-enrollment"],
|
|
3771
|
+
requirements: [
|
|
3772
|
+
workspace,
|
|
3773
|
+
auth,
|
|
3774
|
+
requirement(
|
|
3775
|
+
"agent-service",
|
|
3776
|
+
"External Unity Catalog Agent Service full name",
|
|
3777
|
+
"DBX_TEST_AGENT_SERVICE_FULL_NAME"
|
|
3778
|
+
),
|
|
3779
|
+
requirement(
|
|
3780
|
+
"agent-connection",
|
|
3781
|
+
"Expected Unity Catalog HTTP connection",
|
|
3782
|
+
"DBX_TEST_AGENT_SERVICE_CONNECTION"
|
|
3783
|
+
),
|
|
3784
|
+
requirement(
|
|
3785
|
+
"agent-execute-principal",
|
|
3786
|
+
"Principal expected to have EXECUTE",
|
|
3787
|
+
"DBX_TEST_AGENT_SERVICE_EXECUTE_PRINCIPAL"
|
|
3788
|
+
)
|
|
3789
|
+
],
|
|
3790
|
+
docsUrl: `${DOCS2}#unity-catalog-agent-services-enrollment`,
|
|
3791
|
+
certificationScopes: [
|
|
3792
|
+
`${AZURE_M2M_SCOPE}; Agent Services Beta registration/discovery/ACL only; runtime invocation is not claimed`
|
|
3793
|
+
]
|
|
3794
|
+
});
|
|
3795
|
+
}
|
|
3342
3796
|
function createAdvancedTargetPacks() {
|
|
3343
3797
|
return [
|
|
3344
3798
|
createAdvancedStreamingTargetPack(),
|
|
@@ -3348,7 +3802,12 @@ function createAdvancedTargetPacks() {
|
|
|
3348
3802
|
createModelServingTargetPack(),
|
|
3349
3803
|
createVectorRagAgentsTargetPack(),
|
|
3350
3804
|
createFederationSharingTargetPack(),
|
|
3351
|
-
createSecurityCostDrTargetPack()
|
|
3805
|
+
createSecurityCostDrTargetPack(),
|
|
3806
|
+
createMlflow3AgentQualityTargetPack(),
|
|
3807
|
+
createManagedMcpAgentsTargetPack(),
|
|
3808
|
+
createDataQualityObservabilityTargetPack(),
|
|
3809
|
+
createAgentServicesEnrollmentTargetPack(),
|
|
3810
|
+
createAgentEvaluationPipelineTargetPack()
|
|
3352
3811
|
];
|
|
3353
3812
|
}
|
|
3354
3813
|
|
|
@@ -3368,6 +3827,8 @@ init_workspace_client();
|
|
|
3368
3827
|
exports.DatabricksAdvancedWorkloadsAdapter = DatabricksAdvancedWorkloadsAdapter;
|
|
3369
3828
|
exports.DatabricksJobsAdapter = DatabricksJobsAdapter;
|
|
3370
3829
|
exports.advancedStreamingCdcCheck = advancedStreamingCdcCheck;
|
|
3830
|
+
exports.agentEvaluationPipelineCheck = agentEvaluationPipelineCheck;
|
|
3831
|
+
exports.agentServicesEnrollmentCheck = agentServicesEnrollmentCheck;
|
|
3371
3832
|
exports.aiBiDashboardCheck = aiBiDashboardCheck;
|
|
3372
3833
|
exports.appHealthCheck = appHealthCheck;
|
|
3373
3834
|
exports.assertDeltaContract = assertDeltaContract;
|
|
@@ -3378,10 +3839,13 @@ exports.classicComputeCheck = classicComputeCheck;
|
|
|
3378
3839
|
exports.compareToGolden = compareToGolden;
|
|
3379
3840
|
exports.createAdvancedStreamingTargetPack = createAdvancedStreamingTargetPack;
|
|
3380
3841
|
exports.createAdvancedTargetPacks = createAdvancedTargetPacks;
|
|
3842
|
+
exports.createAgentEvaluationPipelineTargetPack = createAgentEvaluationPipelineTargetPack;
|
|
3843
|
+
exports.createAgentServicesEnrollmentTargetPack = createAgentServicesEnrollmentTargetPack;
|
|
3381
3844
|
exports.createAiBiGenieTargetPack = createAiBiGenieTargetPack;
|
|
3382
3845
|
exports.createAppsOperationalFoundationPack = createAppsOperationalFoundationPack;
|
|
3383
3846
|
exports.createBuiltinTargetPacks = createBuiltinTargetPacks;
|
|
3384
3847
|
exports.createClientFromEnv = createClientFromEnv;
|
|
3848
|
+
exports.createDataQualityObservabilityTargetPack = createDataQualityObservabilityTargetPack;
|
|
3385
3849
|
exports.createDuckDbContext = createDuckDbContext;
|
|
3386
3850
|
exports.createEphemeralLakebaseBranch = createEphemeralLakebaseBranch;
|
|
3387
3851
|
exports.createExecutionContext = createExecutionContext;
|
|
@@ -3392,6 +3856,8 @@ exports.createJobsCodeFoundationPack = createJobsCodeFoundationPack;
|
|
|
3392
3856
|
exports.createLakebaseTestProvider = createLakebaseTestProvider;
|
|
3393
3857
|
exports.createLakeflowFoundationPack = createLakeflowFoundationPack;
|
|
3394
3858
|
exports.createLakeflowJobsTargetPack = createLakeflowJobsTargetPack;
|
|
3859
|
+
exports.createManagedMcpAgentsTargetPack = createManagedMcpAgentsTargetPack;
|
|
3860
|
+
exports.createMlflow3AgentQualityTargetPack = createMlflow3AgentQualityTargetPack;
|
|
3395
3861
|
exports.createMlflowLifecycleTargetPack = createMlflowLifecycleTargetPack;
|
|
3396
3862
|
exports.createMockDatabricksClient = createMockDatabricksClient;
|
|
3397
3863
|
exports.createModelServingTargetPack = createModelServingTargetPack;
|
|
@@ -3402,6 +3868,7 @@ exports.createUnityStorageFoundationPack = createUnityStorageFoundationPack;
|
|
|
3402
3868
|
exports.createVectorRagAgentsTargetPack = createVectorRagAgentsTargetPack;
|
|
3403
3869
|
exports.createWorkspaceContext = createWorkspaceContext;
|
|
3404
3870
|
exports.customLiveCheck = customLiveCheck;
|
|
3871
|
+
exports.dataQualityAppTelemetryCheck = dataQualityAppTelemetryCheck;
|
|
3405
3872
|
exports.dbtBuildCheck = dbtBuildCheck;
|
|
3406
3873
|
exports.defineLiveSuite = defineLiveSuite;
|
|
3407
3874
|
exports.defineTargetPack = defineTargetPack;
|
|
@@ -3425,6 +3892,8 @@ exports.jobsOrchestrationCheck = jobsOrchestrationCheck;
|
|
|
3425
3892
|
exports.lakebaseCredentialCheck = lakebaseCredentialCheck;
|
|
3426
3893
|
exports.lakebaseRoundTripCheck = lakebaseRoundTripCheck;
|
|
3427
3894
|
exports.lakeflowConnectCheck = lakeflowConnectCheck;
|
|
3895
|
+
exports.managedMcpAgentsCheck = managedMcpAgentsCheck;
|
|
3896
|
+
exports.mlflow3GenAiQualityCheck = mlflow3GenAiQualityCheck;
|
|
3428
3897
|
exports.mlflowLifecycleCheck = mlflowLifecycleCheck;
|
|
3429
3898
|
exports.modelServingCheck = modelServingCheck;
|
|
3430
3899
|
exports.normalizeJobRun = normalizeJobRun;
|