@fabricorg/databricks-testkit 0.7.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +478 -9
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +37 -1
- package/dist/index.d.ts +37 -1
- package/dist/index.js +468 -10
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1958,6 +1958,15 @@ var DatabricksAdvancedWorkloadsAdapter = class {
|
|
|
1958
1958
|
warehouse(id) {
|
|
1959
1959
|
return this.client.get(`/api/2.0/sql/warehouses/${segment(id)}`);
|
|
1960
1960
|
}
|
|
1961
|
+
app(name) {
|
|
1962
|
+
return this.client.get(`/api/2.0/apps/${segment(name)}`);
|
|
1963
|
+
}
|
|
1964
|
+
agentService(fullName) {
|
|
1965
|
+
return this.client.get(`/api/2.1/unity-catalog/agent-services/${segment(fullName)}`);
|
|
1966
|
+
}
|
|
1967
|
+
agentServicePermissions(fullName) {
|
|
1968
|
+
return this.client.get(`/api/2.1/unity-catalog/permissions/AGENT_SERVICE/${segment(fullName)}`);
|
|
1969
|
+
}
|
|
1961
1970
|
};
|
|
1962
1971
|
function segment(value) {
|
|
1963
1972
|
return encodeURIComponent(value);
|
|
@@ -2004,6 +2013,27 @@ async function booleanSql(label, sql, query) {
|
|
|
2004
2013
|
throw new Error(`${label} assertion did not return a truthy first column`);
|
|
2005
2014
|
}
|
|
2006
2015
|
}
|
|
2016
|
+
async function runCertificationJob(label, jobIdValue, env, client, options = {}) {
|
|
2017
|
+
const jobId = Number(jobIdValue);
|
|
2018
|
+
if (!Number.isSafeInteger(jobId) || jobId <= 0) {
|
|
2019
|
+
throw new Error(`${label} job id must be a positive integer`);
|
|
2020
|
+
}
|
|
2021
|
+
const submitted = await client.post("/api/2.1/jobs/run-now", {
|
|
2022
|
+
job_id: jobId
|
|
2023
|
+
});
|
|
2024
|
+
if (!Number.isSafeInteger(submitted.run_id)) {
|
|
2025
|
+
throw new Error(`${label} certification job returned no run_id`);
|
|
2026
|
+
}
|
|
2027
|
+
const run = await waitForDatabricksRun(client, submitted.run_id, {
|
|
2028
|
+
timeoutMs: options.timeoutMs ?? Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
|
|
2029
|
+
pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
|
|
2030
|
+
});
|
|
2031
|
+
const result = run.state?.result_state;
|
|
2032
|
+
if (result !== "SUCCESS") {
|
|
2033
|
+
throw new Error(`${label} certification job ${jobId} ended in ${result ?? "UNKNOWN"}`);
|
|
2034
|
+
}
|
|
2035
|
+
return { jobId, runId: submitted.run_id };
|
|
2036
|
+
}
|
|
2007
2037
|
function advancedStreamingCdcCheck() {
|
|
2008
2038
|
const names = [
|
|
2009
2039
|
"DBX_TEST_STREAMING_PIPELINE_ID",
|
|
@@ -2383,6 +2413,201 @@ function securityPostureCostCheck() {
|
|
|
2383
2413
|
}
|
|
2384
2414
|
};
|
|
2385
2415
|
}
|
|
2416
|
+
function mlflow3GenAiQualityCheck() {
|
|
2417
|
+
const names = [
|
|
2418
|
+
"DBX_TEST_MLFLOW3_JOB_ID",
|
|
2419
|
+
"DBX_TEST_MLFLOW3_TRACE_ASSERTION_SQL",
|
|
2420
|
+
"DBX_TEST_MLFLOW3_EVAL_ASSERTION_SQL",
|
|
2421
|
+
"DBX_TEST_MLFLOW3_SCORER_ASSERTION_SQL",
|
|
2422
|
+
"DBX_TEST_MLFLOW3_MONITOR_ASSERTION_SQL",
|
|
2423
|
+
"DBX_TEST_AI_SEARCH_EVAL_ASSERTION_SQL"
|
|
2424
|
+
];
|
|
2425
|
+
return {
|
|
2426
|
+
id: "mlflow3-genai-quality",
|
|
2427
|
+
description: "MLflow 3 tracing, offline agent evaluation and production scorer monitoring produce durable evidence",
|
|
2428
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2429
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
2430
|
+
requireEnv(env, names);
|
|
2431
|
+
const job = await runCertificationJob(
|
|
2432
|
+
"MLflow 3 GenAI",
|
|
2433
|
+
env.DBX_TEST_MLFLOW3_JOB_ID,
|
|
2434
|
+
env,
|
|
2435
|
+
client,
|
|
2436
|
+
{ timeoutMs: Number(env.DBX_TEST_MLFLOW3_TIMEOUT_MS ?? 72e5) }
|
|
2437
|
+
);
|
|
2438
|
+
await booleanSql("MLflow 3 trace", env.DBX_TEST_MLFLOW3_TRACE_ASSERTION_SQL, warehouse2.query);
|
|
2439
|
+
await booleanSql(
|
|
2440
|
+
"MLflow 3 offline evaluation",
|
|
2441
|
+
env.DBX_TEST_MLFLOW3_EVAL_ASSERTION_SQL,
|
|
2442
|
+
warehouse2.query
|
|
2443
|
+
);
|
|
2444
|
+
await booleanSql(
|
|
2445
|
+
"MLflow 3 production scorer lifecycle",
|
|
2446
|
+
env.DBX_TEST_MLFLOW3_SCORER_ASSERTION_SQL,
|
|
2447
|
+
warehouse2.query
|
|
2448
|
+
);
|
|
2449
|
+
await booleanSql(
|
|
2450
|
+
"MLflow 3 production monitoring",
|
|
2451
|
+
env.DBX_TEST_MLFLOW3_MONITOR_ASSERTION_SQL,
|
|
2452
|
+
warehouse2.query
|
|
2453
|
+
);
|
|
2454
|
+
await booleanSql(
|
|
2455
|
+
"AI Search retrieval quality",
|
|
2456
|
+
env.DBX_TEST_AI_SEARCH_EVAL_ASSERTION_SQL,
|
|
2457
|
+
warehouse2.query
|
|
2458
|
+
);
|
|
2459
|
+
return {
|
|
2460
|
+
...job,
|
|
2461
|
+
traces: true,
|
|
2462
|
+
offlineEvaluation: true,
|
|
2463
|
+
scorerLifecycle: true,
|
|
2464
|
+
monitor: true,
|
|
2465
|
+
aiSearchEvaluation: true
|
|
2466
|
+
};
|
|
2467
|
+
}
|
|
2468
|
+
};
|
|
2469
|
+
}
|
|
2470
|
+
function agentEvaluationPipelineCheck() {
|
|
2471
|
+
const names = ["DBX_TEST_AGENT_EVAL_E2E_JOB_ID", "DBX_TEST_AGENT_EVAL_ASSERTION_SQL"];
|
|
2472
|
+
return {
|
|
2473
|
+
id: "agent-evaluation-pipeline",
|
|
2474
|
+
description: "Experiments API, outbox, Temporal worker, Model Serving judge, SQL score sink and Quality completion execute end to end",
|
|
2475
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2476
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
2477
|
+
requireEnv(env, names);
|
|
2478
|
+
const job = await runCertificationJob(
|
|
2479
|
+
"Agent evaluation end-to-end",
|
|
2480
|
+
env.DBX_TEST_AGENT_EVAL_E2E_JOB_ID,
|
|
2481
|
+
env,
|
|
2482
|
+
client
|
|
2483
|
+
);
|
|
2484
|
+
await booleanSql(
|
|
2485
|
+
"Agent evaluation terminal Quality evidence",
|
|
2486
|
+
env.DBX_TEST_AGENT_EVAL_ASSERTION_SQL,
|
|
2487
|
+
warehouse2.query
|
|
2488
|
+
);
|
|
2489
|
+
return { ...job, api: true, outbox: true, temporal: true, judged: true, persisted: true };
|
|
2490
|
+
}
|
|
2491
|
+
};
|
|
2492
|
+
}
|
|
2493
|
+
function managedMcpAgentsCheck() {
|
|
2494
|
+
const names = [
|
|
2495
|
+
"DBX_TEST_MCP_JOB_ID",
|
|
2496
|
+
"DBX_TEST_MCP_MANAGED_ASSERTION_SQL",
|
|
2497
|
+
"DBX_TEST_MCP_SERVICE_ASSERTION_SQL",
|
|
2498
|
+
"DBX_TEST_MCP_DENY_ASSERTION_SQL"
|
|
2499
|
+
];
|
|
2500
|
+
return {
|
|
2501
|
+
id: "managed-mcp-agents",
|
|
2502
|
+
description: "Managed MCP and Unity Catalog MCP Services support list, call and deny behavior",
|
|
2503
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2504
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
2505
|
+
requireEnv(env, names);
|
|
2506
|
+
const job = await runCertificationJob("Databricks MCP", env.DBX_TEST_MCP_JOB_ID, env, client);
|
|
2507
|
+
await booleanSql(
|
|
2508
|
+
"Managed MCP list/call",
|
|
2509
|
+
env.DBX_TEST_MCP_MANAGED_ASSERTION_SQL,
|
|
2510
|
+
warehouse2.query
|
|
2511
|
+
);
|
|
2512
|
+
await booleanSql(
|
|
2513
|
+
"Unity Catalog MCP Service list/call",
|
|
2514
|
+
env.DBX_TEST_MCP_SERVICE_ASSERTION_SQL,
|
|
2515
|
+
warehouse2.query
|
|
2516
|
+
);
|
|
2517
|
+
await booleanSql(
|
|
2518
|
+
"MCP excluded-tool denial",
|
|
2519
|
+
env.DBX_TEST_MCP_DENY_ASSERTION_SQL,
|
|
2520
|
+
warehouse2.query
|
|
2521
|
+
);
|
|
2522
|
+
return { ...job, managedServer: true, mcpService: true, denyControl: true };
|
|
2523
|
+
}
|
|
2524
|
+
};
|
|
2525
|
+
}
|
|
2526
|
+
function dataQualityAppTelemetryCheck() {
|
|
2527
|
+
const names = [
|
|
2528
|
+
"DBX_TEST_TELEMETRY_APP_NAME",
|
|
2529
|
+
"DBX_TEST_DQM_ASSERTION_SQL",
|
|
2530
|
+
"DBX_TEST_APP_LOGS_ASSERTION_SQL",
|
|
2531
|
+
"DBX_TEST_APP_SPANS_ASSERTION_SQL",
|
|
2532
|
+
"DBX_TEST_APP_METRICS_ASSERTION_SQL"
|
|
2533
|
+
];
|
|
2534
|
+
return {
|
|
2535
|
+
id: "data-quality-app-telemetry",
|
|
2536
|
+
description: "Unity Catalog data quality monitoring and Databricks Apps logs, spans and metrics are current",
|
|
2537
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2538
|
+
async run({ env, client, warehouse: warehouse2 }) {
|
|
2539
|
+
requireEnv(env, names);
|
|
2540
|
+
const appName = env.DBX_TEST_TELEMETRY_APP_NAME;
|
|
2541
|
+
const app = await adapter(client).app(appName);
|
|
2542
|
+
const destinations = Array.isArray(app.telemetry_export_destinations) ? app.telemetry_export_destinations : [];
|
|
2543
|
+
const unityCatalog = destinations.find(
|
|
2544
|
+
(destination) => isObject(destination) && isObject(destination.unity_catalog)
|
|
2545
|
+
);
|
|
2546
|
+
if (!isObject(unityCatalog) || !isObject(unityCatalog.unity_catalog)) {
|
|
2547
|
+
throw new Error(`Databricks App ${appName} has no Unity Catalog telemetry destination`);
|
|
2548
|
+
}
|
|
2549
|
+
const tables = unityCatalog.unity_catalog;
|
|
2550
|
+
for (const field of ["logs_table", "traces_table", "metrics_table"]) {
|
|
2551
|
+
if (typeof tables[field] !== "string" || !tables[field]) {
|
|
2552
|
+
throw new Error(`Databricks App ${appName} telemetry destination has no ${field}`);
|
|
2553
|
+
}
|
|
2554
|
+
}
|
|
2555
|
+
await booleanSql("Data Quality Monitoring", env.DBX_TEST_DQM_ASSERTION_SQL, warehouse2.query);
|
|
2556
|
+
await booleanSql("App logs", env.DBX_TEST_APP_LOGS_ASSERTION_SQL, warehouse2.query);
|
|
2557
|
+
await booleanSql("App spans", env.DBX_TEST_APP_SPANS_ASSERTION_SQL, warehouse2.query);
|
|
2558
|
+
await booleanSql("App metrics", env.DBX_TEST_APP_METRICS_ASSERTION_SQL, warehouse2.query);
|
|
2559
|
+
return { app: appName, unityCatalogTelemetry: true, dataQualityMonitoring: true };
|
|
2560
|
+
}
|
|
2561
|
+
};
|
|
2562
|
+
}
|
|
2563
|
+
function agentServicesEnrollmentCheck() {
|
|
2564
|
+
const names = [
|
|
2565
|
+
"DBX_TEST_AGENT_SERVICE_FULL_NAME",
|
|
2566
|
+
"DBX_TEST_AGENT_SERVICE_CONNECTION",
|
|
2567
|
+
"DBX_TEST_AGENT_SERVICE_EXECUTE_PRINCIPAL"
|
|
2568
|
+
];
|
|
2569
|
+
return {
|
|
2570
|
+
id: "agent-services-enrollment",
|
|
2571
|
+
description: "A customer-owned Harness agent is registered as an external Unity Catalog Agent Service with governed access",
|
|
2572
|
+
configured: (env) => required2(env, names).length === 0,
|
|
2573
|
+
async run({ env, client }) {
|
|
2574
|
+
requireEnv(env, names);
|
|
2575
|
+
const fullName = env.DBX_TEST_AGENT_SERVICE_FULL_NAME;
|
|
2576
|
+
const api = adapter(client);
|
|
2577
|
+
const service = await api.agentService(fullName);
|
|
2578
|
+
if (service.agent_service_type !== "AGENT_SERVICE_TYPE_EXTERNAL") {
|
|
2579
|
+
throw new Error(`${fullName} is not an external Agent Service`);
|
|
2580
|
+
}
|
|
2581
|
+
const expectedConnection = env.DBX_TEST_AGENT_SERVICE_CONNECTION;
|
|
2582
|
+
const actualConnection = service.config?.connection?.name ?? "";
|
|
2583
|
+
if (actualConnection !== expectedConnection && actualConnection.split("/").at(-1) !== expectedConnection) {
|
|
2584
|
+
throw new Error(`${fullName} uses unexpected connection ${actualConnection || "UNKNOWN"}`);
|
|
2585
|
+
}
|
|
2586
|
+
if (!service.config?.base_path?.startsWith("/")) {
|
|
2587
|
+
throw new Error(`${fullName} has no absolute external base path`);
|
|
2588
|
+
}
|
|
2589
|
+
const permissions = await api.agentServicePermissions(fullName);
|
|
2590
|
+
const assignments = Array.isArray(permissions.privilege_assignments) ? permissions.privilege_assignments : [];
|
|
2591
|
+
const expectedPrincipal = env.DBX_TEST_AGENT_SERVICE_EXECUTE_PRINCIPAL;
|
|
2592
|
+
const assignment = assignments.find(
|
|
2593
|
+
(entry) => isObject(entry) && entry.principal === expectedPrincipal
|
|
2594
|
+
);
|
|
2595
|
+
const privileges = isObject(assignment) && Array.isArray(assignment.privileges) ? assignment.privileges : [];
|
|
2596
|
+
const canExecute = privileges.some(
|
|
2597
|
+
(privilege) => isObject(privilege) && privilege.privilege === "EXECUTE"
|
|
2598
|
+
);
|
|
2599
|
+
if (!canExecute) {
|
|
2600
|
+
throw new Error(`${expectedPrincipal} does not have EXECUTE on ${fullName}`);
|
|
2601
|
+
}
|
|
2602
|
+
return {
|
|
2603
|
+
fullName: service.full_name ?? service.name ?? fullName,
|
|
2604
|
+
connection: actualConnection,
|
|
2605
|
+
executePrincipal: expectedPrincipal,
|
|
2606
|
+
runtimeInvocationAvailable: false
|
|
2607
|
+
};
|
|
2608
|
+
}
|
|
2609
|
+
};
|
|
2610
|
+
}
|
|
2386
2611
|
function regionalDrCheck(options = {}) {
|
|
2387
2612
|
const names = [
|
|
2388
2613
|
"DBX_TEST_DR_HOST",
|
|
@@ -2467,7 +2692,7 @@ function createAdvancedStreamingTargetPack() {
|
|
|
2467
2692
|
"lakeflow-connect.replication"
|
|
2468
2693
|
],
|
|
2469
2694
|
checks: () => [restrictedPrincipalCheck(), advancedStreamingCdcCheck(), lakeflowConnectCheck()],
|
|
2470
|
-
requiredChecks: ["streaming-cdc", "lakeflow-connect"],
|
|
2695
|
+
requiredChecks: ["restricted-principal", "streaming-cdc", "lakeflow-connect"],
|
|
2471
2696
|
requirements: [
|
|
2472
2697
|
workspace,
|
|
2473
2698
|
auth,
|
|
@@ -2506,7 +2731,7 @@ function createAiBiGenieTargetPack() {
|
|
|
2506
2731
|
maturity: "live-gated",
|
|
2507
2732
|
capabilities: ["aibi.dashboard", "genie.space", "genie.conversation"],
|
|
2508
2733
|
checks: () => [restrictedPrincipalCheck(), aiBiDashboardCheck(), genieCheck()],
|
|
2509
|
-
requiredChecks: ["aibi-dashboard", "genie"],
|
|
2734
|
+
requiredChecks: ["restricted-principal", "aibi-dashboard", "genie"],
|
|
2510
2735
|
requirements: [
|
|
2511
2736
|
workspace,
|
|
2512
2737
|
auth,
|
|
@@ -2538,7 +2763,7 @@ function createMlflowLifecycleTargetPack() {
|
|
|
2538
2763
|
"models.lifecycle"
|
|
2539
2764
|
],
|
|
2540
2765
|
checks: () => [restrictedPrincipalCheck(), mlflowLifecycleCheck()],
|
|
2541
|
-
requiredChecks: ["mlflow-lifecycle"],
|
|
2766
|
+
requiredChecks: ["restricted-principal", "mlflow-lifecycle"],
|
|
2542
2767
|
requirements: [
|
|
2543
2768
|
workspace,
|
|
2544
2769
|
auth,
|
|
@@ -2570,7 +2795,7 @@ function createFeatureEngineeringTargetPack() {
|
|
|
2570
2795
|
"features.serving"
|
|
2571
2796
|
],
|
|
2572
2797
|
checks: () => [restrictedPrincipalCheck(), featureEngineeringServingCheck()],
|
|
2573
|
-
requiredChecks: ["feature-engineering"],
|
|
2798
|
+
requiredChecks: ["restricted-principal", "feature-engineering"],
|
|
2574
2799
|
requirements: [
|
|
2575
2800
|
workspace,
|
|
2576
2801
|
auth,
|
|
@@ -2607,7 +2832,7 @@ function createModelServingTargetPack() {
|
|
|
2607
2832
|
"ai-gateway.configuration"
|
|
2608
2833
|
],
|
|
2609
2834
|
checks: () => [restrictedPrincipalCheck(), modelServingCheck()],
|
|
2610
|
-
requiredChecks: ["model-serving"],
|
|
2835
|
+
requiredChecks: ["restricted-principal", "model-serving"],
|
|
2611
2836
|
requirements: [
|
|
2612
2837
|
workspace,
|
|
2613
2838
|
auth,
|
|
@@ -2643,7 +2868,7 @@ function createVectorRagAgentsTargetPack() {
|
|
|
2643
2868
|
"agents.serving"
|
|
2644
2869
|
],
|
|
2645
2870
|
checks: () => [restrictedPrincipalCheck(), vectorSearchCheck(), ragAgentCheck()],
|
|
2646
|
-
requiredChecks: ["vector-search", "rag-agent"],
|
|
2871
|
+
requiredChecks: ["restricted-principal", "vector-search", "rag-agent"],
|
|
2647
2872
|
requirements: [
|
|
2648
2873
|
workspace,
|
|
2649
2874
|
auth,
|
|
@@ -2673,7 +2898,7 @@ function createFederationSharingTargetPack() {
|
|
|
2673
2898
|
"clean-rooms.access"
|
|
2674
2899
|
],
|
|
2675
2900
|
checks: () => [restrictedPrincipalCheck(), federationSharingCleanRoomsCheck()],
|
|
2676
|
-
requiredChecks: ["federation-sharing-cleanrooms"],
|
|
2901
|
+
requiredChecks: ["restricted-principal", "federation-sharing-cleanrooms"],
|
|
2677
2902
|
requirements: [
|
|
2678
2903
|
workspace,
|
|
2679
2904
|
auth,
|
|
@@ -2707,7 +2932,7 @@ function createSecurityCostDrTargetPack() {
|
|
|
2707
2932
|
"dr.secondary-region"
|
|
2708
2933
|
],
|
|
2709
2934
|
checks: () => [restrictedPrincipalCheck(), securityPostureCostCheck(), regionalDrCheck()],
|
|
2710
|
-
requiredChecks: ["security-cost", "regional-dr"],
|
|
2935
|
+
requiredChecks: ["restricted-principal", "security-cost", "regional-dr"],
|
|
2711
2936
|
requirements: [
|
|
2712
2937
|
workspace,
|
|
2713
2938
|
auth,
|
|
@@ -2729,6 +2954,234 @@ function createSecurityCostDrTargetPack() {
|
|
|
2729
2954
|
]
|
|
2730
2955
|
});
|
|
2731
2956
|
}
|
|
2957
|
+
function createMlflow3AgentQualityTargetPack() {
|
|
2958
|
+
return defineTargetPack({
|
|
2959
|
+
id: "mlflow3-agent-quality",
|
|
2960
|
+
name: "MLflow 3 agent quality and monitoring",
|
|
2961
|
+
description: "Native MLflow 3 traces, offline agent evaluation, production scorer lifecycle and online monitoring.",
|
|
2962
|
+
version: "1.0.0",
|
|
2963
|
+
maturity: "shipped",
|
|
2964
|
+
capabilities: [
|
|
2965
|
+
"mlflow3.tracing",
|
|
2966
|
+
"mlflow3.agent-evaluation",
|
|
2967
|
+
"mlflow3.production-scorers",
|
|
2968
|
+
"mlflow3.production-monitoring",
|
|
2969
|
+
"ai-search.retrieval-evaluation"
|
|
2970
|
+
],
|
|
2971
|
+
checks: () => [restrictedPrincipalCheck(), mlflow3GenAiQualityCheck()],
|
|
2972
|
+
requiredChecks: ["restricted-principal", "mlflow3-genai-quality"],
|
|
2973
|
+
requirements: [
|
|
2974
|
+
workspace,
|
|
2975
|
+
auth,
|
|
2976
|
+
warehouse,
|
|
2977
|
+
requirement("mlflow3-job", "Native MLflow 3 certification Job", "DBX_TEST_MLFLOW3_JOB_ID"),
|
|
2978
|
+
requirement(
|
|
2979
|
+
"mlflow3-traces",
|
|
2980
|
+
"Boolean SQL proving a fresh MLflow 3 trace",
|
|
2981
|
+
"DBX_TEST_MLFLOW3_TRACE_ASSERTION_SQL"
|
|
2982
|
+
),
|
|
2983
|
+
requirement(
|
|
2984
|
+
"mlflow3-evaluation",
|
|
2985
|
+
"Boolean SQL proving offline agent evaluation results",
|
|
2986
|
+
"DBX_TEST_MLFLOW3_EVAL_ASSERTION_SQL"
|
|
2987
|
+
),
|
|
2988
|
+
requirement(
|
|
2989
|
+
"mlflow3-scorers",
|
|
2990
|
+
"Boolean SQL proving production scorer lifecycle evidence",
|
|
2991
|
+
"DBX_TEST_MLFLOW3_SCORER_ASSERTION_SQL"
|
|
2992
|
+
),
|
|
2993
|
+
requirement(
|
|
2994
|
+
"mlflow3-monitoring",
|
|
2995
|
+
"Boolean SQL proving production monitoring output",
|
|
2996
|
+
"DBX_TEST_MLFLOW3_MONITOR_ASSERTION_SQL"
|
|
2997
|
+
),
|
|
2998
|
+
requirement(
|
|
2999
|
+
"ai-search-evaluation",
|
|
3000
|
+
"Boolean SQL proving AI Search retrieval-quality thresholds",
|
|
3001
|
+
"DBX_TEST_AI_SEARCH_EVAL_ASSERTION_SQL"
|
|
3002
|
+
)
|
|
3003
|
+
],
|
|
3004
|
+
docsUrl: `${DOCS2}#mlflow-3-agent-quality-and-monitoring`,
|
|
3005
|
+
certificationScopes: [
|
|
3006
|
+
`${AZURE_M2M_SCOPE}; Python-native MLflow 3 trace, evaluation, complete scorer lifecycle and monitor evidence`
|
|
3007
|
+
]
|
|
3008
|
+
});
|
|
3009
|
+
}
|
|
3010
|
+
function createAgentEvaluationPipelineTargetPack() {
|
|
3011
|
+
return defineTargetPack({
|
|
3012
|
+
id: "agent-evaluation-pipeline",
|
|
3013
|
+
name: "Agent evaluation delivery pipeline",
|
|
3014
|
+
description: "Exact-candidate API submission, outbox dispatch, Temporal execution, Databricks judge inference, SQL score persistence and Quality completion.",
|
|
3015
|
+
version: "1.0.0",
|
|
3016
|
+
maturity: "live-gated",
|
|
3017
|
+
capabilities: [
|
|
3018
|
+
"agent-evals.api-submission",
|
|
3019
|
+
"agent-evals.temporal-dispatch",
|
|
3020
|
+
"agent-evals.model-serving-judge",
|
|
3021
|
+
"agent-evals.sql-persistence",
|
|
3022
|
+
"agent-evals.quality-evidence"
|
|
3023
|
+
],
|
|
3024
|
+
checks: () => [
|
|
3025
|
+
restrictedPrincipalCheck(),
|
|
3026
|
+
evalJudgeModelServingCheck(),
|
|
3027
|
+
agentEvaluationPipelineCheck()
|
|
3028
|
+
],
|
|
3029
|
+
requiredChecks: ["restricted-principal", "model-eval-judge", "agent-evaluation-pipeline"],
|
|
3030
|
+
requirements: [
|
|
3031
|
+
workspace,
|
|
3032
|
+
auth,
|
|
3033
|
+
warehouse,
|
|
3034
|
+
requirement(
|
|
3035
|
+
"eval-serving-endpoint",
|
|
3036
|
+
"Databricks Model Serving endpoint satisfying the evaluator chat contract",
|
|
3037
|
+
"DBX_TEST_EVAL_SERVING_ENDPOINT"
|
|
3038
|
+
),
|
|
3039
|
+
requirement(
|
|
3040
|
+
"agent-eval-job",
|
|
3041
|
+
"Exact-candidate end-to-end agent evaluation certification Job",
|
|
3042
|
+
"DBX_TEST_AGENT_EVAL_E2E_JOB_ID"
|
|
3043
|
+
),
|
|
3044
|
+
requirement(
|
|
3045
|
+
"agent-eval-evidence",
|
|
3046
|
+
"Boolean SQL proving terminal scores and Quality evidence",
|
|
3047
|
+
"DBX_TEST_AGENT_EVAL_ASSERTION_SQL"
|
|
3048
|
+
)
|
|
3049
|
+
],
|
|
3050
|
+
docsUrl: `${DOCS2}#agent-evaluation-delivery-pipeline`,
|
|
3051
|
+
certificationScopes: [
|
|
3052
|
+
`${AZURE_M2M_SCOPE}; Experiments API through customer worker and Databricks judge to SQL/Quality`
|
|
3053
|
+
]
|
|
3054
|
+
});
|
|
3055
|
+
}
|
|
3056
|
+
function createManagedMcpAgentsTargetPack() {
|
|
3057
|
+
return defineTargetPack({
|
|
3058
|
+
id: "managed-mcp-agents",
|
|
3059
|
+
name: "Managed MCP and MCP Services",
|
|
3060
|
+
description: "Streamable HTTP tool discovery and calls for Databricks managed MCP servers and governed Unity Catalog MCP Services, including a deny control.",
|
|
3061
|
+
version: "1.0.0",
|
|
3062
|
+
maturity: "live-gated",
|
|
3063
|
+
capabilities: [
|
|
3064
|
+
"mcp.managed.list-call",
|
|
3065
|
+
"mcp.services.list-call",
|
|
3066
|
+
"mcp.unity-catalog-governance",
|
|
3067
|
+
"mcp.denial"
|
|
3068
|
+
],
|
|
3069
|
+
checks: () => [restrictedPrincipalCheck(), managedMcpAgentsCheck()],
|
|
3070
|
+
requiredChecks: ["restricted-principal", "managed-mcp-agents"],
|
|
3071
|
+
requirements: [
|
|
3072
|
+
workspace,
|
|
3073
|
+
auth,
|
|
3074
|
+
warehouse,
|
|
3075
|
+
requirement("mcp-job", "Native Streamable HTTP MCP certification Job", "DBX_TEST_MCP_JOB_ID"),
|
|
3076
|
+
requirement(
|
|
3077
|
+
"managed-mcp",
|
|
3078
|
+
"Boolean SQL proving managed MCP list/call behavior",
|
|
3079
|
+
"DBX_TEST_MCP_MANAGED_ASSERTION_SQL"
|
|
3080
|
+
),
|
|
3081
|
+
requirement(
|
|
3082
|
+
"mcp-service",
|
|
3083
|
+
"Boolean SQL proving UC MCP Service list/call behavior",
|
|
3084
|
+
"DBX_TEST_MCP_SERVICE_ASSERTION_SQL"
|
|
3085
|
+
),
|
|
3086
|
+
requirement(
|
|
3087
|
+
"mcp-denial",
|
|
3088
|
+
"Boolean SQL proving an MCP tool excluded by UC selectors was denied",
|
|
3089
|
+
"DBX_TEST_MCP_DENY_ASSERTION_SQL"
|
|
3090
|
+
)
|
|
3091
|
+
],
|
|
3092
|
+
docsUrl: `${DOCS2}#managed-mcp-and-mcp-services`,
|
|
3093
|
+
certificationScopes: [
|
|
3094
|
+
`${AZURE_M2M_SCOPE}; managed AI Search or SQL MCP plus a UC MCP Service over Streamable HTTP`
|
|
3095
|
+
]
|
|
3096
|
+
});
|
|
3097
|
+
}
|
|
3098
|
+
function createDataQualityObservabilityTargetPack() {
|
|
3099
|
+
return defineTargetPack({
|
|
3100
|
+
id: "data-quality-observability",
|
|
3101
|
+
name: "Data quality and Databricks Apps observability",
|
|
3102
|
+
description: "Unity Catalog Data Quality Monitoring plus Databricks Apps OpenTelemetry logs, spans and metrics.",
|
|
3103
|
+
version: "1.0.0",
|
|
3104
|
+
maturity: "shipped",
|
|
3105
|
+
capabilities: [
|
|
3106
|
+
"data-quality.freshness-completeness",
|
|
3107
|
+
"apps.telemetry.logs",
|
|
3108
|
+
"apps.telemetry.traces",
|
|
3109
|
+
"apps.telemetry.metrics"
|
|
3110
|
+
],
|
|
3111
|
+
checks: () => [restrictedPrincipalCheck(), dataQualityAppTelemetryCheck()],
|
|
3112
|
+
requiredChecks: ["restricted-principal", "data-quality-app-telemetry"],
|
|
3113
|
+
requirements: [
|
|
3114
|
+
workspace,
|
|
3115
|
+
auth,
|
|
3116
|
+
warehouse,
|
|
3117
|
+
requirement("telemetry-app", "App with UC telemetry export", "DBX_TEST_TELEMETRY_APP_NAME"),
|
|
3118
|
+
requirement(
|
|
3119
|
+
"data-quality",
|
|
3120
|
+
"Boolean SQL over current Data Quality Monitoring results",
|
|
3121
|
+
"DBX_TEST_DQM_ASSERTION_SQL"
|
|
3122
|
+
),
|
|
3123
|
+
requirement(
|
|
3124
|
+
"app-logs",
|
|
3125
|
+
"Boolean SQL proving current app logs",
|
|
3126
|
+
"DBX_TEST_APP_LOGS_ASSERTION_SQL"
|
|
3127
|
+
),
|
|
3128
|
+
requirement(
|
|
3129
|
+
"app-spans",
|
|
3130
|
+
"Boolean SQL proving current app spans",
|
|
3131
|
+
"DBX_TEST_APP_SPANS_ASSERTION_SQL"
|
|
3132
|
+
),
|
|
3133
|
+
requirement(
|
|
3134
|
+
"app-metrics",
|
|
3135
|
+
"Boolean SQL proving current app metrics",
|
|
3136
|
+
"DBX_TEST_APP_METRICS_ASSERTION_SQL"
|
|
3137
|
+
)
|
|
3138
|
+
],
|
|
3139
|
+
docsUrl: `${DOCS2}#data-quality-and-databricks-apps-observability`,
|
|
3140
|
+
certificationScopes: [
|
|
3141
|
+
`${AZURE_M2M_SCOPE}; system.data_quality_monitoring.table_results and UC OTel tables`
|
|
3142
|
+
]
|
|
3143
|
+
});
|
|
3144
|
+
}
|
|
3145
|
+
function createAgentServicesEnrollmentTargetPack() {
|
|
3146
|
+
return defineTargetPack({
|
|
3147
|
+
id: "agent-services-enrollment",
|
|
3148
|
+
name: "Unity Catalog Agent Services enrollment",
|
|
3149
|
+
description: "Customer-owned Fabric Harness agents registered as external Unity Catalog Agent Services with explicit EXECUTE grants.",
|
|
3150
|
+
version: "1.0.0",
|
|
3151
|
+
maturity: "shipped",
|
|
3152
|
+
capabilities: [
|
|
3153
|
+
"agent-services.external-registration",
|
|
3154
|
+
"agent-services.discovery",
|
|
3155
|
+
"agent-services.permissions",
|
|
3156
|
+
"harness.customer-worker-enrollment"
|
|
3157
|
+
],
|
|
3158
|
+
checks: () => [restrictedPrincipalCheck(), agentServicesEnrollmentCheck()],
|
|
3159
|
+
requiredChecks: ["restricted-principal", "agent-services-enrollment"],
|
|
3160
|
+
requirements: [
|
|
3161
|
+
workspace,
|
|
3162
|
+
auth,
|
|
3163
|
+
requirement(
|
|
3164
|
+
"agent-service",
|
|
3165
|
+
"External Unity Catalog Agent Service full name",
|
|
3166
|
+
"DBX_TEST_AGENT_SERVICE_FULL_NAME"
|
|
3167
|
+
),
|
|
3168
|
+
requirement(
|
|
3169
|
+
"agent-connection",
|
|
3170
|
+
"Expected Unity Catalog HTTP connection",
|
|
3171
|
+
"DBX_TEST_AGENT_SERVICE_CONNECTION"
|
|
3172
|
+
),
|
|
3173
|
+
requirement(
|
|
3174
|
+
"agent-execute-principal",
|
|
3175
|
+
"Principal expected to have EXECUTE",
|
|
3176
|
+
"DBX_TEST_AGENT_SERVICE_EXECUTE_PRINCIPAL"
|
|
3177
|
+
)
|
|
3178
|
+
],
|
|
3179
|
+
docsUrl: `${DOCS2}#unity-catalog-agent-services-enrollment`,
|
|
3180
|
+
certificationScopes: [
|
|
3181
|
+
`${AZURE_M2M_SCOPE}; Agent Services Beta registration/discovery/ACL only; runtime invocation is not claimed`
|
|
3182
|
+
]
|
|
3183
|
+
});
|
|
3184
|
+
}
|
|
2732
3185
|
function createAdvancedTargetPacks() {
|
|
2733
3186
|
return [
|
|
2734
3187
|
createAdvancedStreamingTargetPack(),
|
|
@@ -2738,7 +3191,12 @@ function createAdvancedTargetPacks() {
|
|
|
2738
3191
|
createModelServingTargetPack(),
|
|
2739
3192
|
createVectorRagAgentsTargetPack(),
|
|
2740
3193
|
createFederationSharingTargetPack(),
|
|
2741
|
-
createSecurityCostDrTargetPack()
|
|
3194
|
+
createSecurityCostDrTargetPack(),
|
|
3195
|
+
createMlflow3AgentQualityTargetPack(),
|
|
3196
|
+
createManagedMcpAgentsTargetPack(),
|
|
3197
|
+
createDataQualityObservabilityTargetPack(),
|
|
3198
|
+
createAgentServicesEnrollmentTargetPack(),
|
|
3199
|
+
createAgentEvaluationPipelineTargetPack()
|
|
2742
3200
|
];
|
|
2743
3201
|
}
|
|
2744
3202
|
|
|
@@ -2752,6 +3210,6 @@ function createBuiltinTargetPacks() {
|
|
|
2752
3210
|
}
|
|
2753
3211
|
var builtinTargetPacks = createBuiltinTargetPacks();
|
|
2754
3212
|
|
|
2755
|
-
export { DatabricksAdvancedWorkloadsAdapter, DatabricksJobsAdapter, advancedStreamingCdcCheck, aiBiDashboardCheck, appHealthCheck, assertDeltaContract, autoLoaderIngestionCheck, backupRestoreCheck, builtinTargetPacks, classicComputeCheck, createAdvancedStreamingTargetPack, createAdvancedTargetPacks, createAiBiGenieTargetPack, createAppsOperationalFoundationPack, createBuiltinTargetPacks, createEphemeralLakebaseBranch, createExecutionContext, createFeatureEngineeringTargetPack, createFederationSharingTargetPack, createFoundationTargetPacks, createJobsCodeFoundationPack, createLakebaseTestProvider, createLakeflowFoundationPack, createLakeflowJobsTargetPack, createMlflowLifecycleTargetPack, createMockDatabricksClient, createModelServingTargetPack, createSecurityCostDrTargetPack, createSqlDeltaFoundationPack, createTargetPackRegistry, createUnityStorageFoundationPack, createVectorRagAgentsTargetPack, customLiveCheck, dbtBuildCheck, defineLiveSuite, defineTargetPack, defineTargetPackRun, deleteEphemeralLakebaseBranch, deleteVolumeFile, deltaContractCheck, doctorTargetPack, ensureVolumeDirectory, evalJudgeModelServingCheck, expectQueryToMatchGolden, expectTable, failureRecoveryCheck, featureEngineeringServingCheck, federationSharingCleanRoomsCheck, foundationTargetPacks, genieCheck, jobRunCheck, jobsCertificationCheck, jobsOrchestrationCheck, lakebaseCredentialCheck, lakebaseRoundTripCheck, lakeflowConnectCheck, mlflowLifecycleCheck, modelServingCheck, normalizeJobRun, normalizeVolumeRoot, notebookSubmitCheck, parseJobOrchestrationLiveSpec, performanceBudgetCheck, pipelineRefreshCheck, ragAgentCheck, regionalDrCheck, renderJUnit, resolveLakebaseProject, resolveProfile, restrictedPrincipalCheck, rollbackCheck, runLiveSuite, runTargetPack, secretRotationCheck, secretScopeCheck, securityPostureCostCheck, serverlessComputeCheck, sqlRoundTripCheck, toLakebaseBranchId, unityCatalogAccessCheck, uploadVolumeFile, validateJobGraph, vectorSearchCheck, volumeIOCheck, waitForPipelineUpdate };
|
|
3213
|
+
export { DatabricksAdvancedWorkloadsAdapter, DatabricksJobsAdapter, advancedStreamingCdcCheck, agentEvaluationPipelineCheck, agentServicesEnrollmentCheck, aiBiDashboardCheck, appHealthCheck, assertDeltaContract, autoLoaderIngestionCheck, backupRestoreCheck, builtinTargetPacks, classicComputeCheck, createAdvancedStreamingTargetPack, createAdvancedTargetPacks, createAgentEvaluationPipelineTargetPack, createAgentServicesEnrollmentTargetPack, createAiBiGenieTargetPack, createAppsOperationalFoundationPack, createBuiltinTargetPacks, createDataQualityObservabilityTargetPack, createEphemeralLakebaseBranch, createExecutionContext, createFeatureEngineeringTargetPack, createFederationSharingTargetPack, createFoundationTargetPacks, createJobsCodeFoundationPack, createLakebaseTestProvider, createLakeflowFoundationPack, createLakeflowJobsTargetPack, createManagedMcpAgentsTargetPack, createMlflow3AgentQualityTargetPack, createMlflowLifecycleTargetPack, createMockDatabricksClient, createModelServingTargetPack, createSecurityCostDrTargetPack, createSqlDeltaFoundationPack, createTargetPackRegistry, createUnityStorageFoundationPack, createVectorRagAgentsTargetPack, customLiveCheck, dataQualityAppTelemetryCheck, dbtBuildCheck, defineLiveSuite, defineTargetPack, defineTargetPackRun, deleteEphemeralLakebaseBranch, deleteVolumeFile, deltaContractCheck, doctorTargetPack, ensureVolumeDirectory, evalJudgeModelServingCheck, expectQueryToMatchGolden, expectTable, failureRecoveryCheck, featureEngineeringServingCheck, federationSharingCleanRoomsCheck, foundationTargetPacks, genieCheck, jobRunCheck, jobsCertificationCheck, jobsOrchestrationCheck, lakebaseCredentialCheck, lakebaseRoundTripCheck, lakeflowConnectCheck, managedMcpAgentsCheck, mlflow3GenAiQualityCheck, mlflowLifecycleCheck, modelServingCheck, normalizeJobRun, normalizeVolumeRoot, notebookSubmitCheck, parseJobOrchestrationLiveSpec, performanceBudgetCheck, pipelineRefreshCheck, ragAgentCheck, regionalDrCheck, renderJUnit, resolveLakebaseProject, resolveProfile, restrictedPrincipalCheck, rollbackCheck, runLiveSuite, runTargetPack, secretRotationCheck, secretScopeCheck, securityPostureCostCheck, serverlessComputeCheck, sqlRoundTripCheck, toLakebaseBranchId, unityCatalogAccessCheck, uploadVolumeFile, validateJobGraph, vectorSearchCheck, volumeIOCheck, waitForPipelineUpdate };
|
|
2756
3214
|
//# sourceMappingURL=index.js.map
|
|
2757
3215
|
//# sourceMappingURL=index.js.map
|