@fabricorg/databricks-testkit 0.3.0 → 0.4.0

package/dist/index.cjs

@@ -1722,7 +1722,7 @@ function createSqlDeltaFoundationPack() {
     ],
     requiredChecks: ["sql"],
     requirements: [workspaceRequirement, authRequirement, warehouseRequirement],
-    docsUrl: `${DOCS}#existing-foundation-to-migrate-into-packs`,
+    docsUrl: `${DOCS}#sql-and-delta-foundation`,
     certificationScopes: [AZURE_SCOPE]
   });
 }
@@ -1747,7 +1747,7 @@ function createLakeflowFoundationPack() {
     ],
     requiredChecks: ["pipeline"],
     requirements: [workspaceRequirement, authRequirement],
-    docsUrl: `${DOCS}#existing-foundation-to-migrate-into-packs`,
+    docsUrl: `${DOCS}#lakeflow-ingestion-foundation`,
     certificationScopes: [AZURE_SCOPE]
   });
 }
@@ -1775,7 +1775,7 @@ function createJobsCodeFoundationPack() {
     ],
     requiredChecks: ["jobs"],
     requirements: [workspaceRequirement, authRequirement],
-    docsUrl: `${DOCS}#existing-foundation-to-migrate-into-packs`,
+    docsUrl: `${DOCS}#jobs-and-code-foundation`,
     certificationScopes: [AZURE_SCOPE]
   });
 }
@@ -1801,7 +1801,7 @@ function createUnityStorageFoundationPack() {
     ],
     requiredChecks: ["uc-grants"],
     requirements: [workspaceRequirement, authRequirement, warehouseRequirement],
-    docsUrl: `${DOCS}#existing-foundation-to-migrate-into-packs`,
+    docsUrl: `${DOCS}#unity-catalog-and-storage-foundation`,
     certificationScopes: [AZURE_SCOPE]
   });
 }
@@ -1827,7 +1827,7 @@ function createAppsOperationalFoundationPack() {
     ],
     requiredChecks: ["app"],
     requirements: [workspaceRequirement, authRequirement],
-    docsUrl: `${DOCS}#existing-foundation-to-migrate-into-packs`,
+    docsUrl: `${DOCS}#apps-and-operational-data-foundation`,
     certificationScopes: [AZURE_SCOPE]
   });
 }
@@ -1858,7 +1858,8 @@ var DatabricksJobsAdapter = class {
   async get(runId) {
     const raw = await this.client.get("/api/2.1/jobs/runs/get", {
       run_id: runId,
-      include_history: true
+      include_history: true,
+      include_resolved_values: true
     });
     return normalizeJobRun(runId, raw);
   }
@@ -1872,15 +1873,46 @@ var DatabricksJobsAdapter = class {
   async cancel(runId) {
     await this.client.post("/api/2.1/jobs/runs/cancel", { run_id: runId });
   }
+  async waitUntilActive(runId, options = {}) {
+    const timeoutMs = options.timeoutMs ?? 3e5;
+    const pollIntervalMs = options.pollIntervalMs ?? 2e3;
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+      const run = await this.get(runId);
+      if (run.lifeCycleState === "RUNNING") return run;
+      if (isTerminal(run.lifeCycleState)) {
+        throw new Error(
+          `Databricks run ${runId} reached ${run.lifeCycleState}/${run.resultState} before it could be canceled`
+        );
+      }
+      await delay2(pollIntervalMs);
+    }
+    throw new Error(`Databricks run ${runId} did not become active within ${timeoutMs}ms`);
+  }
   async repair(runId, options = {}) {
     const response = await this.client.post("/api/2.1/jobs/runs/repair", {
       run_id: runId,
       ...options.latestRepairId ? { latest_repair_id: options.latestRepairId } : {},
+      ...options.rerunDependentTasks ? { rerun_dependent_tasks: true } : {},
       ...options.rerunTasks?.length ? { rerun_tasks: options.rerunTasks } : { rerun_all_failed_tasks: true }
     });
     if (!response.repair_id) throw new Error("Databricks Jobs repair returned no repair_id");
     return response.repair_id;
   }
+  async waitForRepair(runId, repairId, options = {}) {
+    const timeoutMs = options.timeoutMs ?? 9e5;
+    const pollIntervalMs = options.pollIntervalMs ?? 5e3;
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+      const run = await this.get(runId);
+      const repair = run.repairs.find((candidate) => candidate.repairId === repairId);
+      if (repair && isTerminal(repair.lifeCycleState)) return run;
+      await delay2(pollIntervalMs);
+    }
+    throw new Error(
+      `Databricks repair ${repairId} for run ${runId} timed out after ${timeoutMs}ms`
+    );
+  }
 };
 function validateJobGraph(tasks) {
   if (tasks.length === 0) throw new Error("Jobs orchestration requires at least one task");
@@ -1932,10 +1964,14 @@ function jobsOrchestrationCheck() {
         timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
         pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
       });
+      let repairId;
       if (spec.repairFailedTasks && run.resultState !== "SUCCESS") {
         const failed = run.tasks.filter((task) => task.resultState === "FAILED").map((task) => task.taskKey);
-        await adapter.repair(runId, { rerunTasks: failed.length > 0 ? failed : void 0 });
-        run = await adapter.wait(runId, {
+        repairId = await adapter.repair(runId, {
+          rerunTasks: failed.length > 0 ? failed : void 0,
+          rerunDependentTasks: true
+        });
+        run = await adapter.waitForRepair(runId, repairId, {
           timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
           pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
         });
@@ -1953,10 +1989,32 @@ function jobsOrchestrationCheck() {
           throw new Error(`Task '${taskKey}' ended in ${task.resultState}, expected ${expected}`);
         }
       }
+      let cancellation;
+      if (spec.cancellation) {
+        const cancellationRunId = await adapter.submit(spec.cancellation.request);
+        await adapter.waitUntilActive(cancellationRunId, {
+          timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
+          pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
+        });
+        await adapter.cancel(cancellationRunId);
+        const canceled = await adapter.wait(cancellationRunId, {
+          timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
+          pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 1e4)
+        });
+        const expectedCancellation = spec.cancellation.expectedResult ?? "CANCELED";
+        if (canceled.resultState !== expectedCancellation) {
+          throw new Error(
+            `Cancellation run ${cancellationRunId} ended in ${canceled.resultState}, expected ${expectedCancellation}`
+          );
+        }
+        cancellation = { runId: cancellationRunId, resultState: canceled.resultState };
+      }
       return {
         runId,
+        ...repairId ? { repairId } : {},
         resultState: run.resultState,
-        tasks: run.tasks
+        tasks: run.tasks,
+        ...cancellation ? { cancellation } : {}
       };
     }
   };
@@ -1976,11 +2034,24 @@ function parseJobOrchestrationLiveSpec(value) {
     throw new Error("Jobs orchestration spec requires request.tasks");
   }
   validateJobGraph(spec.request.tasks);
+  if (spec.cancellation) {
+    if (typeof spec.cancellation !== "object" || !spec.cancellation.request || !Array.isArray(spec.cancellation.request.tasks)) {
+      throw new Error("Jobs orchestration cancellation requires request.tasks");
+    }
+    validateJobGraph(spec.cancellation.request.tasks);
+  }
   return spec;
 }
 function normalizeJobRun(fallbackRunId, raw) {
   const state = objectOf(raw.state);
-  const tasks = Array.isArray(raw.tasks) ? raw.tasks.filter(
+  const repairEntries = Array.isArray(raw.repair_history) ? raw.repair_history.filter(
+    (repair) => Boolean(repair && typeof repair === "object")
+  ) : [];
+  const taskRunRanks = /* @__PURE__ */ new Map();
+  for (const [rank, repair] of repairEntries.entries()) {
+    for (const runId of numberArray(repair.task_run_ids)) taskRunRanks.set(runId, rank);
+  }
+  const taskAttempts = Array.isArray(raw.tasks) ? raw.tasks.filter(
     (task) => Boolean(task && typeof task === "object")
   ).map((task) => {
     const taskState = objectOf(task.state);
@@ -1989,20 +2060,236 @@ function normalizeJobRun(fallbackRunId, raw) {
       ...Number.isFinite(Number(task.run_id)) ? { runId: Number(task.run_id) } : {},
       lifeCycleState: String(taskState.life_cycle_state ?? "UNKNOWN"),
       resultState: String(taskState.result_state ?? "UNKNOWN"),
-      ...typeof taskState.state_message === "string" ? { stateMessage: taskState.state_message } : {}
+      ...typeof taskState.state_message === "string" ? { stateMessage: taskState.state_message } : {},
+      ...Number.isFinite(Number(task.attempt_number)) ? { attemptNumber: Number(task.attempt_number) } : {}
     };
   }) : [];
+  const latestTasks = /* @__PURE__ */ new Map();
+  for (const task of taskAttempts) {
+    const current = latestTasks.get(task.taskKey);
+    const currentAttempt = current?.attemptNumber ?? -1;
+    const candidateAttempt = task.attemptNumber ?? -1;
+    const currentRank = current?.runId ? taskRunRanks.get(current.runId) ?? -1 : -1;
+    const candidateRank = task.runId ? taskRunRanks.get(task.runId) ?? -1 : -1;
+    if (!current || candidateAttempt > currentAttempt || candidateAttempt === currentAttempt && candidateRank >= currentRank) {
+      latestTasks.set(task.taskKey, task);
+    }
+  }
+  const tasks = [...latestTasks.values()];
+  const repairs = repairEntries.map((repair) => {
+    const repairState = objectOf(repair.state);
+    return {
+      repairId: Number(repair.id ?? repair.repair_id ?? 0),
+      type: String(repair.type ?? "UNKNOWN"),
+      lifeCycleState: String(repairState.life_cycle_state ?? "UNKNOWN"),
+      resultState: String(repairState.result_state ?? "UNKNOWN"),
+      taskRunIds: numberArray(repair.task_run_ids)
+    };
+  });
   return {
     runId: Number(raw.run_id ?? fallbackRunId),
     lifeCycleState: String(state.life_cycle_state ?? "UNKNOWN"),
     resultState: String(state.result_state ?? "UNKNOWN"),
     tasks,
+    repairs,
     raw
   };
 }
+function isTerminal(state) {
+  return ["TERMINATED", "SKIPPED", "INTERNAL_ERROR", "BLOCKED"].includes(state);
+}
+function delay2(ms) {
+  return new Promise((resolve2) => setTimeout(resolve2, ms));
+}
 function objectOf(value) {
   return value && typeof value === "object" && !Array.isArray(value) ? value : {};
 }
+function numberArray(value) {
+  return Array.isArray(value) ? value.map(Number).filter((candidate) => Number.isFinite(candidate)) : [];
+}
+
+// src/jobs-certification-check.ts
+var NOTEBOOKS = {
+  seed: `# Databricks notebook source
+dbutils.jobs.taskValues.set(key="branch", value="run")
+dbutils.jobs.taskValues.set(key="items", value=["alpha", "beta"])
+dbutils.notebook.exit("seeded")
+`,
+  process: `# Databricks notebook source
+dbutils.widgets.text("item", "")
+item = dbutils.widgets.get("item")
+assert item, "item parameter is required"
+dbutils.notebook.exit(item)
+`,
+  retry: `# Databricks notebook source
+dbutils.widgets.text("execution_count", "0")
+execution_count = int(dbutils.widgets.get("execution_count"))
+if execution_count <= 1:
+    raise RuntimeError("intentional retryable failure")
+dbutils.notebook.exit("retried")
+`,
+  repair: `# Databricks notebook source
+dbutils.widgets.text("repair_count", "0")
+repair_count = int(dbutils.widgets.get("repair_count"))
+if repair_count == 0:
+    raise RuntimeError("intentional first-run failure")
+dbutils.notebook.exit("repaired")
+`,
+  sleep: `# Databricks notebook source
+import time
+time.sleep(300)
+`
+};
+function jobsCertificationCheck() {
+  return {
+    id: "jobs-certification",
+    description: "Disposable Lakeflow Jobs branch, loop, failure, repair, cancellation, and cleanup certification",
+    configured: (env) => env.DBX_TEST_JOBS_CERTIFY === "1",
+    async run({ env, client }) {
+      const root = `${env.DBX_TEST_JOBS_FIXTURE_ROOT ?? "/Workspace/Shared/fabric-experiments-target-pack"}/${crypto.randomUUID()}`;
+      const paths = Object.fromEntries(
+        Object.keys(NOTEBOOKS).map((name) => [name, `${root}/${name}`])
+      );
+      try {
+        await importNotebooks(client, paths);
+        const adapter = new DatabricksJobsAdapter(client);
+        const runId = await adapter.submit(certificationRequest(paths));
+        const initial = await adapter.wait(runId, waitOptions(env));
+        if (initial.resultState !== "FAILED") {
+          throw new Error(
+            `Certification run ${runId} expected FAILED before repair, got ${initial.resultState}`
+          );
+        }
+        const retried = assertTask(initial.tasks, "retry", "SUCCESS");
+        if ((retried.attemptNumber ?? 0) < 1) {
+          throw new Error("Certification retry task did not record a second attempt");
+        }
+        assertTask(initial.tasks, "seed", "SUCCESS");
+        assertTask(initial.tasks, "branch", "SUCCESS");
+        assertTask(initial.tasks, "foreach", "SUCCESS");
+        assertTask(initial.tasks, "repair", "FAILED");
+        const repairId = await adapter.repair(runId, {
+          rerunTasks: ["repair"],
+          rerunDependentTasks: true
+        });
+        const repaired = await adapter.waitForRepair(runId, repairId, waitOptions(env));
+        if (repaired.resultState !== "SUCCESS") {
+          throw new Error(`Certification repair ${repairId} ended in ${repaired.resultState}`);
+        }
+        assertTask(repaired.tasks, "repair", "SUCCESS");
+        const cancellationRunId = await adapter.submit(cancellationRequest(paths));
+        await adapter.waitUntilActive(cancellationRunId, waitOptions(env));
+        await adapter.cancel(cancellationRunId);
+        const canceled = await adapter.wait(cancellationRunId, waitOptions(env));
+        if (canceled.resultState !== "CANCELED") {
+          throw new Error(`Certification cancellation ended in ${canceled.resultState}`);
+        }
+        return {
+          runId,
+          repairId,
+          cancellationRunId,
+          taskValues: true,
+          retry: true,
+          condition: true,
+          forEach: true,
+          repair: true,
+          cancellation: true,
+          fixtureRoot: root
+        };
+      } finally {
+        await client.post("/api/2.0/workspace/delete", { path: root, recursive: true });
+      }
+    }
+  };
+}
+async function importNotebooks(client, paths) {
+  await client.post("/api/2.0/workspace/mkdirs", {
+    path: paths.seed.slice(0, paths.seed.lastIndexOf("/"))
+  });
+  for (const [name, source] of Object.entries(NOTEBOOKS)) {
+    await client.post("/api/2.0/workspace/import", {
+      path: paths[name],
+      format: "SOURCE",
+      language: "PYTHON",
+      overwrite: true,
+      content: Buffer.from(source).toString("base64")
+    });
+  }
+}
+function certificationRequest(paths) {
+  return {
+    run_name: "fabric-experiments-jobs-certification",
+    tasks: [
+      {
+        task_key: "retry",
+        notebook_task: {
+          notebook_path: paths.retry,
+          base_parameters: { execution_count: "{{task.execution_count}}" }
+        },
+        max_retries: 1
+      },
+      {
+        task_key: "seed",
+        depends_on: [{ task_key: "retry" }],
+        notebook_task: { notebook_path: paths.seed }
+      },
+      {
+        task_key: "branch",
+        depends_on: [{ task_key: "seed" }],
+        condition_task: {
+          left: "{{tasks.seed.values.branch}}",
+          op: "EQUAL_TO",
+          right: "run"
+        }
+      },
+      {
+        task_key: "foreach",
+        depends_on: [{ task_key: "branch", outcome: "true" }],
+        for_each_task: {
+          inputs: "{{tasks.seed.values.items}}",
+          concurrency: 2,
+          task: {
+            task_key: "process_item",
+            notebook_task: {
+              notebook_path: paths.process,
+              base_parameters: { item: "{{input}}" }
+            }
+          }
+        }
+      },
+      {
+        task_key: "repair",
+        depends_on: [{ task_key: "foreach" }],
+        notebook_task: {
+          notebook_path: paths.repair,
+          base_parameters: { repair_count: "{{job.repair_count}}" }
+        }
+      }
+    ]
+  };
+}
+function cancellationRequest(paths) {
+  return {
+    run_name: "fabric-experiments-jobs-cancellation-certification",
+    tasks: [{ task_key: "sleep", notebook_task: { notebook_path: paths.sleep } }]
+  };
+}
+function waitOptions(env) {
+  return {
+    timeoutMs: Number(env.DBX_TEST_TIMEOUT_MS ?? 9e5),
+    pollIntervalMs: Number(env.DBX_TEST_POLL_INTERVAL_MS ?? 5e3)
+  };
+}
+function assertTask(tasks, taskKey, resultState) {
+  const task = tasks.find((candidate) => candidate.taskKey === taskKey);
+  if (!task) throw new Error(`Certification result did not contain task '${taskKey}'`);
+  if (task.resultState !== resultState) {
+    throw new Error(
+      `Certification task '${taskKey}' ended in ${task.resultState}, expected ${resultState}`
+    );
+  }
+  return task;
+}
 
 // src/jobs-target-pack.ts
 function createLakeflowJobsTargetPack() {
@@ -2010,8 +2297,8 @@ function createLakeflowJobsTargetPack() {
     id: "lakeflow-jobs",
     name: "Lakeflow Jobs orchestration",
     description: "Typed multi-task DAG, branch/loop, retry, cancellation, repair and task-outcome validation.",
-    version: "0.1.0",
-    maturity: "shipped",
+    version: "0.2.0",
+    maturity: "live-gated",
     capabilities: [
       "jobs.multi-task-dag",
       "jobs.dependencies",
@@ -2022,7 +2309,12 @@ function createLakeflowJobsTargetPack() {
       "jobs.repair",
       "jobs.task-outcomes"
     ],
-    checks: () => [restrictedPrincipalCheck(), jobRunCheck(), jobsOrchestrationCheck()],
+    checks: () => [
+      restrictedPrincipalCheck(),
+      jobRunCheck(),
+      jobsOrchestrationCheck(),
+      jobsCertificationCheck()
+    ],
     requiredChecks: ["jobs-orchestration"],
     requirements: [
       {
@@ -2045,11 +2337,14 @@ function createLakeflowJobsTargetPack() {
       {
         id: "orchestration-spec",
         description: "JSON multi-task orchestration scenario",
-        anyOf: ["DBX_TEST_JOBS_ORCHESTRATION_JSON"]
+        anyOf: ["DBX_TEST_JOBS_ORCHESTRATION_JSON"],
+        required: false
       }
     ],
-    docsUrl: "https://experiments.fabric.pro/docs/testing/target-packs/#pack-1--lakeflow-jobs-orchestration",
-    certificationScopes: []
+    docsUrl: "https://experiments.fabric.pro/docs/testing/target-packs/#lakeflow-jobs-orchestration",
+    certificationScopes: [
+      "Azure Databricks serverless Jobs, West US, workspace OAuth, public network path"
+    ]
   });
 }
 
@@ -2106,6 +2401,7 @@ exports.expectTable = expectTable;
 exports.failureRecoveryCheck = failureRecoveryCheck;
 exports.foundationTargetPacks = foundationTargetPacks;
 exports.jobRunCheck = jobRunCheck;
+exports.jobsCertificationCheck = jobsCertificationCheck;
 exports.jobsOrchestrationCheck = jobsOrchestrationCheck;
 exports.lakebaseCredentialCheck = lakebaseCredentialCheck;
 exports.lakebaseRoundTripCheck = lakebaseRoundTripCheck;