@truedat/dd 8.13.6 → 8.13.8

package/src/sagas/graph/__tests__/generation.spec.js

@@ -0,0 +1,247 @@
+import { apiJson } from "@truedat/core/services/api";
+import { runSaga, stdChannel } from "redux-saga";
+import { graphLoading } from "../../../reducers/graphLoading";
+import { graphGenerationState } from "../../../reducers/graphGenerationState";
+import { generateGraph } from "../generation";
+import {
+  backgroundGraph,
+  createGraph,
+  fetchGraph,
+  fetchLineageEvents,
+} from "../../../routines";
+
+const run = (api) => {
+  const actions = [];
+  let state = { graphGenerationState: undefined, graphLoading: false };
+  const dispatch = (action) => {
+    actions.push(action);
+    state = {
+      graphGenerationState: graphGenerationState(
+        state.graphGenerationState,
+        action,
+      ),
+      graphLoading: graphLoading(state.graphLoading, action),
+    };
+  };
+  const task = runSaga(
+    { dispatch, getState: () => state },
+    generateGraph,
+    createGraph,
+    api,
+    ["/graphs", {}],
+    true,
+  );
+  return { task, actions, dispatch, getState: () => state };
+};
+
+test("progress is loading, not failure", () => {
+  const progress = { data: { graph_hash: "a", retry: 2 } };
+  expect(graphLoading(true, fetchGraph.request(progress))).toEqual(progress);
+});
+
+test("close before the response keeps the graph in background", async () => {
+  let resolve;
+  const api = () =>
+    new Promise((done) => {
+      resolve = done;
+    });
+  const job = run(api);
+  job.dispatch(backgroundGraph());
+  resolve({ status: 201, data: { id: 42 } });
+  await job.task.toPromise();
+  expect(
+    job.actions.some((action) => action.type === createGraph.SUCCESS),
+  ).toBe(false);
+  expect(job.getState().graphLoading).toBe(false);
+  job.dispatch(
+    fetchLineageEvents.success({
+      data: [{ graph_id: 42, status: "COMPLETED" }],
+    }),
+  );
+  expect(
+    Object.values(job.getState().graphGenerationState.jobs)[0].status,
+  ).toBe("completed");
+});
+
+test("a foreground success publishes the graph", async () => {
+  const job = run(async () => ({ status: 201, data: { id: 42 } }));
+  await job.task.toPromise();
+  expect(
+    job.actions.find((action) => action.type === createGraph.SUCCESS).payload
+      .data,
+  ).toEqual({ id: 42, isRedirected: true });
+  expect(job.getState().graphLoading).toBe(false);
+});
+
+test("network failure is explicit and ends loading", async () => {
+  const job = run(async () => {
+    throw new Error("network");
+  });
+  await job.task.toPromise();
+  expect(
+    Object.values(job.getState().graphGenerationState.jobs)[0].status,
+  ).toBe("failed");
+  expect(
+    job.actions.filter((action) => action.type === createGraph.FAILURE),
+  ).toHaveLength(1);
+  expect(job.getState().graphLoading).toBe(false);
+});
+
+test("normalizes the GET graph envelope before publishing", async () => {
+  const job = run(async () => ({
+    status: 200,
+    data: { data: { id: 42, resources: [] } },
+  }));
+  await job.task.toPromise();
+  expect(
+    job.actions.find((action) => action.type === createGraph.SUCCESS).payload
+      .data.id,
+  ).toBe(42);
+});
+
+test("a late response cannot replace or finish the newer foreground request", async () => {
+  const actions = [];
+  let state = { graphGenerationState: undefined, graphLoading: false };
+  const dispatch = (action) => {
+    actions.push(action);
+    state = {
+      graphGenerationState: graphGenerationState(
+        state.graphGenerationState,
+        action,
+      ),
+      graphLoading: graphLoading(state.graphLoading, action),
+    };
+  };
+  let firstDone;
+  let secondDone;
+  const environment = { dispatch, getState: () => state };
+  const first = runSaga(
+    environment,
+    generateGraph,
+    createGraph,
+    () =>
+      new Promise((resolve) => {
+        firstDone = resolve;
+      }),
+    ["/graphs", {}],
+  );
+  const second = runSaga(
+    environment,
+    generateGraph,
+    createGraph,
+    () =>
+      new Promise((resolve) => {
+        secondDone = resolve;
+      }),
+    ["/graphs", {}],
+  );
+  firstDone({ status: 201, data: { id: 1 } });
+  await first.toPromise();
+  expect(state.graphLoading).toBeTruthy();
+  expect(actions.some((action) => action.type === createGraph.SUCCESS)).toBe(
+    false,
+  );
+  secondDone({ status: 201, data: { id: 2 } });
+  await second.toPromise();
+  expect(
+    actions
+      .filter((action) => action.type === createGraph.SUCCESS)
+      .map((action) => action.payload.data.id),
+  ).toEqual([2]);
+  expect(state.graphLoading).toBe(false);
+});
+
+test.each(["COMPLETED", "FAILED"])(
+  "stops polling on a %s event without publishing or failing the active graph",
+  async (status) => {
+    jest.useFakeTimers();
+    const channel = stdChannel();
+    const actions = [];
+    let state = { graphGenerationState: undefined, graphLoading: false };
+    const dispatch = (action) => {
+      actions.push(action);
+      state = {
+        graphGenerationState: graphGenerationState(
+          state.graphGenerationState,
+          action,
+        ),
+        graphLoading: graphLoading(state.graphLoading, action),
+      };
+      channel.put(action);
+    };
+    const api = jest.fn(async () => ({
+      status: 202,
+      data: { graph_hash: "a" },
+    }));
+    const task = runSaga(
+      { channel, dispatch, getState: () => state },
+      generateGraph,
+      createGraph,
+      api,
+      [],
+    );
+    try {
+      await Promise.resolve();
+      await Promise.resolve();
+      dispatch(backgroundGraph());
+      dispatch(
+        fetchLineageEvents.success({
+          data: [{ graph_hash: "a", graph_id: 5, status }],
+        }),
+      );
+      await Promise.resolve();
+      await Promise.resolve();
+      expect(task.isRunning()).toBe(false);
+      expect(jest.getTimerCount()).toBe(0);
+      expect(
+        actions.some(({ type }) =>
+          [createGraph.SUCCESS, createGraph.FAILURE].includes(type),
+        ),
+      ).toBe(false);
+      expect(Object.values(state.graphGenerationState.jobs)[0].status).toBe(
+        status === "COMPLETED" ? "completed" : "failed",
+      );
+    } finally {
+      task.cancel();
+      jest.useRealTimers();
+    }
+  },
+);
+
+jest.mock("@truedat/core/services/api", () => ({
+  apiJson: jest.fn(),
+  JSON_OPTS: {},
+}));
+
+test.each([false, true])(
+  "transient polling errors preserve the job (background=%s)",
+  async (background) => {
+    jest.useFakeTimers();
+    apiJson.mockReset();
+    apiJson
+      .mockRejectedValueOnce({ response: { status: 503 } })
+      .mockResolvedValueOnce({ status: 200, data: { id: 42 } });
+    const job = run(async () => ({ status: 202, data: { graph_hash: "a" } }));
+    try {
+      if (background) job.dispatch(backgroundGraph());
+      await jest.advanceTimersByTimeAsync(1000);
+      expect(
+        Object.values(job.getState().graphGenerationState.jobs)[0].status,
+      ).toBe("pending");
+      expect(job.actions.some(({ type }) => type === createGraph.FAILURE)).toBe(
+        false,
+      );
+      await jest.runAllTimersAsync();
+      await job.task.toPromise();
+      expect(
+        Object.values(job.getState().graphGenerationState.jobs)[0].status,
+      ).toBe(background ? "awaitingEvent" : "completed");
+      expect(job.actions.some(({ type }) => type === createGraph.SUCCESS)).toBe(
+        !background,
+      );
+    } finally {
+      job.task.cancel();
+      jest.useRealTimers();
+    }
+  },
+);

package/src/sagas/graph/__tests__/polling.spec.js

@@ -0,0 +1,88 @@
+import { runSaga } from "redux-saga";
+import { apiJson } from "@truedat/core/services/api";
+import { polling } from "../polling";
+
+jest.mock("@truedat/core/services/api", () => ({
+  apiJson: jest.fn(),
+  JSON_OPTS: {},
+}));
+const accepted = { status: 202, data: { graph_hash: "a" } };
+const completed = { status: 200, data: { id: 42 } };
+const serverError = { response: { status: 503 } };
+const start = () => {
+  const onProgress = jest.fn();
+  const onError = jest.fn();
+  const task = runSaga({ onError }, polling, accepted, onProgress);
+  const result = task.toPromise().catch((error) => error);
+  return { task, result, onProgress, onError };
+};
+beforeEach(() => {
+  jest.useFakeTimers();
+  apiJson.mockReset();
+});
+afterEach(() => jest.useRealTimers());
+
+test.each([new Error("network"), serverError])(
+  "recovers from a transient error: %j",
+  async (error) => {
+    apiJson.mockRejectedValueOnce(error).mockResolvedValueOnce(completed);
+    const job = start();
+    await jest.runAllTimersAsync();
+    expect(await job.result).toEqual(completed);
+    expect(apiJson).toHaveBeenCalledTimes(2);
+    expect(job.onError).not.toHaveBeenCalled();
+  },
+);
+
+test("stops after the initial failure and three retries", async () => {
+  apiJson.mockRejectedValue(serverError);
+  const job = start();
+  await jest.runAllTimersAsync();
+  expect(await job.result).toBe(serverError);
+  expect(apiJson).toHaveBeenCalledTimes(4);
+  expect(jest.getTimerCount()).toBe(0);
+});
+
+test("resets consecutive failures after a pending response", async () => {
+  apiJson
+    .mockRejectedValueOnce(serverError)
+    .mockRejectedValueOnce(serverError)
+    .mockRejectedValueOnce(serverError)
+    .mockResolvedValueOnce(accepted)
+    .mockRejectedValueOnce(serverError)
+    .mockRejectedValueOnce(serverError)
+    .mockRejectedValueOnce(serverError)
+    .mockResolvedValueOnce(completed);
+  const job = start();
+  await jest.runAllTimersAsync();
+  expect(await job.result).toEqual(completed);
+  expect(apiJson).toHaveBeenCalledTimes(8);
+  expect(job.onProgress).toHaveBeenCalledWith({
+    ...accepted,
+    data: { graph_hash: "a", retry: 1 },
+  });
+});
+
+test.each([400, 401, 403, 404, 429])(
+  "does not retry HTTP %s",
+  async (status) => {
+    const error = { response: { status } };
+    apiJson.mockRejectedValue(error);
+    const job = start();
+    await jest.runAllTimersAsync();
+    expect(await job.result).toBe(error);
+    expect(apiJson).toHaveBeenCalledTimes(1);
+  },
+);
+
+test("cancels the retry delay without further requests", async () => {
+  apiJson.mockRejectedValueOnce(serverError).mockResolvedValueOnce(completed);
+  const job = start();
+  await jest.advanceTimersByTimeAsync(1000);
+  expect(job.task.isRunning()).toBe(true);
+  job.task.cancel();
+  await job.result;
+  await jest.runAllTimersAsync();
+  expect(apiJson).toHaveBeenCalledTimes(1);
+  expect(jest.getTimerCount()).toBe(0);
+});

package/src/sagas/graph/generation.js

@@ -0,0 +1,99 @@
+import uniqueId from "lodash/fp/uniqueId";
+import { call, delay, put, race, select, take } from "redux-saga/effects";
+import {
+  backgroundGraph,
+  clearGraph,
+  dismissGraphGeneration,
+  fetchGraph,
+  fetchLineageEvents,
+  graphGeneration,
+} from "../../routines";
+import { fetchLineageEventsSaga } from "../fetchLineageEvents";
+import { polling } from "./polling";
+
+export const selectGeneration = (state) => state.graphGenerationState;
+
+function* isForeground(requestId) {
+  const state = yield select(selectGeneration);
+  const job = state?.jobs[requestId];
+  return state?.active === requestId && Boolean(job) && !job.background;
+}
+
+function* waitForGenerationEnd(requestId) {
+  while (true) {
+    const state = yield select(selectGeneration);
+    const job = state?.jobs[requestId];
+    if (!job || ["completed", "failed"].includes(job.status)) return;
+    yield take([fetchLineageEvents.SUCCESS, dismissGraphGeneration.TRIGGER]);
+  }
+}
+
+export function* generateGraph(routine, api, args, isRedirected = false) {
+  const requestId = uniqueId("graph-");
+  yield put(graphGeneration.request({ requestId }));
+  yield put(routine.request());
+  function* progress(response) {
+    yield put(graphGeneration({ requestId, hash: response.data.graph_hash }));
+    if (yield call(isForeground, requestId))
+      yield put(fetchGraph.request(response));
+  }
+  try {
+    const accepted = yield call(api, ...args);
+    if (accepted.status === 202) yield call(progress, accepted);
+    const { response } =
+      accepted.status === 202
+        ? yield race({
+            completed: call(waitForGenerationEnd, requestId),
+            response: call(polling, accepted, progress),
+          })
+        : { response: accepted };
+    if (!response) return;
+    if (![200, 201].includes(response.status))
+      throw new Error("Unexpected graph response");
+    // GET /graphs/:id wraps the graph in data; POST and hash polling may return it directly.
+    const graph = response.data?.data || response.data;
+    if (graph?.id == null) throw new Error("Missing graph id");
+    const foreground = yield call(isForeground, requestId);
+    if (foreground) {
+      yield put(
+        routine.success({ ...response, data: { ...graph, isRedirected } }),
+      );
+    }
+    yield put(
+      graphGeneration.success({
+        requestId,
+        graphId: graph.id,
+        status: foreground ? "completed" : "awaitingEvent",
+      }),
+    );
+    yield put(fetchLineageEvents());
+  } catch (error) {
+    if (yield call(isForeground, requestId))
+      yield put(routine.failure(error.response || error.message));
+    yield put(graphGeneration.failure({ requestId, status: "failed" }));
+  } finally {
+    // A late job must not clear the loading state of a newer request.
+    if (yield call(isForeground, requestId)) yield put(routine.fulfill());
+  }
+}
+
+export function* watchBackgroundGraphs() {
+  while (true) {
+    yield take([
+      backgroundGraph.TRIGGER,
+      clearGraph.TRIGGER,
+      graphGeneration.REQUEST,
+    ]);
+    while (true) {
+      const generation = yield select(selectGeneration);
+      const pending = Object.values(generation?.jobs || {}).some(
+        (job) =>
+          job.notify && ["pending", "awaitingEvent"].includes(job.status),
+      );
+      if (!pending) break;
+      const loading = yield select((state) => state.lineageEventsLoading);
+      if (!loading) yield call(fetchLineageEventsSaga);
+      yield delay(3000);
+    }
+  }
+}

package/src/sagas/graph/polling.js

@@ -1,14 +1,11 @@
 /* eslint-disable fp/no-mutation */
 /* eslint-disable fp/no-let */
 
-import { call, cancelled, delay, put } from "redux-saga/effects";
+import { call, delay } from "redux-saga/effects";
 import { apiJson, JSON_OPTS } from "@truedat/core/services/api";
 import { compile } from "path-to-regexp";
-import { fetchGraph } from "../../routines";
 import { API_GRAPH_HASH } from "../../api";
 
-export const POLLING_CANCELLED = "Polling cancelled!";
-
 function* exponentialBackoff(base, power) {
   while (true) {
     yield Math.pow(base, power);
@@ -17,15 +14,16 @@ function* exponentialBackoff(base, power) {
   }
 }
 
-export function* polling(routine, graphAccepted, isRedirected) {
+// Return the result to its owner so background work cannot replace the canvas.
+export function* polling(graphAccepted, onProgress) {
   const backoff = exponentialBackoff(1.2, 0);
   let backoffTime,
     retry = 0,
-    graphResp,
-    isCancelled = false;
+    consecutiveErrors = 0,
+    graphResp;
 
   do {
-    backoffTime = backoff.next().value * 1000;
+    backoffTime = Math.min(backoff.next().value * 1000, 10000);
     yield delay(backoffTime);
 
     try {
@@ -36,35 +34,28 @@ export function* polling(routine, graphAccepted, isRedirected) {
         }),
         JSON_OPTS,
       );
+
+      consecutiveErrors = 0;
     } catch (error) {
-      if (error.response) {
-        const { status, data } = error.response;
-        yield put(routine.failure({ status, data }));
-      } else {
-        yield put(routine.failure(error.message));
-      }
+      const status = error.response?.status;
+      if (
+        (status != null && (status < 500 || status >= 600)) ||
+        ++consecutiveErrors > 3
+      )
+        throw error;
+      continue;
     }
 
     graphResp?.status == 202 &&
-      (yield put(
-        fetchGraph.failure({
-          data: {
-            graph_hash: graphAccepted.data.graph_hash,
-            retry: ++retry,
-          },
-        }),
-      ));
-
-    isCancelled = yield cancelled();
-    isCancelled && (yield put(routine.failure(POLLING_CANCELLED)));
-  } while (!isCancelled && graphResp?.status == 202);
-
-  graphResp?.status == 200 && !isCancelled
-    ? yield put(
-        routine.success({
-          ...graphResp,
-          data: { ...graphResp.data, isRedirected: isRedirected || false },
-        }),
-      )
-    : yield put(routine.failure({}));
+      (yield call(onProgress, {
+        ...graphResp,
+        data: {
+          ...graphResp.data,
+          graph_hash: graphAccepted.data.graph_hash,
+          retry: ++retry,
+        },
+      }));
+  } while (consecutiveErrors > 0 || graphResp?.status == 202);
+
+  return graphResp;
 }

package/src/sagas/index.js

@@ -52,6 +52,7 @@ import { updateSystemRequestSaga } from "./updateSystem";
 import { uploadStructuresDomainsRequestSaga } from "./uploadStructuresDomains";
 import { updateTagRequestSaga } from "./updateTag";
 import { upsertCatalogViewConfigRequestSaga } from "./upsertCatalogViewConfig";
+import { watchBackgroundGraphs } from "./graph/generation";
 
 export {
   bulkUpdateStructuresRequestsSaga,
@@ -165,4 +166,5 @@ export default [
   updateSystemRequestSaga(),
   uploadStructuresDomainsRequestSaga(),
   upsertCatalogViewConfigRequestSaga(),
+  watchBackgroundGraphs(),
 ];

package/src/styles/LineageGraph.less

@@ -4125,6 +4125,123 @@ button.lineage-graph__dock-context-reopen {
   flex-shrink: 0;
 }
 
+.ui.dimmer.lineage-generation-dimmer {
+  background-color: rgba(0, 0, 0, 0.24);
+  backdrop-filter: blur(4px);
+  -webkit-backdrop-filter: blur(4px);
+}
+.ui.basic.modal.lineage-generation-dialog {
+  width: min(16rem, calc(100% - 2rem));
+  padding: 0;
+  &.lineage-generation-dialog--expanded {
+    width: min(32rem, calc(100% - 2rem));
+    .lineage-graph__loading-card {
+      padding: 2.5rem 1.5rem 1.5rem;
+      gap: 1rem;
+    }
+  }
+  .lineage-graph__loading-card {
+    position: relative;
+    min-width: 0;
+    max-width: none;
+    width: 100%;
+    padding: 1.25rem 1.5rem;
+    gap: 0.5rem;
+  }
+}
+.lineage-generation-dialog__close {
+  position: absolute;
+  top: 0.25rem;
+  right: 0.25rem;
+  width: 2.75rem;
+  height: 2.75rem;
+  border: 0;
+  background: transparent;
+  color: #435064;
+  cursor: pointer;
+  font-size: 1.5rem;
+}
+.lineage-generation-dialog__actions {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 0.75rem;
+  width: 100%;
+  .ui.button {
+    flex: 1 1 10rem;
+    margin: 0;
+    min-height: 2.75rem;
+  }
+}
+.lineage-generation-dialog__close:focus,
+.lineage-generation-dialog__close:focus-visible {
+  outline: none;
+  box-shadow: none;
+  color: var(--td-graph-accent, #ed5c17);
+}
+.lineage-generation-dialog__actions .ui.button:focus-visible,
+.lineage-generation-notices :is(button, a):focus-visible {
+  outline: 2px solid var(--td-graph-accent, #ed5c17);
+  outline-offset: 3px;
+}
+.lineage-generation-notices__item {
+  position: relative;
+  display: flex;
+  align-items: center;
+  flex-wrap: wrap;
+  gap: 0.75rem;
+  padding: 0.75rem 1rem;
+  background: #f8fafc;
+  color: #435064;
+  border-bottom: 1px solid #e2e6ed;
+  span {
+    flex: 1 1 15rem;
+  }
+  button {
+    border: 0;
+    background: transparent;
+    cursor: pointer;
+    min-width: 2.75rem;
+    min-height: 2.75rem;
+  }
+}
+.lineage-generation-notices__item--ready::after {
+  content: "";
+  position: absolute;
+  inset: 0;
+  border: 2px solid var(--td-graph-accent, #ed5c17);
+  border-radius: inherit;
+  pointer-events: none;
+  opacity: 0;
+  // Animate only opacity so the highlight does not shift the notice or its actions.
+  animation: lineage-generation-ready 2.4s ease-out;
+}
+@keyframes lineage-generation-ready {
+  0%,
+  100% {
+    opacity: 0;
+  }
+  15%,
+  40% {
+    opacity: 1;
+  }
+}
+@media (prefers-reduced-motion: reduce) {
+  .lineage-generation-dialog .lineage-graph__loading-spinner,
+  .lineage-generation-notices__item--ready::after {
+    animation: none;
+  }
+}
+
+.lineage-generation-notices {
+  position: fixed;
+  right: 1rem;
+  bottom: 1rem;
+  width: min(32rem, calc(100% - 2rem));
+  max-height: 40vh;
+  overflow-y: auto;
+  z-index: 900;
+}
+
 .lineage-graph__stage-surface
   .react-flow__node.lineage-node--pinned
   > :is(.td-lineage-resource-node, .td-lineage-group-node) {