@truedat/dd 4.33.3 → 4.33.8

package/src/components/__tests__/__snapshots__/LineageRoutes.spec.js.snap

@@ -10,6 +10,10 @@ exports[`<LineageRoutes /> matches the latest snapshot 1`] = `
     path="/graphs/:id"
     render={[Function]}
   />
+  <Route
+    path="/lineage_events"
+    render={[Function]}
+  />
   <Route
     path="/sample"
     render={[Function]}

package/src/messages/en.js

@@ -103,12 +103,30 @@ export default {
   "graph.header.target_class": "Target Type",
   "graph.header.target_external_id": "Target External Id",
   "graph.header.target_name": "Target Name",
+  "lineage.events.header": "My graphs",
+  "lineage.events.props.graph_data": "Graph",
+  "lineage.events.props.inserted_at": "Date",
+  "lineage.events.props.message": "Message",
+  "lineage.events.props.status": "Status",
+  "lineage.events.search.results.empty": "No graphs found",
+  "lineage.events.subheader": "Lineage graphs",
   "lineage.form.header": "Select Structures",
   "lineage.form.levels.all.label": "All",
   "lineage.form.levels.header": "Levels",
   "lineage.form.levels.select.input": "Select...",
   "lineage.form.levels.select.label": "Depth",
   "lineage.graph.loading": "Analyzing...",
+  "lineage.graph.loading.background.launched": "Background task launched",
+  "lineage.graph.loading.background.existing":
+    "The task for this graph was already launched",
+  "lineage.graph.loading.polling": "Poll retry #{retry}",
+  "lineage.graph.status.already_calculated": "Already calculated",
+  "lineage.graph.status.already_started": "Already started",
+  "lineage.graph.status.completed": "Completed",
+  "lineage.graph.status.failed": "Failed",
+  "lineage.graph.status.just_started": "Just started",
+  "lineage.graph.status.started": "Started",
+  "lineage.graph.status.timed_out": "Timed out",
   "lineage.group.placeholder": "Select Top Level Structure",
   "lineage.resources.placeholder": "Select Child Structures",
   "lineage.subgroup.placeholder": "Select Child Structure",

package/src/messages/es.js

@@ -105,12 +105,31 @@ export default {
   "graph.header.target_class": "Tipo Destino",
   "graph.header.target_external_id": "Id Externo Destino",
   "graph.header.target_name": "Nombre Destino",
+  "lineage.events.header": "Mis grafos",
+  "lineage.events.props.graph_data": "Grafo",
+  "lineage.events.props.inserted_at": "Fecha",
+  "lineage.events.props.message": "Mensaje",
+  "lineage.events.props.status": "Estado",
+  "lineage.events.search.results.empty": "Ningún grafo encontrado",
+  "lineage.events.subheader": "Grafos de linaje",
   "lineage.form.header": "Seleccionar Estructuras",
   "lineage.form.levels.all.label": "Completo",
   "lineage.form.levels.header": "Niveles",
   "lineage.form.levels.select.input": "Seleccionar...",
   "lineage.form.levels.select.label": "Profundidad",
   "lineage.graph.loading": "Analizando...",
+  "lineage.graph.loading.background.launched":
+    "Tarea en segundo plano lanzada para grafo",
+  "lineage.graph.loading.background.existing":
+    "La tarea en segundo plano para este grafo ya fue lanzada",
+  "lineage.graph.loading.polling": "Sondeo #{retry}",
+  "lineage.graph.status.already_calculated": "Previamente calculado",
+  "lineage.graph.status.already_started": "Previamente iniciado",
+  "lineage.graph.status.completed": "Finalizado",
+  "lineage.graph.status.failed": "Fallido",
+  "lineage.graph.status.just_started": "Acaba de iniciarse",
+  "lineage.graph.status.started": "Iniciado",
+  "lineage.graph.status.timed_out": "Tiempo de espera máximo sobrepasado",
   "lineage.group.placeholder": "Seleccionar Estructura Padre",
   "lineage.resources.placeholder": "Seleccionar Estructuras Hijos",
   "lineage.subgroup.placeholder": "Seleccionar Estructura Hijo",

package/src/reducers/__tests__/graphLoading.spec.js

@@ -5,7 +5,7 @@ import {
   createStructureGraph,
   fetchGraph,
   fetchStructure,
-  fetchStructureGraph
+  fetchStructureGraph,
 } from "../../routines";
 import { graphLoading } from "..";
 
@@ -17,7 +17,7 @@ describe("reducers: graphLoading", () => {
   });
 
   it("should be true after receiving the createGraph.REQUEST action", () => {
-    expect(graphLoading(false, { type: createGraph.REQUEST })).toBe(true);
+    expect(graphLoading(false, { type: createGraph.REQUEST })).toBeTruthy();
   });
 
   it("should be true after receiving the createStructureGraph.REQUEST action", () => {

package/src/reducers/__tests__/lineageEventsLoading.spec.js

@@ -0,0 +1,26 @@
+import { lineageEventsLoading } from "../lineageEventsLoading";
+import { fetchLineageEvents } from "../../routines";
+
+const fooState = { foo: "bar" };
+
+describe("reducers: lineageEventsLoading", () => {
+  it("should provide the initial state", () => {
+    expect(lineageEventsLoading(undefined, {})).toBe(false);
+  });
+
+  it("returns to initial state on fetchLineageEvents.FULFILL action", () => {
+    expect(
+      lineageEventsLoading(fooState, { type: fetchLineageEvents.FULFILL })
+    ).toBe(false);
+  });
+
+  it("returns true on fetchLineageEvents.REQUEST action", () => {
+    expect(
+      lineageEventsLoading(fooState, { type: fetchLineageEvents.REQUEST })
+    ).toBe(true);
+  });
+
+  it("should ignore unhandled actions", () => {
+    expect(lineageEventsLoading(fooState, { type: "FOO" })).toBe(fooState);
+  });
+});

package/src/reducers/graph.js

@@ -3,7 +3,8 @@ import {
   clearGraph,
   clearStructure,
   fetchGraph,
-  fetchStructure
+  fetchStructure,
+  createGraph,
 } from "../routines";
 
 export const initialState = {};
@@ -18,6 +19,8 @@ export const graph = (state = initialState, { type, payload }) => {
       return initialState;
     case fetchGraph.SUCCESS:
       return _.prop("data")(payload);
+    case createGraph.SUCCESS:
+      return _.prop("data")(payload);
     default:
       return state;
   }

package/src/reducers/graphLoading.js

@@ -5,25 +5,29 @@ import {
   createStructureGraph,
   fetchGraph,
   fetchStructure,
-  fetchStructureGraph
+  fetchStructureGraph,
 } from "../routines";
 
-export const graphLoading = (state = false, { type }) => {
+export const graphLoading = (state = false, { type, payload }) => {
   switch (type) {
     case clearGraph.TRIGGER:
       return false;
     case clearStructure.TRIGGER:
       return false;
+    case createGraph.SUCCESS:
+    case createGraph.FAILURE:
     case createGraph.FULFILL:
       return false;
     case createGraph.REQUEST:
-      return true;
+      return payload || true;
     case createStructureGraph.REQUEST:
       return true;
     case fetchGraph.FULFILL:
       return false;
     case fetchGraph.REQUEST:
-      return true;
+      return payload || true;
+    case fetchGraph.FAILURE:
+      return payload;
     case fetchStructure.TRIGGER:
       return false;
     case fetchStructureGraph.FULFILL:

package/src/reducers/graphRedirect.js

@@ -13,8 +13,7 @@ export const graphRedirect = (state = initialState, { type, payload }) => {
     case clearRedirect.TRIGGER:
       return initialState;
     case createGraph.SUCCESS:
-      const { data } = payload;
-      return linkTo.GRAPH(data);
+      return linkTo.GRAPH(payload?.data);
     default:
       return state;
   }

package/src/reducers/index.js

@@ -19,6 +19,8 @@ import { grantsPageSize } from "./grantsPageSize";
 import { graph } from "./graph";
 import { graphLoading } from "./graphLoading";
 import { graphRedirect } from "./graphRedirect";
+import { lineageEvents } from "./lineageEvents";
+import { lineageEventsLoading } from "./lineageEventsLoading";
 import { lineageQuery } from "./lineageQuery";
 import { linkedStructureTags } from "./linkedStructureTags";
 import { linkedStructureTagsLoading } from "./linkedStructureTagsLoading";
@@ -112,6 +114,8 @@ export {
   graph,
   graphLoading,
   graphRedirect,
+  lineageEvents,
+  lineageEventsLoading,
   lineageQuery,
   linkedStructureTags,
   linkedStructureTagsLoading,

package/src/reducers/lineageEvents.js

@@ -0,0 +1,13 @@
+import _ from "lodash/fp";
+import { fetchLineageEvents } from "../routines";
+
+export const initialState = [];
+
+export const lineageEvents = (state = initialState, { type, payload }) => {
+  switch (type) {
+    case fetchLineageEvents.SUCCESS:
+      return _.prop("data")(payload);
+    default:
+      return state;
+  }
+};

package/src/reducers/lineageEventsLoading.js

@@ -0,0 +1,14 @@
+import { fetchLineageEvents } from "../routines";
+
+const lineageEventsLoading = (state = false, { type }) => {
+  switch (type) {
+    case fetchLineageEvents.REQUEST:
+      return true;
+    case fetchLineageEvents.FULFILL:
+      return false;
+    default:
+      return state;
+  }
+};
+
+export { lineageEventsLoading };

package/src/routines.js

@@ -13,6 +13,7 @@ export const applyStructureUserFilter = createRoutine(
   "APPLY_STRUCTURE_USER_FILTER"
 );
 export const bulkUpdateStructures = createRoutine("BULK_UPDATE_STRUCTURES");
+export const cancelPoll = createRoutine("CANCEL_POLL");
 export const clearGrantFilters = createRoutine("CLEAR_GRANT_FILTERS");
 export const clearGrants = createRoutine("CLEAR_GRANTS");
 export const clearGrantsSort = createRoutine("CLEAR_GRANTS_SORT");
@@ -68,6 +69,7 @@ export const fetchGrant = createRoutine("FETCH_GRANT");
 export const fetchGrantFilters = createRoutine("FETCH_GRANT_FILTERS");
 export const fetchGrants = createRoutine("FETCH_GRANTS");
 export const fetchGraph = createRoutine("FETCH_GRAPH");
+export const fetchLineageEvents = createRoutine("FETCH_LINEAGE_EVENTS");
 export const fetchLinksStructureTags = createRoutine(
   "FETCH_LINKS_STRUCTURE_TAGS"
 );
@@ -93,6 +95,7 @@ export const fetchUnitDomains = createRoutine("FETCH_UNIT_DOMAINS");
 export const filterStructures = createRoutine("FILTER_STRUCTURES");
 export const hoverNode = createRoutine("HOVER_NODE");
 export const initSvg = createRoutine("INIT_SVG");
+export const loadLineageEvents = createRoutine("LOAD_LINEAGE_EVENTS");
 export const linkStructureTag = createRoutine("LINK_STRUCTURE_TAG");
 export const openGrantFilter = createRoutine("OPEN_GRANT_FILTER");
 export const openStructureFilter = createRoutine("OPEN_STRUCTURE_FILTER");

package/src/sagas/__tests__/createGraph.spec.js

@@ -1,13 +1,48 @@
+import { compile } from "path-to-regexp";
 import { takeLatest } from "redux-saga/effects";
 import { testSaga } from "redux-saga-test-plan";
-import { apiJsonPost, JSON_OPTS } from "@truedat/core/services/api";
+import { apiJson, apiJsonPost, JSON_OPTS } from "@truedat/core/services/api";
 import {
   createGraphRequestSaga,
   createGraphSaga,
-  selectLineageQuery
+  polling,
+  POLLING_CANCELLED,
+  selectLineageQuery,
 } from "../createGraph";
-import { createGraph, excludeNode } from "../../routines";
-import { API_GRAPHS } from "../../api";
+import {
+  cancelPoll,
+  createGraph,
+  fetchGraph,
+  excludeNode,
+} from "../../routines";
+import { API_GRAPHS, API_GRAPH_HASH } from "../../api";
+
+const graphAcceptedData = {
+  graph_hash: "CUYAfW8NAUhqDByfK1dSOBnKcv-rE2CjLtqoIKFMWoc=",
+  status: "just_started",
+  task_reference: "0.3951494048.1635778562.126161",
+};
+
+const graphPendingData = {
+  graph_data: '["BASILEAII_HIST_PROA_MOROSIDAD [ 2 ] [ 1 ].SOBREGIROCAPITAL"]',
+  graph_hash: "CUYAfW8NAUhqDByfK1dSOBnKcv-rE2CjLtqoIKFMWoc=",
+  inserted_at: "2021-11-25T16:03:29.145893",
+  status: "already_started",
+  task_reference: "0.3951494048.1635778562.126161",
+};
+
+const graphData = {
+  hash: "CUYAfW8NAUhqDByfK1dSOBnKcv-rE2CjLtqoIKFMWoc=",
+  id: 1692,
+  excludes: [],
+  groups: [],
+  ids: ["BASILEAII_HIST_PROA_MOROSIDAD [ 2 ] [ 1 ].SOBREGIROCAPITAL"],
+  opts: {
+    type: "lineage",
+  },
+  paths: [],
+  resources: [],
+};
 
 describe("sagas: createGraphRequestSaga", () => {
   it("should invoke createGraphSaga on trigger", () => {
@@ -16,7 +51,7 @@ describe("sagas: createGraphRequestSaga", () => {
         .next()
         .all([
           takeLatest(createGraph.TRIGGER, createGraphSaga),
-          takeLatest(excludeNode.TRIGGER, createGraphSaga)
+          takeLatest(excludeNode.TRIGGER, createGraphSaga),
         ])
         .finish()
         .isDone();
@@ -33,20 +68,57 @@ describe("sagas: createGraphRequestSaga", () => {
 });
 
 describe("sagas: createGraphSaga", () => {
-  const query = { type: "lineage", ids: ["foo"], excludes: [] };
-  const data = { foo: "bar" };
+  const query = {
+    ids: ["BASILEAII_HIST_PROA_MOROSIDAD [ 2 ] [ 1 ].SOBREGIROCAPITAL"],
+    type: "lineage",
+    exclude: [],
+  };
+
+  const graphAccepted = {
+    data: graphAcceptedData,
+    status: 202,
+  };
+
+  const graphCreated = {
+    data: graphData,
+    status: 201,
+  };
 
-  it("should put a success action when uuids are requested and a response is returned", () => {
+  it("should put a success action when uuids are requested and a 201 (created) graph data response is returned", () => {
     expect(() => {
-      testSaga(createGraphSaga, {})
+      testSaga(createGraphSaga)
         .next()
         .select(selectLineageQuery)
         .next(query)
         .put(createGraph.request(query))
         .next()
         .call(apiJsonPost, API_GRAPHS, query, JSON_OPTS)
-        .next({ data })
-        .put(createGraph.success(data))
+        .next(graphCreated)
+        .put(createGraph.success(graphCreated))
+        .next()
+        .put(createGraph.fulfill())
+        .next()
+        .isDone();
+    }).not.toThrow();
+  });
+
+  it("should fork a polling task when uuids are requested and a 202 (accepted) graph hash response is returned. Then, wait for task cancellation.", () => {
+    expect(() => {
+      testSaga(createGraphSaga)
+        .next()
+        .select(selectLineageQuery)
+        .next(query)
+        .put(createGraph.request(query))
+        .next()
+        .call(apiJsonPost, API_GRAPHS, query, JSON_OPTS)
+        .next(graphAccepted)
+        .put(fetchGraph.request(graphAccepted))
+        .next()
+        .fork(polling, graphAccepted)
+        .next()
+        .take(cancelPoll.TRIGGER)
+        .next()
+        .cancel()
         .next()
         .put(createGraph.fulfill())
         .next()
@@ -59,8 +131,8 @@ describe("sagas: createGraphSaga", () => {
     const error = { message };
 
     expect(() => {
-      testSaga(createGraphSaga, {})
-        .next(query)
+      testSaga(createGraphSaga)
+        .next()
         .select(selectLineageQuery)
         .next(query)
         .put(createGraph.request(query))
@@ -75,3 +147,80 @@ describe("sagas: createGraphSaga", () => {
     }).not.toThrow();
   });
 });
+
+describe("generator: polling", () => {
+  const graphAccepted = {
+    data: graphAcceptedData,
+    status: 202,
+  };
+
+  const graphPending = {
+    data: graphPendingData,
+    status: 202,
+  };
+
+  const graphCompleted = {
+    data: graphData,
+    status: 200,
+  };
+
+  it("polling should keep requesting by graph hash until graph is received", () => {
+    expect(() => {
+      testSaga(polling, graphAccepted)
+        .next()
+        .delay(1000)
+        .next()
+        .call(
+          apiJson,
+          compile(API_GRAPH_HASH)({ hash: graphAccepted.data.graph_hash }),
+          JSON_OPTS
+        )
+        .next(graphPending)
+        .put(
+          fetchGraph.failure({
+            data: { graph_hash: graphAccepted.data.graph_hash, retry: 1 },
+          })
+        )
+        .next()
+        .cancelled()
+        .next(false)
+        .delay(1200)
+        .next()
+        .call(
+          apiJson,
+          compile(API_GRAPH_HASH)({ hash: graphAccepted.data.graph_hash }),
+          JSON_OPTS
+        )
+        .next(graphCompleted)
+        .cancelled()
+        .next(false)
+        .put(createGraph.success(graphCompleted))
+        .next()
+        .isDone();
+    }).not.toThrow();
+  });
+
+  it("cancel should stop polling and return createGraph failure", () => {
+    expect(() => {
+      testSaga(polling, graphAccepted)
+        .next()
+        .delay(1000)
+        .next()
+        .call(
+          apiJson,
+          compile(API_GRAPH_HASH)({ hash: graphAccepted.data.graph_hash }),
+          JSON_OPTS
+        )
+        .next(graphPending)
+        .put(
+          fetchGraph.failure({
+            data: { graph_hash: graphAccepted.data.graph_hash, retry: 1 },
+          })
+        )
+        .next()
+        .cancelled()
+        .next(true)
+        .put(createGraph.failure(POLLING_CANCELLED));
+    }).not.toThrow();
+  });
+});

package/src/sagas/createGraph.js

@@ -1,17 +1,63 @@
+/* eslint-disable fp/no-mutation */
+/* eslint-disable fp/no-let */
 import _ from "lodash/fp";
-import { all, call, put, select, takeLatest } from "redux-saga/effects";
-import { apiJsonPost, JSON_OPTS } from "@truedat/core/services/api";
-import { excludeNode, createGraph } from "../routines";
-import { API_GRAPHS } from "../api";
+import {
+  all,
+  call,
+  cancel,
+  cancelled,
+  fork,
+  put,
+  select,
+  take,
+  takeLatest,
+  delay,
+} from "redux-saga/effects";
+import { apiJson, apiJsonPost, JSON_OPTS } from "@truedat/core/services/api";
+import { compile } from "path-to-regexp";
+import { excludeNode, cancelPoll, createGraph, fetchGraph } from "../routines";
+import { API_GRAPHS, API_GRAPH_HASH } from "../api";
 
 export const selectLineageQuery = _.prop("lineageQuery");
 
+export const POLLING_CANCELLED = "Polling cancelled!";
+
+function* exponentialBackoff(base, power) {
+  while (true) {
+    yield Math.pow(base, power);
+    // eslint-disable-next-line fp/no-mutation, no-param-reassign
+    power++;
+  }
+}
+
 export function* createGraphSaga() {
+  let graphAcceptedOrCreated;
   try {
     const body = yield select(selectLineageQuery);
     yield put(createGraph.request(body));
-    const { data } = yield call(apiJsonPost, API_GRAPHS, body, JSON_OPTS);
-    yield put(createGraph.success(data));
+
+    /* graph is either new (accepted) or "created" (actually previously stored
+      from an earlier accepted one) */
+    graphAcceptedOrCreated = yield call(
+      apiJsonPost,
+      API_GRAPHS,
+      body,
+      JSON_OPTS
+    );
+
+    if (graphAcceptedOrCreated.status == 202) {
+      yield put(fetchGraph.request(graphAcceptedOrCreated));
+      // starts the task in the background
+      const pollingTask = yield fork(polling, graphAcceptedOrCreated);
+
+      // wait for the user stop action
+      yield take(cancelPoll.TRIGGER);
+      // user clicked stop. cancel the background task
+      // this will cause the forked bgSync task to jump into its finally block
+      yield cancel(pollingTask);
+    } else if (graphAcceptedOrCreated.status == 201) {
+      yield put(createGraph.success(graphAcceptedOrCreated));
+    }
   } catch (error) {
     if (error.response) {
       const { status, data } = error.response;
@@ -24,9 +70,56 @@ export function* createGraphSaga() {
   }
 }
 
+export function* polling(graphAccepted) {
+  const backoff = exponentialBackoff(1.2, 0);
+  let backoffTime,
+    retry = 0,
+    graphResp,
+    isCancelled = false;
+
+  do {
+    backoffTime = backoff.next().value * 1000;
+    yield delay(backoffTime);
+
+    try {
+      graphResp = yield call(
+        apiJson,
+        compile(API_GRAPH_HASH)({
+          hash: graphAccepted.data.graph_hash,
+        }),
+        JSON_OPTS
+      );
+    } catch (error) {
+      if (error.response) {
+        const { status, data } = error.response;
+        yield put(createGraph.failure({ status, data }));
+      } else {
+        yield put(createGraph.failure(error.message));
+      }
+    }
+
+    graphResp?.status == 202 &&
+      (yield put(
+        fetchGraph.failure({
+          data: {
+            graph_hash: graphAccepted.data.graph_hash,
+            retry: ++retry,
+          },
+        })
+      ));
+
+    isCancelled = yield cancelled();
+    isCancelled && (yield put(createGraph.failure(POLLING_CANCELLED)));
+  } while (!isCancelled && graphResp?.status == 202);
+
+  graphResp?.status == 200 && !isCancelled
+    ? yield put(createGraph.success(graphResp))
+    : yield put(createGraph.failure({}));
+}
+
 export function* createGraphRequestSaga() {
   yield all([
     takeLatest(createGraph.TRIGGER, createGraphSaga),
-    takeLatest(excludeNode.TRIGGER, createGraphSaga)
+    takeLatest(excludeNode.TRIGGER, createGraphSaga),
   ]);
 }

package/src/sagas/fetchGraph.js

@@ -11,6 +11,7 @@ export function* fetchGraphSaga({ payload }) {
     const url = toApiPath(payload);
     yield put(fetchGraph.request(payload));
     const { data } = yield call(apiJson, url, JSON_OPTS);
+
     yield put(fetchGraph.success(data));
   } catch (error) {
     if (error.response) {

package/src/sagas/fetchLineageEvents.js

@@ -0,0 +1,26 @@
+import { call, put, takeLatest } from "redux-saga/effects";
+import { apiJson, JSON_OPTS } from "@truedat/core/services/api";
+import { fetchLineageEvents } from "../routines";
+import { API_LINEAGE_EVENTS } from "../api";
+
+export function* fetchLineageEventsSaga() {
+  try {
+    const url = API_LINEAGE_EVENTS;
+    yield put(fetchLineageEvents.request());
+    const { data, headers } = yield call(apiJson, url, JSON_OPTS);
+    yield put(fetchLineageEvents.success({ data, headers }));
+  } catch (error) {
+    if (error.response) {
+      const { status, data } = error.response;
+      yield put(fetchLineageEvents.failure({ status, data }));
+    } else {
+      yield put(fetchLineageEvents.failure(error.message));
+    }
+  } finally {
+    yield put(fetchLineageEvents.fulfill());
+  }
+}
+
+export function* fetchLineageEventsRequestSaga() {
+  yield takeLatest(fetchLineageEvents.TRIGGER, fetchLineageEventsSaga);
+}