@eeacms/volto-globalsearch 1.0.20 → 1.0.21

package/src/config/healthcheck.js

@@ -1,14 +1,235 @@
-import runRequest from '@eeacms/search/lib/runRequest';
-import buildRequest from '@eeacms/search/lib/search/query';
-import getInfo from '@eeacms/search/lib/getIndexInfo';
+import { runRequest, buildRequest } from '@eeacms/search';
+//import getInfo from '@eeacms/search/lib/getIndexInfo';
+
+import failed_scheduled_atempts_since_last_started from './healthcheck_queries/failed_scheduled_atempts_since_last_started.json';
+//import failed_site_since_last_started from './healthcheck_queries/failed_site_since_last_started.json';
+//import last_scheduled_indexing from './healthcheck_queries/last_scheduled_indexing.json';
+import last_scheduled_started_indexing from './healthcheck_queries/last_scheduled_started_indexing.json';
+import last_sync_task_since_last_start from './healthcheck_queries/last_sync_task_since_last_start.json';
+import latest_tasks_for_site from './healthcheck_queries/latest_tasks_for_site.json';
+import started_or_finished_site_since_last_started from './healthcheck_queries/started_or_finished_site_since_last_started.json';
 
 const default_documentCountThreshold = 60000;
 const default_queryTimeSecondsThreshold_OK = 2;
 const default_queryTimeSecondsThreshold_WARNING = 5;
-const default_indexUpdatedHoursThreshold_OK = 1;
-const default_indexUpdatedHoursThreshold_WARNING = 1;
+const default_failedSyncThreshold_WARNING = 5;
+const default_failedSyncThreshold_OK = 2;
+
+export function buildQuery(query, values) {
+  let q = JSON.stringify(query);
+  Object.keys(values).forEach(function (key, value) {
+    q = q.split('<' + key + '>').join(values[key]);
+  });
+  return JSON.parse(q);
+}
+
+async function executeQuery(q, appConfig, params = {}, callback) {
+  return new Promise(async (resolve, reject) => {
+    try {
+      params['index_name'] = 'status_' + appConfig['index_name'];
+      const query = buildQuery(q, params);
+      //console.log(JSON.stringify(query));
+      const resp = await runRequest(query, appConfig);
+      // console.log(JSON.stringify(resp.body));
+      resolve(callback(resp.body, params));
+    } catch (e) {
+      reject({ error: e.message });
+    }
+  });
+}
+
+export function getlastandnext_started_execution(body, params = {}) {
+  if (body.hits.total.value > 0) {
+    return {
+      last_started: body.hits.hits[0]._source.start_time_ts,
+      next_execution_date: body.hits.hits[0]._source.next_execution_date_ts,
+    };
+  } else {
+    throw new Error('no results');
+  }
+}
+
+export function getlastfailed_execution(body, params = {}) {
+  if (body.hits.total.value > 0) {
+    return {
+      last_started: body.hits.hits[0]._source.start_time_ts,
+      next_execution_date: body.hits.hits[0]._source.next_execution_date_ts,
+    };
+  } else {
+    throw new Error('no results');
+  }
+}
+
+export function getlastsynctaskssincestarted(body, params = {}) {
+  if (body.hits.total.value > 0) {
+    return {
+      sites: body.hits.hits[0]._source.sites,
+    };
+  } else {
+    throw new Error('no results');
+  }
+}
+
+export function getlastsuccessfultasks_for_site(body, params = {}) {
+  if (body.hits.total.value > 0) {
+    return true;
+  } else {
+    return false;
+  }
+}
+
+export function getlatesttasks_for_site(body, params = {}) {
+  if (body.hits.total.value > 0) {
+    let status = 'OK';
+    let i = 0;
+    while (true) {
+      const doc = body.hits.hits[i]._source;
+      if (doc.status === 'Finished') {
+        break;
+      }
+      i++;
+      if (i === params.THRESHOLD_WARNING) {
+        break;
+      }
+    }
+    if (i >= params.THRESHOLD_OK && i < params.THRESHOLD_WARNING) {
+      status = 'WARNING';
+    }
+    if (i >= params.THRESHOLD_WARNING) {
+      status = 'CRITICAL';
+    }
+    return status;
+  } else {
+    throw new Error('Failed to get info');
+  }
+}
 
-export default async function healthcheck(appConfig, query) {
+export async function getStatus(appConfig, params) {
+  return new Promise(async (resolve, reject) => {
+    let resp = 'OK';
+    let error = null;
+    // console.log('=======================================');
+    // console.log('STEP 1');
+    const step1 = await executeQuery(
+      last_scheduled_started_indexing,
+      appConfig,
+      {},
+      getlastandnext_started_execution,
+    );
+
+    // console.log(step1);
+
+    // const last_successful_schedule = step1.last_started;
+    let next_schedule = step1.next_execution_date;
+
+    const now = params.now || Date.now() - 60 * 1000;
+    if (now >= next_schedule) {
+      try {
+        // console.log('=======================================');
+        // console.log('STEP 2');
+        const step2 = await executeQuery(
+          failed_scheduled_atempts_since_last_started,
+          appConfig,
+          step1,
+          getlastfailed_execution,
+        );
+        next_schedule = step2.next_execution_date;
+        // console.log(step2);
+      } catch {
+        resp = 'CRITICAL';
+        error = 'Failed to get status info from elasticsearch';
+      }
+    }
+    if (error === null) {
+      if (now > next_schedule) {
+        resp = 'CRITICAL';
+        error = 'Airflow stopped indexing, no new schedules in the queue';
+      } else {
+        try {
+          const step3 = await executeQuery(
+            last_sync_task_since_last_start,
+            appConfig,
+            step1,
+            getlastsynctaskssincestarted,
+          );
+          // console.log(step3.sites);
+          const all_sites_status = {};
+          for (let i = 0; i < step3.sites.length; i++) {
+            try {
+              // console.log('=======================================');
+              // console.log('STEP 4');
+              // const step4 =
+              await executeQuery(
+                started_or_finished_site_since_last_started,
+                appConfig,
+                {
+                  site_name: step3.sites[i],
+                  last_started: step1['last_started'],
+                },
+                getlastsuccessfultasks_for_site,
+              );
+              all_sites_status[step3.sites[i]] = 'OK';
+              // console.log(step4);
+            } catch {
+              // console.log('=======================================');
+              // console.log('STEP 5');
+              const step5 = await executeQuery(
+                latest_tasks_for_site,
+                appConfig,
+                {
+                  site_name: step3.sites[i],
+                  last_started: step1['last_started'],
+                  THRESHOLD_WARNING: parseInt(
+                    params.FAILED_SYNC_THRESHOLD_WARNING,
+                  ),
+                  THRESHOLD_OK: parseInt(params.FAILED_SYNC_THRESHOLD_OK),
+                },
+                getlatesttasks_for_site,
+              );
+              all_sites_status[step3.sites[i]] = step5;
+            }
+          }
+          // console.log(all_sites_status);
+          const oks = [];
+          const warnings = [];
+          const criticals = [];
+          for (let i = 0; i < step3.sites.length; i++) {
+            if (all_sites_status[step3.sites[i]] === 'OK') {
+              oks.push(step3.sites[i]);
+            }
+            if (all_sites_status[step3.sites[i]] === 'WARNING') {
+              warnings.push(step3.sites[i]);
+            }
+            if (all_sites_status[step3.sites[i]] === 'CRITICAL') {
+              criticals.push(step3.sites[i]);
+            }
+            if (criticals.length > 0) {
+              error =
+                'Clusters with too many fails: ' +
+                criticals.concat(warnings).join(', ');
+              resp = 'CRITICAL';
+            } else {
+              if (warnings.length > 0) {
+                error =
+                  'Clusters with too many fails: ' +
+                  criticals.concat(warnings).join(', ');
+                resp = 'WARNING';
+              }
+            }
+          }
+        } catch {
+          error = 'Failed to get status info from elasticsearch';
+        }
+      }
+    }
+    const status = { status: resp };
+    if (error !== null) {
+      status.error = error;
+    }
+    resolve(status);
+  });
+}
+export default async function healthcheck(appConfig, params) {
   // is index ok?
   // return index update date
   // run default query, see number of results
@@ -21,9 +242,10 @@ export default async function healthcheck(appConfig, query) {
         documentCountThreshold,
         queryTimeSecondsThreshold_OK,
         queryTimeSecondsThreshold_WARNING,
-        indexUpdatedHoursThreshold_OK,
-        indexUpdatedHoursThreshold_WARNING,
-      } = query;
+        failedSyncThreshold_OK,
+        failedSyncThreshold_WARNING,
+        now,
+      } = params;
       documentCountThreshold =
         documentCountThreshold || default_documentCountThreshold;
       queryTimeSecondsThreshold_OK =
@@ -31,17 +253,30 @@ export default async function healthcheck(appConfig, query) {
       queryTimeSecondsThreshold_WARNING =
         queryTimeSecondsThreshold_WARNING ||
         default_queryTimeSecondsThreshold_WARNING;
-      indexUpdatedHoursThreshold_OK =
-        indexUpdatedHoursThreshold_OK || default_indexUpdatedHoursThreshold_OK;
-      indexUpdatedHoursThreshold_WARNING =
-        indexUpdatedHoursThreshold_WARNING ||
-        default_indexUpdatedHoursThreshold_WARNING;
+      failedSyncThreshold_OK =
+        failedSyncThreshold_OK || default_failedSyncThreshold_OK;
+      failedSyncThreshold_WARNING =
+        failedSyncThreshold_WARNING || default_failedSyncThreshold_WARNING;
 
-      const body_total = buildRequest({ filters: [] }, appConfig);
+      const airflow_params = {
+        FAILED_SYNC_THRESHOLD_WARNING: failedSyncThreshold_WARNING,
+        FAILED_SYNC_THRESHOLD_OK: failedSyncThreshold_OK,
+        now: now,
+      };
 
+      ///////////////////
+      const body_total = buildRequest({ filters: [] }, appConfig);
+      //console.log(body_total);
       const resp_total = await runRequest(body_total, appConfig);
       const total = resp_total.body.hits.total.value;
-      const total_status = total > documentCountThreshold ? 'Ok' : 'CRITICAL';
+      const total_status =
+        total > documentCountThreshold
+          ? { status: 'OK' }
+          : {
+              status: 'CRITICAL',
+              error:
+                'The number of documents in elasticsearch dropped drastically',
+            };
       const body_nlp = buildRequest(
         { filters: [], searchTerm: 'what is bise?' },
         appConfig,
@@ -57,42 +292,47 @@ export default async function healthcheck(appConfig, query) {
           });
         });
       });
+
       const elapsed_status =
         total_elapsed < queryTimeSecondsThreshold_OK
-          ? 'OK'
+          ? { status: 'OK' }
           : total_elapsed < queryTimeSecondsThreshold_WARNING
-          ? 'WARNING'
-          : 'CRITICAL';
-
-      const indexedAt = await getInfo(appConfig);
-      const currentDate = new Date();
-      const indexedDelta = (currentDate - indexedAt) / 36000000;
-      const indexed_status =
-        indexedDelta < indexUpdatedHoursThreshold_OK
-          ? 'OK'
-          : indexedDelta < indexUpdatedHoursThreshold_WARNING
-          ? 'WARNING'
-          : 'CRITICAL';
-
-      let status = 'OK';
-      if (elapsed_status === 'WARNING' || indexed_status === 'WARNING') {
-        status = 'WARNING';
+          ? { status: 'WARNING', error: 'Slow response from NLP' }
+          : { status: 'CRITICAL', error: 'Slow response from NLP' };
+
+      const airflow_status = await getStatus(appConfig, airflow_params);
+
+      let status = { status: 'OK' };
+      if (
+        elapsed_status.status === 'WARNING' ||
+        airflow_status.status === 'WARNING'
+      ) {
+        status = { status: 'WARNING' };
       }
       if (
-        total_status === 'CRITICAL' ||
-        elapsed_status === 'CRITICAL' ||
-        indexed_status === 'CRITICAL'
+        total_status.status === 'CRITICAL' ||
+        elapsed_status.status === 'CRITICAL' ||
+        airflow_status.status === 'CRITICAL'
       ) {
-        status = 'CRITICAL';
+        status = { status: 'CRITICAL' };
       }
-      resolve({
-        status: status,
-        documentCount: total_status,
-        queryTime: elapsed_status,
-        indexUpdated: indexed_status,
-      });
-    } catch {
-      reject({ status: 'Critical' });
+
+      const errors_list = [];
+      if (total_status.error) {
+        errors_list.push(total_status.error);
+      }
+      if (elapsed_status.error) {
+        errors_list.push(elapsed_status.error);
+      }
+      if (airflow_status.error) {
+        errors_list.push(airflow_status.error);
+      }
+      if (errors_list.length > 0) {
+        status.error = errors_list.join('\n');
+      }
+      resolve(status);
+    } catch (e) {
+      reject({ status: 'Critical', error: e.message });
     }
   });
 }

package/src/config/healthcheck.test.js

@@ -0,0 +1,300 @@
+import {
+  buildQuery,
+  getlastfailed_execution,
+  getlastandnext_started_execution,
+  getlastsynctaskssincestarted,
+  getlatesttasks_for_site,
+  getlastsuccessfultasks_for_site,
+  getStatus,
+} from './healthcheck';
+import healthcheck from './healthcheck';
+
+import { runRequest } from '@eeacms/search';
+
+import failed_scheduled_atempts_since_last_started_resp from './healthcheck_queries/failed_scheduled_atempts_since_last_started_resp.json';
+import last_scheduled_started_indexing_resp from './healthcheck_queries/last_scheduled_started_indexing_resp.json';
+import last_sync_task_since_last_start_resp from './healthcheck_queries/last_sync_task_since_last_start_resp.json';
+import latest_tasks_for_site_resp from './healthcheck_queries/latest_tasks_for_site_resp.json';
+import started_or_finished_site_since_last_started_resp from './healthcheck_queries/started_or_finished_site_since_last_started_resp.json';
+import empty_resp from './healthcheck_queries/empty_resp.json';
+const SLOTS = [
+  'aboveSearchInput',
+  'belowSearchInput',
+  'aboveResults',
+  'belowResults',
+];
+jest.mock('@eeacms/search', () => ({
+  SLOTS: SLOTS,
+  runRequest: jest.fn(),
+  buildRequest: jest.fn(),
+}));
+
+const query1 = {
+  query: {
+    bool: {
+      must: [],
+      must_not: [],
+      should: [],
+    },
+  },
+  from: 0,
+  size: 1,
+  index: '<index_name>',
+};
+
+const built_query1 = {
+  query: {
+    bool: {
+      must: [],
+      must_not: [],
+      should: [],
+    },
+  },
+  from: 0,
+  size: 1,
+  index: 'test_index',
+};
+
+const query2 = {
+  query: {
+    bool: {
+      must: [],
+      must_not: [],
+      should: [],
+    },
+  },
+  from: 0,
+  size: '<size>',
+};
+
+const built_query2 = {
+  query: {
+    bool: {
+      must: [],
+      must_not: [],
+      should: [],
+    },
+  },
+  from: 0,
+  size: '10',
+};
+
+const query3 = {
+  query: {
+    bool: {
+      must: [
+        {
+          terms: {
+            term_field: ['<site_name>'],
+          },
+        },
+      ],
+      must_not: [],
+      should: [],
+    },
+  },
+  from: 0,
+  size: '<size>',
+  sort: [
+    {
+      start_time_ts: 'desc',
+    },
+  ],
+  index: '<index_name>',
+};
+
+const built_query3 = {
+  query: {
+    bool: {
+      must: [
+        {
+          terms: {
+            term_field: ['test_site'],
+          },
+        },
+      ],
+      must_not: [],
+      should: [],
+    },
+  },
+  from: 0,
+  size: '10',
+  sort: [
+    {
+      start_time_ts: 'desc',
+    },
+  ],
+  index: 'test_index',
+};
+
+describe('test building the queries', () => {
+  it('should replace 1 string variable with the value in the query', () => {
+    const params = { index_name: 'test_index' };
+    const bQuery = buildQuery(query1, params);
+    expect(bQuery).toEqual(built_query1);
+  });
+
+  it('should replace 1 numeric variable with the value in the query', () => {
+    const params = { size: 10 };
+    const bQuery = buildQuery(query2, params);
+    expect(bQuery).toEqual(built_query2);
+  });
+
+  it('should replace more variables with the values in the query', () => {
+    const params = {
+      index_name: 'test_index',
+      site_name: 'test_site',
+      size: 10,
+    };
+    const bQuery = buildQuery(query3, params);
+    expect(bQuery).toEqual(built_query3);
+  });
+});
+
+describe('test parsing the response from elasticsearch for correct response', () => {
+  it('should return last_started and next_execution_date', () => {
+    const resp = getlastandnext_started_execution(
+      last_scheduled_started_indexing_resp,
+    );
+    expect(resp).toEqual({
+      last_started: 1695732613000,
+      next_execution_date: 1695732900000,
+    });
+  });
+
+  it('should return last_started and next_execution_date of failed task', () => {
+    const resp = getlastfailed_execution(
+      failed_scheduled_atempts_since_last_started_resp,
+    );
+    expect(resp).toEqual({
+      last_started: 1695732613000,
+      next_execution_date: 1695732900000,
+    });
+  });
+
+  it('should return list of clusters', () => {
+    const resp = getlastsynctaskssincestarted(
+      last_sync_task_since_last_start_resp,
+    );
+    expect(resp).toEqual({ sites: ['test_site1', 'test_site2'] });
+  });
+
+  it('if the last task for a site did not fail, return "OK"', async () => {
+    const resp = await getlatesttasks_for_site(latest_tasks_for_site_resp);
+    expect(resp).toEqual('OK');
+  });
+
+  it('if the last task for a site was succesful, return true', async () => {
+    const resp = getlastsuccessfultasks_for_site(
+      started_or_finished_site_since_last_started_resp,
+    );
+    expect(resp).toEqual(true);
+  });
+});
+
+describe('test parsing the response from elasticsearch for empty response', () => {
+  it('should return no results', () => {
+    try {
+      getlastandnext_started_execution(empty_resp);
+    } catch (e) {
+      expect(e.message).toEqual('no results');
+    }
+  });
+
+  it('should return no results', () => {
+    try {
+      getlastfailed_execution(empty_resp);
+    } catch (e) {
+      expect(e.message).toEqual('no results');
+    }
+  });
+
+  it('should return list of clusters', () => {
+    try {
+      getlastsynctaskssincestarted(empty_resp);
+    } catch (e) {
+      expect(e.message).toEqual('no results');
+    }
+  });
+
+  it('if the last task for a site did not fail, return "OK"', async () => {
+    try {
+      await getlatesttasks_for_site(empty_resp);
+    } catch (e) {
+      expect(e.message).toEqual('Failed to get info');
+    }
+  });
+
+  it("if the can't take the status for the last task for a site, return false", async () => {
+    const resp = getlastsuccessfultasks_for_site(empty_resp);
+    expect(resp).toEqual(false);
+  });
+});
+
+describe('test the status of the index', () => {
+  it('test', async () => {
+    runRequest
+      .mockReturnValueOnce(
+        Promise.resolve({ body: last_scheduled_started_indexing_resp }),
+      )
+      .mockReturnValueOnce(
+        Promise.resolve({
+          body: failed_scheduled_atempts_since_last_started_resp,
+        }),
+      )
+      .mockReturnValueOnce(
+        Promise.resolve({
+          body: last_sync_task_since_last_start_resp,
+        }),
+        // )
+        // .mockReturnValueOnce(
+        //   Promise.resolve({
+        //     body: started_or_finished_site_since_last_started_resp,
+        //   }),
+      );
+    const params = { index_name: 'test_index', now: 1695732000000 };
+    const status = await getStatus({}, params);
+    expect(status).toEqual({ status: 'OK' });
+  });
+});
+
+describe('test the healthcheck', () => {
+  it('test', async () => {
+    runRequest
+      .mockReturnValueOnce(
+        Promise.resolve({ body: { hits: { total: { value: 60001 } } } }),
+      )
+      .mockReturnValueOnce(
+        Promise.resolve({
+          body: {
+            elapsed: {
+              step1: [{ query: { delta: 0.5 } }],
+              step2: [{ query: { delta: 0.6 } }],
+            },
+          },
+        }),
+      )
+
+      .mockReturnValueOnce(
+        Promise.resolve({ body: last_scheduled_started_indexing_resp }),
+      )
+      .mockReturnValueOnce(
+        Promise.resolve({
+          body: failed_scheduled_atempts_since_last_started_resp,
+        }),
+      )
+      .mockReturnValueOnce(
+        Promise.resolve({
+          body: last_sync_task_since_last_start_resp,
+        }),
+      )
+      .mockReturnValueOnce(
+        Promise.resolve({
+          body: started_or_finished_site_since_last_started_resp,
+        }),
+      );
+    const params = { index_name: 'test_index', now: 1695732000000 };
+    const status = await healthcheck({}, params);
+    expect(status).toEqual({ status: 'OK' });
+  });
+});

package/src/config/healthcheck_queries/empty_resp.json

@@ -0,0 +1,18 @@
+{
+  "took": 18,
+  "timed_out": false,
+  "_shards": {
+    "total": 1,
+    "successful": 1,
+    "skipped": 0,
+    "failed": 0
+  },
+  "hits": {
+    "total": {
+      "value": 0,
+      "relation": "eq"
+    },
+    "max_score": null,
+    "hits": []
+  }
+}