@opengis/bi 1.2.0 → 1.2.2

package/server/plugins/docs.js

@@ -0,0 +1,48 @@
+import path, { dirname } from 'path';
+import { fileURLToPath } from 'url';
+import fs from 'fs';
+
+const dir = dirname(fileURLToPath(import.meta.url));
+const root = `${dir}/../../`;
+
+async function plugin(fastify, opts) {
+  fastify.get('/docs*', async (req, reply) => {
+    if (!fs.existsSync(path.join(root, 'docs/.vitepress/dist/'))) {
+      return reply.status(404).send('docs not exists');
+    }
+
+    const { params } = req;
+    const url = params['*'];
+
+    const filePath =
+      url && url[url.length - 1] !== '/'
+        ? path.join(root, 'docs/.vitepress/dist/', url)
+        : path.join(root, 'docs/.vitepress/dist/', url, 'index.html');
+
+    if (!fs.existsSync(filePath)) {
+      return reply.status(404).send('File not found');
+    }
+
+    const ext = path.extname(filePath);
+    const mime = {
+      '.js': 'text/javascript',
+      '.css': 'text/css',
+      '.woff2': 'application/font-woff',
+      '.png': 'image/png',
+      '.svg': 'image/svg+xml',
+      '.jpg': 'image/jpg',
+      '.html': 'text/html',
+      '.json': 'application/json',
+      '.pdf': 'application/pdf',
+    }[ext];
+
+    const stream = fs.createReadStream(filePath);
+    stream.on('error', (err) => {
+      reply.status(500).send('Error reading file');
+    });
+
+    return mime ? reply.type(mime).send(stream) : reply.send(stream);
+  });
+}
+
+export default plugin;

package/server/plugins/hook.js

@@ -0,0 +1,89 @@
+import fp from 'fastify-plugin';
+import fs from 'fs';
+
+// the use of fastify-plugin is required to be able
+// to export the decorators to the outer scope
+
+async function plugin(fastify) {
+  // preSerialization
+  fastify.addHook('preSerialization', async (req, reply, payload) => {
+    if (!req.session?.passport?.user?.uid) {
+      // return reply.redirect('/login');
+    }
+    if (req.url.includes('/suggest/') && !req.query.json) {
+      return payload?.data;
+    }
+    if (payload.redirect) {
+      return reply.redirect(payload.redirect);
+    }
+    if (reply.sent) {
+      return null;
+    }
+
+    if ([200, 400, 403, 409, 404, 500].includes(payload.status)) {
+      reply.status(payload.status);
+    }
+    /* if (payload.headers) {
+          reply.headers(payload.headers);
+        } */
+    if (payload.buffer) {
+      return payload.buffer;
+    }
+    if (payload.file) {
+      // const buffer = await readFile(payload.file);
+      // return reply.send(buffer);
+      const stream = fs.createReadStream(payload.file);
+      return stream;
+      // return reply.send(stream);
+    }
+
+    if (payload.message) {
+      return payload.message;
+    }
+    return payload;
+  });
+
+  //  preValidation
+  fastify.addHook('preValidation', async (req) => {
+    const parseRawBody =
+      ['POST', 'PUT'].includes(req.method) &&
+      req.body &&
+      typeof req.body === 'string' &&
+      req.body.trim(/\r\n/g).startsWith('{') &&
+      req.body.trim(/\r\n/g).endsWith('}');
+    if (parseRawBody) {
+      try {
+        req.body = JSON.parse(req.body || '{}');
+      } catch (err) {
+        // throw new Error('invalid body');
+        // return { error: 'invalid body', status: 400 };
+      }
+    }
+  });
+
+  // allow upload file
+  const kIsMultipart = Symbol.for('[FastifyMultipart.isMultipart]');
+  fastify.addContentTypeParser('multipart', (request, _, done) => {
+    request[kIsMultipart] = true;
+    done(null);
+  });
+
+  // parse Body
+  function contentParser(req, body, done) {
+    const parseBody = decodeURIComponent(body.toString())
+      .split('&')
+      .reduce((acc, el) => {
+        const [key, val] = el.split('=');
+        return { ...acc, [key]: val };
+      }, {});
+    done(null, parseBody);
+  }
+
+  fastify.addContentTypeParser(
+    'application/x-www-form-urlencoded',
+    { parseAs: 'buffer' },
+    contentParser
+  );
+}
+
+export default fp(plugin);

package/server/plugins/vite.js

@@ -0,0 +1,69 @@
+import fs from 'fs';
+import path from 'path';
+import config from '../../config.js';
+
+const { disableAuth } = config;
+const isProduction = process.env.NODE_ENV === 'production';
+
+async function plugin(fastify) {
+  // vite server
+  if (!isProduction) {
+    const vite = await import('vite');
+
+    const viteServer = await vite.createServer({
+      server: {
+        middlewareMode: true,
+      },
+    });
+    // hot reload
+    viteServer.watcher.on('all', (d, t) => {
+      if (!t.includes('module') && !t.includes('templates')) return;
+      // console.log(d, t);
+      viteServer.ws.send({ type: 'full-reload' });
+    });
+
+    // this is middleware for vite's dev servert
+    fastify.addHook('onRequest', async (req, reply) => {
+      // const { user } = req.session?.passport || {};
+      const next = () => new Promise((resolve) => {
+        viteServer.middlewares(req.raw, reply.raw, () => resolve());
+      });
+      await next();
+    });
+    fastify.get('*', async () => {});
+    return;
+  }
+
+  // From Build
+  fastify.get('*', async (req, reply) => {
+    // console.log(disableAuth)
+    if (!req.user && !disableAuth) return reply.redirect('/login');
+    const stream = fs.createReadStream('dist/index.html');
+    return reply
+      .headers({ 'Cache-Control': 'public, no-cache' })
+      .type('text/html')
+      .send(stream);
+  });
+  fastify.get('/assets/:file', async (req, reply) => {
+    const stream = fs.createReadStream(`dist/assets/${req.params.file}`);
+    const ext = path.extname(req.params.file);
+    const mime = {
+      '.js': 'text/javascript',
+      '.css': 'text/css',
+      '.woff2': 'application/font-woff',
+      '.png': 'image/png',
+    }[ext];
+    // reply.cacheControl('max-age', '1d');
+    return mime
+      ? reply
+        .headers({
+          'Cache-Control': 'public, max-age=3600',
+          'Content-Encoding': 'identity',
+        })
+        .type(mime)
+        .send(stream)
+      : stream;
+  });
+}
+
+export default plugin;

package/server/routes/dashboard/controllers/dashboard.import.js

@@ -0,0 +1,103 @@
+import path from 'node:path';
+
+import {
+    config, logger, pgClients, getTemplate, getTemplatePath, dataInsert, dataUpdate,
+} from '@opengis/fastify-table/utils.js';
+
+export default async function dashboardImport({ pg = pgClients.client, query = {}, user = {} }, reply) {
+    const time = Date.now();
+
+    if (!query?.dashboard) {
+        return reply.status(400).send('not enough query params: dashboard');
+    }
+
+    const list = getTemplatePath('dashboard').filter(el => el[0] === query.dashboard);
+
+    if (!list?.length) {
+        return reply.status(404).send(`dashboard not found`);
+    }
+
+    const client = await pg.connect();
+    Object.assign(client, {
+        options: pg.options,
+        pk: pg.pk,
+        pgType: pg.pgType,
+        tlist: pg.tlist,
+    });
+
+    const ids = [];
+    try {
+        await client.query('BEGIN');
+        const rows = await Promise.all(list.map(async ([filename]) => {
+
+            const obj = await getTemplate('dashboard', filename);
+            const index = obj?.find((el) => el[0] === 'index.yml')?.[1];
+            const { db = pg.options?.database, description, filters, panels, table_name, title, tags, style, grid } = index || {};
+
+            const dashboardId = await dataInsert({
+                pg,
+                table: 'bi.dashboard',
+                data: {
+                    db,
+                    name: filename,
+                    description,
+                    filters,
+                    panels,
+                    table_name,
+                    title,
+                    words: tags?.join(','),
+                    style,
+                    grid,
+                    source: 'file',
+                },
+                uid: user?.uid,
+            }).then(el => el.rows?.[0]?.dashboard_id);
+            ids.push(dashboardId);
+
+            const widgetList = panels?.reduce((acc, curr) => {
+                curr.widgets?.forEach(item => acc.push(item));
+                if (curr.widget) acc.push(curr.widget);
+                return acc;
+            }, []).filter(el => el) || [];
+
+            const widgets = widgetList.length
+                ? await Promise.all(obj.filter(el => widgetList.includes(path.parse(el[0]).name)).map(async ([widget, widgetData]) => {
+                    const { ext, name } = path.parse(widget);
+                    const data = ext === '.yml' ? widgetData : { type: 'text', data: { text: widgetData } };
+                    Object.assign(data, { name, x: widgetData?.data?.x, table_name: widgetData?.data?.table, dashboard_id: dashboardId });
+
+                    const res = await dataInsert({
+                        pg: client,
+                        table: 'bi.widget',
+                        data,
+                        uid: user?.uid,
+                    }).then(el => el.rows?.[0] || {});
+                    return res;
+                }))
+                : [];
+            const test = await dataUpdate({
+                pg: client,
+                table: 'bi.dashboard',
+                id: dashboardId,
+                data: {
+                    widgets,
+                },
+                uid: user?.uid,
+            });
+            return { dashboardId, name: filename, widgets };
+        }));
+
+        await client.query('COMMIT');
+        return { time: Date.now() - time, rows };
+    } catch (err) {
+        await client.query('ROLLBACK');
+        if (ids.length) {
+            const { rowCount = 0 } = await pg.query('delete from bi.dashboard where dashboard_id = any($1)', [ids]);
+            console.log('ROLLBACK delete', rowCount, 'dashboard');
+            // const { rowCount: rowCount1 = 0 } = await pg.query('delete from bi.widget where dashboard_id = any($1)', [ids]);
+            // console.log('delete', rowCount1, 'widget');
+        }
+        logger.file('bi/dashboardImport/error', { error: err.toString(), stack: err.stack, ids });
+        return reply.status(500).send(config.debug ? err.toString() : 'import error');
+    }
+}

package/server/routes/dashboard/controllers/dashboard.js

@@ -0,0 +1,157 @@
+import yaml from 'js-yaml';
+import {
+  pgClients,
+  getTemplatePath,
+  getTemplate,
+  getPGAsync,
+  getMeta,
+} from '@opengis/fastify-table/utils.js';
+
+export default async function dashboard({
+  pg: pg1 = pgClients.client, params = {},
+}) {
+  const time = Date.now();
+  const { id } = params;
+
+  if (!id) {
+    return { message: 'not enough params: dashboard required', status: 400 };
+  }
+  const dashboards = getTemplatePath('dashboard');
+
+  const fileDashboard = dashboards.find((el) => el[0] === id);
+  if (!fileDashboard) {
+    const sql = `select title, description, table_name, panels, grid, widgets, filters, style, words, db, public 
+    from bi.dashboard where $1 in (dashboard_id, name)`;
+
+    const data = await pg1.query(sql, [id]).then(el => el.rows?.[0] || {});
+
+    let pg = pg1;
+    try {
+      pg = data.pg || (data.db ? await getPGAsync(data.db) : null) || pg1;
+    } catch (err) {
+      data.error = err.toString();
+    }
+
+    data.type = 'bd';
+    const { table_name: table } = data;
+
+    if (!table) {
+      return { message: 'not enough params: table required', status: 400 };
+    }
+
+    const loadTemplate = pg.pk?.['admin.doc_template'] ? await pg.query(
+      'select body from admin.doc_template where doc_type=5 and title=$1',
+      [table]
+    ).then(el => el.rows?.[0]?.body) : null;
+
+    const sqlList = loadTemplate?.sql ? loadTemplate?.sql
+      ?.filter?.(el => !el.disabled && el?.sql?.replace)
+      ?.map?.((el, i) => `left join lateral (${el.sql.replace(/limit 1/ig, '')}) t${i} on 1=1`)?.join?.(' ') : '';
+
+    const { fields = [] } = table && pg.pk?.[loadTemplate?.table || table] ? await pg.query(`select * from ${loadTemplate?.table || table} t ${sqlList || ''} limit 0`) : {};
+
+    data?.widgets?.forEach?.(el => {
+      const { style, data = {}, type, title, x, metrics } = el;
+      el.yml = yaml.dump({ title, type, data: { x, metrics, ...data }, style, })
+      // el.yml = yaml.dump({ style, data, type, title });
+    });
+
+    data.panels?.forEach?.(el => {
+      const { title, type } = data?.widgets?.find(item => item.name === el.widget) || {};
+      Object.assign(el, { title, type });
+    });
+
+    const meta = table ? await getMeta({ pg, table: loadTemplate?.table || table }) : {};
+    const columnIndexes = meta?.columns?.reduce((acc, curr, idx) => Object.assign(acc, { [curr.name]: idx }), {}) || [];
+
+    const columns = fields?.map(el => ({ name: el.name, title: meta?.columns[columnIndexes[el.name]]?.title || el.name, type: pg.pgType?.[el.dataTypeID] }));
+
+    return {
+      ...data || {},
+      widgets: data?.widgets?.filter?.(el => el?.widget_id) || [],
+      panels: data?.panels?.filter?.(el => el?.widget) || [],
+      geom: !meta?.geom,
+      error:
+        table && !pg.pk?.[loadTemplate?.table || table] ? `table pkey not found: ${loadTemplate?.table || table}` : undefined,
+      table_name: table,
+      templateTable: loadTemplate?.table,
+      time: Date.now() - time,
+      columns,
+    };
+  }
+
+  const fileData = await getTemplate('dashboard', id);
+  const index = fileData.find((el) => el[0] === 'index.yml')[1];
+
+  if (!index) {
+    return { message: `not found ${id}`, status: 404 };
+  }
+
+  const data = index;
+  data.type = 'file';
+  const { table } = data?.data || { table: data?.table_name };
+
+  let pg = pg1;
+  try {
+    pg = data.pg || (data.db ? await getPGAsync(data.db) : null) || pg1;
+  } catch (err) {
+    data.error = err.toString();
+  }
+
+  const { fields = [] } = table ? await pg.query(`select * from ${table} limit 0`) : {};
+
+  const meta = await getMeta({ pg: pg1, table });
+
+  const columns = meta?.columns
+    ?.filter(el => fields.map(field => field.name).includes(el.name))
+    ?.map(el => ({ name: el.name, title: el.title, type: pg.pgType?.[el.dataTypeID] }));
+
+  const checks = index?.filters?.filter((el) => el?.id && fields.map((el) => el?.name).includes(el?.id) && el?.type === 'Check');
+  if (checks?.length) {
+    await Promise.all(checks.map(async (el) => {
+      if (el?.data) {
+        const options = await getTemplate('cls', el.data);
+        Object.assign(el, { options });
+      } else if (index?.table_name || index?.table) {
+        const { rows = [] } = await pg.query(`select "${el.id}" as id, count(*) from ${index?.table_name || index?.table} group by "${el.id}"`);
+        Object.assign(el, { options: rows });
+      }
+    }));
+  }
+  const ranges = index?.filters?.filter((el) => el?.id && fields.map((el) => el?.name).includes(el?.id) && el?.type === 'Range');
+  if (ranges?.length) {
+    await Promise.all(ranges.map(async (el) => {
+      const rows = await pg.query(`select array[
+        percentile_cont(0) within group (order by ${el.id}), percentile_cont(0.25) within group (order by ${el.id}), 
+        percentile_cont(0.5) within group (order by ${el.id}), percentile_cont(0.75) within group (order by ${el.id}), 
+        percentile_cont(1.0) within group (order by ${el.id})
+      ] from ${index?.table_name || index?.table}`).then(el => el.rows?.[0]?.array || []);
+      Object.assign(el, { options: rows });
+    }));
+  }
+
+  // console.log(fileData)
+  const widgets = fileData
+    .filter((el) => el[0] !== 'index.yml')
+    .map((el) =>
+      el[1].data
+        ? {
+          name: el[0].split('.')[0],
+          type: el[1].type,
+          title: el[1].title,
+          style: el[1].style,
+          data: el[1].data,
+
+        }
+        : { name: el[0].split('.')[0], title: el[1] }
+    );
+
+  return {
+    ...data,
+    geom: !!meta?.geom,
+    table_name: table,
+    time: Date.now() - time,
+    columns,
+    widgets,
+  };
+}

package/server/routes/dashboard/controllers/dashboard.list.js

@@ -0,0 +1,40 @@
+import { pgClients, getTemplatePath, getTemplate } from '@opengis/fastify-table/utils.js';
+
+const q = `select dashboard_id as name, 'db' as type, title, description, table_name, words, public from bi.dashboard`;
+
+export default async function data({
+  pg = pgClients.client, query = {}, user = {}
+}) {
+  const time = Date.now();
+  const { type = 'file' } = query;
+
+  const data = getTemplatePath('dashboard');
+  const dir = type === 'file' ? await Promise.all(
+    data.map(async ([filename, filepath]) => {
+      const obj = await getTemplate('dashboard', filename);
+      const index = obj?.find?.((el) => el[0] === 'index.yml')?.[1];
+      const { table_name, description, title } = index || {};
+
+      return { name: filename, path: user?.user_type?.includes('admin') ? filepath : undefined, type: 'file', title, description, table_name };
+    })
+  ) : [];
+
+  const { rows = [] } = ['db', 'viewer'].includes(type) ? await pg.query(q) : {};
+
+  if (type === 'viewer') {
+    return {
+      time: Date.now() - time,
+      db: pg.options?.database,
+      rows: rows.filter(el => el.table_name && !el.table_name.startsWith('demo.')),
+    };
+  }
+
+  const list = dir.concat(rows);
+
+  const res = {
+    time: Date.now() - time,
+    db: pg.options?.database,
+    rows: list,
+  };
+  return res;
+}

package/server/routes/dashboard/controllers/utils/yaml.js

@@ -0,0 +1,11 @@
+import yaml from 'js-yaml';
+
+yaml.loadSafe = (yml) => {
+  try {
+    return yaml.load(yml);
+  } catch (err) {
+    return { error: err.toString() };
+  }
+};
+
+export default yaml;

package/server/routes/dashboard/index.mjs

@@ -0,0 +1,26 @@
+import dashboard from './controllers/dashboard.js';
+import dashboardList from './controllers/dashboard.list.js';
+import dashboardImport from './controllers/dashboard.import.js';
+
+const biSchema = {
+  type: 'object',
+  properties: {
+    querystring: {
+      widget: { type: 'string', pattern: '^([\\d\\w]+)$' },
+      dashboard: { type: 'string', pattern: '^([\\d\\w]+)$' },
+      list: { type: 'string', pattern: '^([\\d])$' },
+      sql: { type: 'string', pattern: '^([\\d])$' },
+    },
+    params: {
+      id: { type: 'string', pattern: '^([\\d\\w]+)$' },
+    },
+  },
+};
+
+const policy = ['public'];
+
+export default async function route(fastify) {
+  fastify.get(`/bi-dashboard/:id`, { config: { policy }, schema: biSchema }, dashboard);
+  fastify.get(`/bi-dashboard`, { config: { policy } }, dashboardList);
+  fastify.get(`/bi-dashboard-import`, { config: { policy } }, dashboardImport);
+}