@hestia-earth/data-api 0.0.2-3 → 0.0.2-4

package/src/lambdas/update-aggregated-nodes/handler.spec.ts

@@ -1,77 +0,0 @@
-import 'mocha';
-import sinon from 'sinon';
-import { of } from 'rxjs';
-import { expect } from 'chai';
-import * as s3 from '@hestia-earth/pipeline-utils/dist/s3';
-import { sortBy as _sortBy } from 'lodash';
-
-import { loadFixture, initDb } from '../../../test/utils';
-import { run } from './handler';
-import { query } from '../../../database';
-
-const stubs: sinon.SinonStub[] = [];
-
-const cycles = ['abyssinianKaleSeedWhole-cycle_pivoted.csv', 'tomatoFruit-cycle_pivoted.csv'];
-
-const impactAssessments = [
-  'abyssinianKaleSeedWhole-impactassessment_pivoted.csv',
-  'tomatoFruit-impactassessment_pivoted.csv'
-];
-
-const keys = [
-  ...cycles,
-  ...impactAssessments,
-  // abyssinianKaleStraw should be ignored as no ref to a cycle
-  'abyssinianKaleStraw-impactassessment_pivoted.csv',
-  // hould be ignored as cycle.id refers to a non-existent cycle
-  'cycle-missing-impactassessment_pivoted.csv'
-];
-
-const fixture = (fileName: string) => loadFixture(`update-aggregated-nodes/${fileName}`);
-
-const getJsonResults = (rows) =>
-  rows
-    .map((r) => r.jsonld_pivoted);
-
-describe('update-aggregated-nodes > run', () => {
-  beforeEach(async () => {
-    await initDb();
-    stubs.push(sinon.stub(s3, 'listFolder').resolves(keys.map((Key) => ({ Key }))));
-    stubs.push(sinon.stub(s3, 'getObject').callsFake(({ Key }) => of(fixture(Key)) as any));
-  });
-
-  afterEach(() => {
-    stubs.forEach((stub) => stub.restore());
-  });
-
-  it('should save the pivoted csv files in the db', async () => {
-    await run();
-
-    const { rows: siteRows } = await query('SELECT * FROM aggregated_sites');
-    const siteResult = _sortBy(getJsonResults(siteRows), '@id');
-    const jsonPivotedSites = _sortBy(
-      cycles.flatMap((key) => JSON.parse(fixture(`${key}.site.json`))),
-      '@id'
-    );
-
-    expect(siteResult.sort()).to.deep.equal(jsonPivotedSites.sort());
-
-    const { rows: cycleRows } = await query('SELECT * FROM aggregated_cycles');
-    const cycleResult = _sortBy(getJsonResults(cycleRows), '@id');
-    const jsonPivotedCycles = _sortBy(
-      cycles.flatMap((key) => JSON.parse(fixture(`${key}.cycle.json`))),
-      '@id'
-    );
-
-    expect(cycleResult).to.deep.equal(jsonPivotedCycles);
-
-    const { rows: iaRows } = await query('SELECT * FROM aggregated_impact_assessments');
-    const iaResult = _sortBy(getJsonResults(iaRows), '@id');
-    const jsonPivotedIas = _sortBy(
-      impactAssessments.flatMap((key) => JSON.parse(fixture(`${key}.impactAssessment.json`))),
-      '@id'
-    );
-
-    expect(iaResult.sort()).to.deep.equal(jsonPivotedIas.sort());
-  });
-});

package/src/lambdas/update-aggregated-nodes/handler.ts

@@ -1,129 +0,0 @@
-import { info, debug, timeSpent, listFolder, getObject, bucket } from '@hestia-earth/pipeline-utils';
-import csvtojson from 'csvtojson';
-import uniqBy from 'lodash.uniqby';
-import chunk from 'lodash.chunk';
-
-import { wrapHandler } from '../sentry';
-import { query } from '../../../database';
-
-// If a cycle or IA is missing its referenced site/cycle, it is out of date: omit
-// JOIN ensures only records with a valid fkey will be selected for insert
-const upsertCyclesQuery = `
-INSERT INTO aggregated_cycles (jsonld_pivoted)
-SELECT vals.jsonld_pivoted
-FROM (
-    SELECT * FROM unnest($1::jsonb[], $2::text[])
-) vals (jsonld_pivoted, site_hestia_id)
-JOIN aggregated_sites ON vals.site_hestia_id = aggregated_sites.hestia_id
-ON CONFLICT(hestia_id) DO UPDATE
-    SET jsonld_pivoted = EXCLUDED.jsonld_pivoted;
-`;
-
-const upsertImpactAsessmentsQuery = `
-INSERT INTO aggregated_impact_assessments (jsonld_pivoted)
-SELECT vals.jsonld_pivoted
-FROM (
-    SELECT * FROM unnest($1::jsonb[], $2::text[])
-) vals (jsonld_pivoted, cycle_hestia_id)
-JOIN aggregated_cycles ON vals.cycle_hestia_id = aggregated_cycles.hestia_id
-ON CONFLICT(hestia_id) DO UPDATE
-    SET jsonld_pivoted = EXCLUDED.jsonld_pivoted;
-`;
-
-const upsertSitesQuery = `
-INSERT INTO aggregated_sites (jsonld_pivoted)
-    VALUES(unnest($1::jsonb[]))
-ON CONFLICT(hestia_id) DO UPDATE
-    SET jsonld_pivoted = EXCLUDED.jsonld_pivoted;
-`;
-
-const foreignKeyGetters: { [key in AggregationTypes]: (node) => boolean } = {
-  cycle: (cycle) => cycle.site['@id'],
-  impactAssessment: (ia) => ia.cycle?.['@id'],
-  site: (_site) => true
-};
-
-const queries: { [key in AggregationTypes]: string } = {
-  cycle: upsertCyclesQuery,
-  impactAssessment: upsertImpactAsessmentsQuery,
-  site: upsertSitesQuery
-};
-
-enum AggregationTypes {
-  cycle = 'cycle',
-  impactAssessment = 'impactAssessment',
-  site = 'site'
-}
-
-const typesInsertionOrder = [AggregationTypes.site, AggregationTypes.cycle, AggregationTypes.impactAssessment];
-
-const formatJson = (mapObj, type: AggregationTypes) => (json) =>
-  mapObj.default(
-    json[type],
-    ((key, val) => {
-      return val === '-' ? mapObj.mapObjectSkip : [key === 'id' ? '@id' : key, val];
-    }) as any,
-    { deep: true }
-  );
-
-const getFormattedNodes = async (keys: string[]): Promise<{ [key in AggregationTypes]: any[] }> => {
-  const mapObj = await import('map-obj');
-
-  const jsons = (
-    await Promise.all(
-      keys.map(async (Key) => {
-        const body = await getObject({ Key, Bucket: bucket }).toPromise();
-        return await csvtojson({ delimiter: 'auto', checkType: true, ignoreEmpty: true }).fromString(body);
-      })
-    )
-  ).flat();
-
-  const formattedNodes = { cycle: [], site: [], impactAssessment: [] };
-  // Each json represents a row of csv, with top-level keys 'cycle', 'impactAssessment', 'site'
-  for (const type of typesInsertionOrder) {
-    const formattedType = uniqBy(
-      jsons
-        .filter((json) => json[type])
-        .map(formatJson(mapObj, type))
-        .filter((node) => node['@id'] && foreignKeyGetters[type](node)),
-      '@id'
-    );
-
-    formattedNodes[type].push(...formattedType);
-  }
-
-  return formattedNodes;
-};
-
-export const run = async () => {
-  const folder = 'backups/aggregation';
-  const allKeys = (await listFolder(folder)).map(({ Key }) => Key);
-  debug('Found', allKeys.length, 'files in', folder, 'folder on bucket', bucket);
-
-  // cycles must be processed/inserted before impact assessments because of fkey constraints
-  const pivoted = [
-    ...allKeys.filter((Key) => Key.endsWith('cycle_pivoted.csv')),
-    ...allKeys.filter((Key) => Key.endsWith('impactassessment_pivoted.csv'))
-  ];
-  debug(pivoted.length, 'pivoted files to process');
-
-  const chunks = chunk(pivoted, 500);
-  for (const value of chunks) {
-    const nodes = await getFormattedNodes(value);
-    for (const type of typesInsertionOrder) {
-      const params =
-        type === AggregationTypes.site ? [nodes[type]] : [nodes[type], nodes[type].map(foreignKeyGetters[type])];
-      await query(queries[type], params);
-    }
-  }
-};
-
-export const updateAggregatedNodes = async () => {
-  const functionName = 'updateAggregatedNodes';
-  info(functionName);
-  const now = new Date();
-  await run();
-  info(`Finished ${functionName} in ${timeSpent(now)}ms`);
-};
-
-export const updateAggregatedNodesHandler = wrapHandler(updateAggregatedNodes);

package/src/logger.spec.ts

@@ -1,20 +0,0 @@
-import { expect } from 'chai';
-import * as sinon from 'sinon';
-import 'mocha';
-
-import { logger, stream } from './logger';
-
-describe('logger', () => {
-  it('should have 1 transport', () => {
-    expect(logger.transports.length).to.equal(1);
-  });
-});
-
-describe('stream', () => {
-  it('should use logger info', () => {
-    const stub: sinon.SinonStub = sinon.stub(logger, 'info');
-    stream.write('message');
-    expect(stub.calledWith('message')).to.equal(true);
-    stub.restore();
-  });
-});

package/src/logger.ts

@@ -1,45 +0,0 @@
-import { createLogger, format, transports } from 'winston';
-
-const logFilename = process.env.LOG_FILENAME;
-
-const options = {
-  file: {
-    level: (process.env.LOG_LEVEL || 'info').toLowerCase(),
-    filename: logFilename,
-    handleExceptions: true,
-    maxsize: 5242880, // 5MB
-    maxFiles: 5,
-    colorize: false,
-    format: format.combine(
-      format.timestamp(),
-      format.metadata({ fillExcept: ['message', 'level', 'timestamp'] }),
-      format.json()
-    )
-  },
-  console: {
-    level: 'debug',
-    handleExceptions: true,
-    format: format.combine(
-      format.colorize(),
-      format.timestamp(),
-      format.printf((info) => {
-        return `${info.timestamp} - ${info.level}: ${info.message}${
-          Object.keys(info.metadata).length === 0 ? '' : ' ' + JSON.stringify(info.metadata)
-        }`;
-      })
-    )
-  }
-};
-
-export const logger = createLogger({
-  format: options.file.format,
-  transports: [...(logFilename ? [new transports.File(options.file)] : []), new transports.Console(options.console)],
-  exitOnError: false
-});
-
-export const stream = {
-  write: (message) => {
-    // use the 'info' log level so the output will be picked up by both transports (file and console)
-    logger.info(message);
-  }
-};

package/src/maintenance.spec.ts

@@ -1,76 +0,0 @@
-import { expect } from 'chai';
-import * as sinon from 'sinon';
-import 'mocha';
-
-import { maintenance, clearCachedMaintenanceSetting } from './maintenance';
-import * as getService from './settings/services/get';
-import { updateAll } from './settings/services/update';
-import { initDb } from '../test/utils';
-import { SettingKey } from './settings/model';
-import * as updateService from './settings/services/update';
-
-let stubs: sinon.SinonStub[] = [];
-
-describe('maintenance', () => {
-  let nextStub: sinon.SinonStub;
-  let isEnabledStub: sinon.SinonStub;
-
-  beforeEach(() => {
-    stubs = [];
-    nextStub = sinon.stub();
-    clearCachedMaintenanceSetting();
-  });
-
-  afterEach(() => {
-    stubs.forEach((stub) => stub.restore());
-  });
-
-  describe('maintenance', () => {
-    const req: any = {};
-
-    describe('with maintenance setting cache', () => {
-      beforeEach(async () => {
-        await initDb();
-        stubs.push((isEnabledStub = sinon.stub(getService, 'isEnabled').callThrough()));
-        stubs.push(sinon.stub(updateService, 'postMessage').resolves());
-      });
-
-      it('returns the cached value on second call', async () => {
-        await maintenance(req, null, nextStub);
-        await maintenance(req, null, nextStub);
-        expect(isEnabledStub.calledOnce).to.equal(true);
-        expect(nextStub.args).to.deep.equal([[], []]);
-      });
-
-      it('refreshes cache when maintenance setting updated', async () => {
-        await maintenance(req, null, nextStub);
-        await updateAll({ [SettingKey.maintenanceEnabled]: false });
-        await maintenance(req, null, nextStub);
-        expect(isEnabledStub.calledTwice).to.equal(true);
-        expect(nextStub.args).to.deep.equal([[], []]);
-      });
-    });
-
-    describe('maintenance enabled', () => {
-      beforeEach(() => {
-        stubs.push(sinon.stub(getService, 'isEnabled').resolves(true));
-      });
-
-      it('should throw an error', async () => {
-        await maintenance(req, null, nextStub);
-        expect(nextStub.calledWithExactly()).to.equal(false);
-      });
-    });
-
-    describe('maintenance disabled', () => {
-      beforeEach(() => {
-        stubs.push(sinon.stub(getService, 'isEnabled').resolves(false));
-      });
-
-      it('should NOT throw an error', async () => {
-        await maintenance(req, null, nextStub);
-        expect(nextStub.calledWithExactly()).to.equal(true);
-      });
-    });
-  });
-});

package/src/maintenance.ts

@@ -1,19 +0,0 @@
-import { Request, Response, NextFunction } from 'express';
-import { HestiaError, Errors, StatusCodes } from './errors';
-import { SettingKey } from './settings/model';
-import { isEnabled } from './settings/services/get';
-
-const cachedValue: { enabled?: boolean } = {};
-
-export const maintenance = async (_req: Request, _res: Response, next: NextFunction) => {
-  if (!cachedValue.hasOwnProperty('enabled')) {
-    cachedValue.enabled = await isEnabled(SettingKey.maintenanceEnabled);
-  }
-  return cachedValue.enabled
-    ? next(new HestiaError(Errors.UnderMaintenance, {}, StatusCodes.ServiceUnavailable))
-    : next();
-};
-
-export const clearCachedMaintenanceSetting = () => {
-  delete cachedValue.enabled;
-};

package/src/models.ts

@@ -1 +0,0 @@
-export * from './aggregated-nodes/model';

package/src/routes.ts

@@ -1,8 +0,0 @@
-import { Express } from 'express';
-import aggregatedNodes from './aggregated-nodes/routes';
-
-const routes = (app: Express) => {
-  app.use('/aggregated-nodes', aggregatedNodes());
-};
-
-export default routes;

package/src/settings/model/index.ts

@@ -1,21 +0,0 @@
-import { query } from '../../../database';
-import { clearCachedMaintenanceSetting } from '../../maintenance';
-
-export enum SettingKey {
-  maintenanceEnabled = 'maintenanceEnabled'
-}
-
-export const getSetting = (setting: SettingKey) =>
-  query('SELECT setting, active, metadata FROM settings WHERE setting = $1', [setting]);
-
-export const getAllSettings = () => query('SELECT setting, active, metadata FROM settings');
-
-export const setSetting = async (setting: { key: SettingKey; value: boolean; metadata?: any }) => {
-  const result = await query('UPDATE settings SET active = $2, metadata = (metadata || $3) WHERE setting = $1', [
-    setting.key,
-    setting.value,
-    setting.metadata || {}
-  ]);
-  clearCachedMaintenanceSetting();
-  return result;
-};

package/src/settings/routes/get.spec.ts

@@ -1,33 +0,0 @@
-import { expect } from 'chai';
-import * as sinon from 'sinon';
-import 'mocha';
-
-import { get } from './get';
-import * as serviceGet from '../services/get';
-
-let stubs: sinon.SinonStub[] = [];
-
-describe('settings > routes', () => {
-  beforeEach(() => {
-    stubs = [];
-  });
-
-  afterEach(() => {
-    stubs.forEach((stub) => stub.restore());
-  });
-
-  describe('get', () => {
-    describe('get', () => {
-      let getStub: sinon.SinonStub;
-
-      beforeEach(() => {
-        stubs.push((getStub = sinon.stub(serviceGet, 'getAll').resolves({})));
-      });
-
-      it('should call get', async () => {
-        await get();
-        expect(getStub.called).to.equal(true);
-      });
-    });
-  });
-});

package/src/settings/routes/get.ts

@@ -1,3 +0,0 @@
-import { getAll } from '../services/get';
-
-export const get = () => getAll();

package/src/settings/routes/update.spec.ts

@@ -1,33 +0,0 @@
-import { expect } from 'chai';
-import * as sinon from 'sinon';
-import 'mocha';
-
-import { update } from './update';
-import * as serviceUpdate from '../services/update';
-
-let stubs: sinon.SinonStub[] = [];
-
-describe('settings > routes', () => {
-  beforeEach(() => {
-    stubs = [];
-  });
-
-  afterEach(() => {
-    stubs.forEach((stub) => stub.restore());
-  });
-
-  describe('update', () => {
-    describe('update', () => {
-      let setStub: sinon.SinonStub;
-
-      beforeEach(() => {
-        stubs.push((setStub = sinon.stub(serviceUpdate, 'updateAll').resolves([])));
-      });
-
-      it('should call update', async () => {
-        await update({} as any);
-        expect(setStub.called).to.equal(true);
-      });
-    });
-  });
-});

package/src/settings/routes/update.ts

@@ -1,5 +0,0 @@
-import { Request } from 'express';
-
-import { updateAll } from '../services/update';
-
-export const update = ({ body }: Request) => updateAll(body);

package/src/settings/routes.spec.ts

@@ -1,75 +0,0 @@
-import { expect } from 'chai';
-import * as sinon from 'sinon';
-import 'mocha';
-import express from 'express';
-import supertest from 'supertest';
-import bodyParser from 'body-parser';
-
-import router from './routes';
-import { errorHandler } from '../errors';
-import { initDb } from '../../test/utils';
-import * as update from './services/update';
-
-let stubs: sinon.SinonStub[] = [];
-
-const app = express();
-
-app.use(bodyParser.json());
-app.use(router());
-app.use(errorHandler);
-
-let postMessageStub: sinon.SinonStub;
-
-describe('routes', () => {
-  beforeEach(() => {
-    stubs = [];
-    stubs.push((postMessageStub = sinon.stub(update, 'postMessage').resolves()));
-  });
-
-  afterEach(() => {
-    stubs.forEach((stub) => stub.restore());
-  });
-
-  describe('settings', () => {
-    beforeEach(async () => {
-      await initDb();
-    });
-
-    describe('GET /', () => {
-      it('returns all settings', async () => {
-        const result = await supertest(app).get('/').set('Accept', 'application/json');
-        expect(result.body).to.deep.equal({ maintenanceEnabled: false });
-      });
-    });
-
-    describe('PUT /', () => {
-      describe('with correct token', () => {
-        it('updates the passed settings and calls postMessage', async () => {
-          await supertest(app)
-            .put('/')
-            .set('Accept', 'application/json')
-            .set('x-access-token', 'dummy')
-            .type('json')
-            .send({ maintenanceEnabled: true })
-            .expect(200);
-
-          const result = await supertest(app).get('/').set('Accept', 'application/json');
-          expect(result.body).to.deep.equal({ maintenanceEnabled: true });
-          expect(postMessageStub.called).to.equal(true);
-        });
-      });
-
-      describe('with incorrect token', () => {
-        it('returns Unauthorized error', async () => {
-          await supertest(app)
-            .put('/')
-            .set('Accept', 'application/json')
-            .set('x-access-token', '')
-            .type('json')
-            .send({ maintenanceEnabled: true })
-            .expect(401);
-        });
-      });
-    });
-  });
-});

package/src/settings/routes.ts

@@ -1,21 +0,0 @@
-import { Request, Response, NextFunction, Router } from 'express';
-
-import { endpointWrapper } from '../utils/endpoint-wrapper';
-import { get } from './routes/get';
-import { update } from './routes/update';
-import { HestiaError, Errors } from '../errors';
-
-export const canWriteSettings = (req: Request, _res: Response, next: NextFunction) =>
-  req.headers['x-access-token'] === process.env.WRITE_SETTINGS_SECRET
-    ? next()
-    : next(new HestiaError(Errors.Unauthorized));
-
-export default () => {
-  const router = Router();
-
-  router.get('/', endpointWrapper(get));
-
-  router.put('/', canWriteSettings, endpointWrapper(update));
-
-  return router;
-};

package/src/settings/services/get.spec.ts

@@ -1,62 +0,0 @@
-import chai from 'chai';
-import chaiAsPromised from 'chai-as-promised';
-chai.use(chaiAsPromised);
-const { expect } = chai;
-
-import * as sinon from 'sinon';
-import 'mocha';
-
-import { fixtures, initDb } from '../../../test/utils';
-
-import { get, isEnabled } from './get';
-import * as specs from './get';
-import { SettingKey } from '../model';
-
-let stubs: sinon.SinonStub[] = [];
-
-describe('settings > services', () => {
-  beforeEach(() => {
-    stubs = [];
-  });
-
-  afterEach(() => {
-    stubs.forEach((stub) => stub.restore());
-  });
-
-  describe('get', () => {
-    beforeEach(async () => {
-      await initDb();
-    });
-
-    describe('get', () => {
-      describe('setting not found', () => {
-        it('should throw an error', () => {
-          return expect(get('random_key' as any)).to.be.rejected;
-        });
-      });
-
-      describe('setting found', () => {
-        it('should return the setting', async () => {
-          const {
-            rows: [setting]
-          } = await get(SettingKey.maintenanceEnabled);
-          expect(setting).to.deep.equal(fixtures.settings[0]);
-        });
-      });
-    });
-
-    describe('isEnabled', () => {
-      const active = true;
-
-      describe('setting exists', () => {
-        beforeEach(() => {
-          stubs.push(sinon.stub(specs, 'get').resolves({ rows: [{ setting: 'key', active }] } as any));
-        });
-
-        it('should return the setting value', async () => {
-          expect(await isEnabled('key' as any)).to.equal(active);
-        });
-      });
-    });
-  });
-});

package/src/settings/services/get.ts

@@ -1,18 +0,0 @@
-import { SettingKey, getSetting, getAllSettings } from '../model/';
-
-const loadAllSettings = async () => {
-  const { rows } = await getAllSettings();
-  const value = rows.reduce((prev, setting) => ({ ...prev, [setting.setting]: setting.active }), {});
-  return value;
-};
-
-export const getAll = () => loadAllSettings();
-
-export const get = (key: SettingKey) => getSetting(key);
-
-export const isEnabled = async (key: SettingKey) => {
-  const {
-    rows: [setting]
-  } = await get(key);
-  return setting?.active;
-};