@hestia-earth/data-api 0.0.2-2 → 0.0.2-4

package/src/aggregated-nodes/services/pg-get.ts

@@ -1,77 +0,0 @@
-import { partition as _partition } from 'lodash';
-import { query as doQuery } from '../../../database';
-import { AggregatedNodeRow, IFilter } from '../model';
-
-type FilterClauseFunc = (args: { idx: number; exclude: boolean }) => string | Record<string, string>;
-
-const filterClauseMapping: Record<keyof IFilter, FilterClauseFunc> = {
-  periods: ({ idx, exclude = false }) => (exclude ? 'TRUE' : `c.generated_period = ANY($${idx}::text[])`),
-  // IA aggregatedQualityScore is always equal to that of cycle
-  minAggregatedQualityScore: ({ idx, exclude = false }) =>
-    exclude ? 'TRUE' : `(c.jsonld_pivoted->>'aggregatedQualityScore')::integer >= $${idx}`,
-  practices: ({ idx, exclude = false }) => (exclude ? 'TRUE' : `c.jsonld_pivoted['practices'] ?| $${idx}::text[]`),
-  products: ({ idx, exclude = false }) => ({
-    iaFilter: exclude ? 'TRUE' : `ia.jsonld_pivoted['product']['term']->>'@id' = ANY($${idx}::text[])`,
-    cycleFilter: exclude ? 'TRUE' : `c.jsonld_pivoted['products'] ?| $${idx}::text[]`
-  }),
-  // country field also holds values like 'Northern Europe'
-  regions: ({ idx, exclude = false }) =>
-    exclude ? 'TRUE' : `s.jsonld_pivoted['country']->>'@id' = ANY($${idx}::text[])`,
-  defaultMethodClassifications: ({ idx, exclude = false }) =>
-    exclude ? 'TRUE' : `c.jsonld_pivoted->>'defaultMethodClassification' = ANY($${idx}::text[])`
-};
-
-export const getFilterClauses = (filter: IFilter) => {
-  const filterClauses: Partial<Record<keyof IFilter, Record<string, string>>> = {};
-  const params = [];
-
-  const [included, excluded] = _partition(Object.keys(filterClauseMapping), (key) => filter[key]);
-
-  included.forEach((key, idx) => {
-    filterClauses[key] = filterClauseMapping[key]({ idx: idx + 1 });
-    params.push(filter[key]);
-  });
-
-  excluded.forEach((key) => {
-    filterClauses[key] = filterClauseMapping[key]({ exclude: true });
-  });
-  return { filterClauses, params };
-};
-
-const getNodesQuery = (filter: IFilter): [string, any[]] => {
-  const { filterClauses, params } = getFilterClauses(filter);
-  return [
-    `
-SELECT
-  c.jsonld_pivoted AS cycle,
-  s.jsonld_pivoted['country'] AS region,
-  COALESCE(
-    jsonb_agg(ia.jsonld_pivoted ORDER BY ia.jsonld_pivoted->>'@id') FILTER (WHERE ia.jsonld_pivoted IS NOT NULL),
-    '[]'
-  ) AS "impactAssessments"
-FROM aggregated_cycles AS c
-LEFT JOIN aggregated_impact_assessments AS ia
-  ON c.hestia_id = ia.cycle_hestia_id
-  AND ${filterClauses.products.iaFilter}
-JOIN aggregated_sites AS s
-  ON c.site_hestia_id = s.hestia_id
-  AND ${filterClauses.regions}
-WHERE ${filterClauses.practices}
-AND ${filterClauses.defaultMethodClassifications}
-AND ${filterClauses.products.cycleFilter}
-AND ${filterClauses.minAggregatedQualityScore}
-AND ${filterClauses.periods}
-GROUP BY cycle, region
-ORDER BY c.jsonld_pivoted->>'@id'
-LIMIT 500
-`,
-    params
-  ];
-};
-
-export const applyFilter = async ({ filter }: { page?: number; filter: IFilter; query?: string }) => {
-  // TODO: handle page and query later
-  const [pgQuery, params] = getNodesQuery(filter);
-  const rows = await doQuery<AggregatedNodeRow>(pgQuery, params);
-  return rows;
-};

package/src/app.spec.ts

@@ -1,34 +0,0 @@
-import { expect } from 'chai';
-import * as sinon from 'sinon';
-import 'mocha';
-
-import { logger } from './logger';
-import { app, healthcheck } from './app';
-
-let stubs: sinon.SinonStub[] = [];
-
-describe('app', () => {
-  beforeEach(() => {
-    stubs = [];
-    stubs.push(sinon.stub(logger, 'error'));
-    stubs.push(sinon.stub(logger, 'debug'));
-  });
-
-  afterEach(() => {
-    stubs.forEach((stub) => stub.restore());
-  });
-
-  it('should be defined', () => {
-    expect(app !== undefined).to.equal(true);
-  });
-
-  describe('healthcheck', () => {
-    it('should return status 200', () => {
-      const res = {
-        status: sinon.stub().returns({ send() {} })
-      };
-      healthcheck(null, res);
-      expect(res.status.calledWith(200)).to.equal(true);
-    });
-  });
-});

package/src/app.ts

@@ -1,59 +0,0 @@
-import * as express from 'express';
-import 'express-async-errors';
-const http2Express = require('http2-express-bridge');
-import { RequestHandler } from 'express';
-
-const app: express.Express = http2Express(express);
-
-import * as Sentry from '@sentry/node';
-import * as Tracing from '@sentry/tracing';
-import { sentryConfig } from './config';
-
-Sentry.init({
-  ...sentryConfig,
-  integrations: [new Sentry.Integrations.Http({ tracing: true }), new Tracing.Integrations.Express({ app })]
-});
-app.use(Sentry.Handlers.requestHandler());
-app.use(Sentry.Handlers.tracingHandler());
-
-import cors from 'cors';
-import { disableCors } from './config';
-import { whitelistOrigin } from './cors';
-app.use(cors(disableCors ? undefined : { origin: whitelistOrigin }));
-
-import compression from 'compression';
-app.use(compression());
-
-import * as bodyParser from 'body-parser';
-// based on AWS Lambda body size limit
-const limit = '6mb';
-app.use(bodyParser.json({ limit }));
-app.use(bodyParser.urlencoded({ limit, extended: false }));
-
-export const healthcheck = (_req, res) => res.status(200).send();
-app.get('/health', healthcheck);
-
-import swagger from './swagger/routes';
-app.use('/swagger', swagger());
-
-// make sure settings is not blocked by maintenance mode
-import settings from './settings/routes';
-app.use('/settings', settings());
-
-import { maintenance } from './maintenance';
-// Until express v5 promises must be coerced to void
-// See https://github.com/davidbanham/express-async-errors/issues/36#issuecomment-944954003
-app.use(maintenance as RequestHandler);
-
-// log routes after healthcheck and maintenance
-import morgan from 'morgan';
-import { stream } from './logger';
-app.use(morgan('combined', { stream }));
-
-import routes from './routes';
-routes(app);
-
-import errorHandler from './errors';
-errorHandler(app);
-
-export { app };

package/src/config.ts

@@ -1,21 +0,0 @@
-const { version } = require('../package.json');
-
-export const PORT = process.env.PORT || '3000';
-export const apiUrl = (process.env.API_URL || 'http://localhost:{PORT}').replace('{PORT}', PORT);
-export const webappUrl = process.env.WEBAPP_URL || 'http://localhost';
-
-export const disableCors = (process.env.CORS_DISABLED || 'false') === 'true';
-export const whitelist = (process.env.CORS_WHITELIST || '').split(',');
-whitelist.push(apiUrl);
-whitelist.push(webappUrl);
-
-export const sentryConfig = {
-  dsn: process.env.SENTRY_DSN,
-  environment: process.env.SENTRY_ENV,
-  release: version
-};
-
-export const slackConfig = {
-  token: process.env.SLACK_TOKEN,
-  channel: process.env.SLACK_CHANNEL
-};

package/src/cors.spec.ts

@@ -1,32 +0,0 @@
-import { expect } from 'chai';
-import * as sinon from 'sinon';
-import 'mocha';
-
-import { whitelistOrigin } from './cors';
-
-describe('cors', () => {
-  describe('whitelistOrigin', () => {
-    const origin = 'http://localhost';
-    let callback: sinon.SinonStub;
-
-    beforeEach(() => {
-      callback = sinon.stub();
-    });
-
-    it('should allow localhost', () => {
-      whitelistOrigin(origin, callback);
-      expect(callback.calledWith(null, true)).to.equal(true);
-    });
-
-    it('should allow no origin', () => {
-      whitelistOrigin(undefined, callback);
-      expect(callback.calledWith(null, true)).to.equal(true);
-    });
-
-    it('should not allow other origin', () => {
-      const other = 'other origin';
-      whitelistOrigin(other, callback);
-      expect(callback.calledWith(null, true)).to.equal(false);
-    });
-  });
-});

package/src/cors.ts

@@ -1,7 +0,0 @@
-import { whitelist } from './config';
-import { HestiaError } from './errors';
-
-export const whitelistOrigin = (origin: string, callback: Function) =>
-  !origin || whitelist.includes(origin)
-    ? callback(null, true)
-    : callback(new HestiaError(`${origin} is not allowed by CORS`));

package/src/errors.spec.ts

@@ -1,114 +0,0 @@
-import { expect } from 'chai';
-import * as sinon from 'sinon';
-import 'mocha';
-
-import { logger } from './logger';
-import { Errors, errorHandler, HestiaError } from './errors';
-
-class Response {
-  statusCode = 200;
-  json() {}
-}
-
-let stubs: sinon.SinonStub[] = [];
-
-describe('errors', () => {
-  let errorStub: sinon.SinonStub;
-
-  beforeEach(() => {
-    stubs = [];
-    stubs.push((errorStub = sinon.stub(logger, 'error')));
-  });
-
-  afterEach(() => {
-    stubs.forEach((stub) => stub.restore());
-  });
-
-  describe('errorHandler', () => {
-    let error: any = {
-      message: 'error'
-    };
-    const response = new Response();
-
-    it('should set status code to 500', () => {
-      errorHandler(error, null, response as any);
-      expect(response.statusCode).to.equal(500);
-    });
-
-    it('should render error as json', () => {
-      let stub: sinon.SinonStub;
-      stubs.push((stub = sinon.stub(response, 'json')));
-      errorHandler(error, null, response as any);
-
-      expect(
-        stub.calledWith({
-          message: error.message
-        })
-      ).to.equal(true);
-      stub.restore();
-    });
-
-    it('should log Error stack', () => {
-      const err = new Error();
-      errorHandler(err, null, response as any);
-      expect(errorStub.calledWith(err.stack)).to.equal(true);
-    });
-
-    describe('Hestia error', () => {
-      beforeEach(() => {
-        error = new HestiaError('error');
-      });
-
-      it('should not log the error', () => {
-        errorHandler(error, null, response as any);
-        expect(errorStub.called).to.equal(false);
-      });
-
-      it('should set status code to 400', () => {
-        errorHandler(error, null, response as any);
-        expect(response.statusCode).to.equal(400);
-      });
-    });
-
-    describe(Errors.Unauthorized, () => {
-      beforeEach(() => {
-        error = { message: Errors.Unauthorized };
-      });
-
-      it('should set status code to 401', () => {
-        errorHandler(error, null, response as any);
-        expect(response.statusCode).to.equal(401);
-      });
-    });
-
-    describe(Errors.NotFound, () => {
-      beforeEach(() => {
-        error = { message: Errors.NotFound };
-      });
-
-      it('should set status code to 404', () => {
-        errorHandler(error, null, response as any);
-        expect(response.statusCode).to.equal(404);
-      });
-    });
-
-    describe('no message', () => {
-      beforeEach(() => {
-        error = {};
-      });
-
-      it('should render default error', () => {
-        let stub: sinon.SinonStub;
-        stubs.push((stub = sinon.stub(response, 'json')));
-        errorHandler(error, null, response as any);
-
-        expect(
-          stub.calledWith({
-            message: 'There was an error processing your request.'
-          })
-        ).to.equal(true);
-        stub.restore();
-      });
-    });
-  });
-});

package/src/errors.ts

@@ -1,121 +0,0 @@
-import { Express, Request, Response, NextFunction } from 'express';
-import * as Sentry from '@sentry/node';
-
-import { logger } from './logger';
-
-export enum Errors {
-  UnderMaintenance = 'under-maintenance',
-  ReadOnly = 'read-only',
-  InvalidQueryParam = 'invalid-param-query',
-  MissingQueryParam = 'missing-param-query',
-  InvalidBodyParam = 'invalid-param-body',
-  MissingBodyParam = 'missing-param-body',
-  SchemaInvalid = 'invalid-schema',
-  NotFound = 'not-found',
-  Unauthorized = 'Unauthorized'
-}
-
-export enum StatusCodes {
-  MethodNotAllowed = 405,
-  RateLimit = 429,
-  ServiceUnavailable = 503,
-  InternalServerError = 500
-}
-
-export class HestiaError extends Error {
-  constructor(message: string, public details = {}, public statusCode?: StatusCodes) {
-    super(message);
-    this.name = 'HestiaError';
-  }
-}
-
-export const throwError = (message: string, details?: any, statusCode?: StatusCodes) => {
-  throw new HestiaError(message, details, statusCode);
-};
-
-const isHestiaError = (error: Error) => error.name === 'HestiaError';
-
-export const shouldHandleError = (error: Error) => !isHestiaError(error);
-
-const statusCode = (error: Error) =>
-  (error as any).statusCode ||
-  (errorMessage(error).includes(Errors.NotFound)
-    ? 404
-    : error.message === Errors.Unauthorized
-      ? 401
-      : isHestiaError(error)
-        ? 400
-        : 500);
-
-export const errorMessage = (error?: Error) => error?.message || 'There was an error processing your request.';
-
-/**
- * @swagger
- *
- * components:
- *   responses:
- *     DefaultObject:
- *       content:
- *         application/json:
- *           schema:
- *             type: object
- *     BadRequestError:
- *       description: Bad Request
- *       content:
- *         application/json:
- *           schema:
- *             $ref: '#/components/schemas/Error'
- *     UnauthorizedError:
- *       description: Unauthorized
- *       content:
- *         application/json:
- *           schema:
- *             $ref: '#/components/schemas/Error'
- *           example:
- *             - message: Unauthorized
- *     NotFoundError:
- *       description: The requested ressource was not found
- *       content:
- *         application/json:
- *           schema:
- *             $ref: '#/components/schemas/Error'
- *     UnexpectedError:
- *       description: Unexpected error
- *       content:
- *         application/json:
- *           schema:
- *             $ref: '#/components/schemas/Error'
- *     ServiceUnavailable:
- *       description: Service Unavailable due to Maintenance
- *       content:
- *         application/json:
- *           schema:
- *             $ref: '#/components/schemas/Error'
- *     NoContent:
- *       description: Return nothing
- *
- *   schemas:
- *     Error:
- *       type: object
- *       required:
- *         - message
- *       properties:
- *         message:
- *          type: string
- */
-export const errorHandler = (err: any, _r: Request, res: Response, _n?: NextFunction) => {
-  // error was not thrown intentionally
-  const unexpectedError = shouldHandleError(err);
-  unexpectedError && logger.error(err.stack ? err.stack : err);
-  const message = errorMessage(err);
-  res.statusCode = statusCode(err);
-  return res.json({
-    message,
-    ...('details' in err ? { details: err.details } : {})
-  });
-};
-
-export default (app: Express) => {
-  app.use(Sentry.Handlers.errorHandler({ shouldHandleError }));
-  app.use(errorHandler);
-};

package/src/index.spec.ts

@@ -1,94 +0,0 @@
-import { expect } from 'chai';
-import * as sinon from 'sinon';
-import 'mocha';
-
-import { run } from './index';
-import { app } from './app';
-import { logger } from './logger';
-import { PORT } from './config';
-
-let stubs: sinon.SinonStub[] = [];
-
-describe('run', () => {
-  const server: any = {};
-  let loggerStub: sinon.SinonStub;
-
-  beforeEach(() => {
-    stubs = [];
-  });
-
-  afterEach(() => {
-    stubs.forEach((stub) => stub.restore());
-  });
-
-  describe('start error', () => {
-    beforeEach(() => {
-      loggerStub = sinon.stub(logger, 'error');
-      stubs.push(loggerStub);
-    });
-
-    describe('error as string', () => {
-      const error = 'error';
-
-      beforeEach(() => {
-        stubs.push(
-          sinon.stub(app, 'listen').returns({
-            on: (_p, callback: any) => {
-              callback(error);
-              return server;
-            }
-          } as any)
-        );
-      });
-
-      it('should log the error', () => {
-        run();
-        expect(loggerStub.calledWith(error)).to.equal(true);
-      });
-    });
-
-    describe('error as Error', () => {
-      let error: Error;
-
-      beforeEach(() => {
-        error = new Error('error');
-
-        stubs.push(
-          sinon.stub(app, 'listen').returns({
-            on: (_p, callback: any) => {
-              callback(error);
-              return server;
-            }
-          } as any)
-        );
-      });
-
-      it('should log the error', () => {
-        run();
-        expect(loggerStub.calledWith(error.stack)).to.equal(true);
-      });
-    });
-  });
-
-  describe('start success', () => {
-    beforeEach(() => {
-      stubs.push(
-        sinon.stub(app, 'listen').callsFake((_p, callback) => {
-          callback();
-          return {
-            on: (_p1, _cb: any) => {
-              return server;
-            }
-          } as any;
-        })
-      );
-      loggerStub = sinon.stub(logger, 'info');
-      stubs.push(loggerStub);
-    });
-
-    it('should log the port', () => {
-      run();
-      expect(loggerStub.calledWith(`App running on port ${PORT}`)).to.equal(true);
-    });
-  });
-});

package/src/index.ts

@@ -1,14 +0,0 @@
-import '../database';
-import { logger } from './logger';
-import { PORT } from './config';
-import { app } from './app';
-
-export const run = () => {
-  const server = app
-    .listen(PORT, () => logger.info(`App running on port ${PORT}`))
-    .on('error', (err) => logger.error(err.stack || err));
-
-  // ALB idle timeout set to 300 seconds
-  server.keepAliveTimeout = 301 * 1000;
-  server.headersTimeout = 305 * 1000;
-};

package/src/lambdas/sentry.ts

@@ -1,12 +0,0 @@
-import * as Sentry from '@sentry/serverless';
-
-import pkg from '../../package.json';
-
-Sentry.AWSLambda.init({
-  dsn: process.env.SENTRY_DSN,
-  environment: process.env.STAGE,
-  release: pkg.version,
-  enabled: process.env.IS_OFFLINE !== 'true'
-});
-
-export const wrapHandler = handler => Sentry.AWSLambda.wrapHandler(handler);

package/src/lambdas/update-aggregated-nodes/handler.spec.ts

@@ -1,86 +0,0 @@
-import 'mocha';
-import sinon from 'sinon';
-import { of } from 'rxjs';
-import { expect } from 'chai';
-import * as s3 from '@hestia-earth/pipeline-utils/dist/s3';
-import { sortBy as _sortBy } from 'lodash';
-
-import { loadFixture, initDb } from '../../../test/utils';
-import { run } from './handler';
-import { query } from '../../../database';
-import * as testSeeds from '../../../database/seed/test';
-
-const stubs: sinon.SinonStub[] = [];
-
-const cycles = ['abyssinianKaleSeedWhole-cycle_pivoted.csv', 'tomatoFruit-cycle_pivoted.csv'];
-
-const impactAssessments = [
-  'abyssinianKaleSeedWhole-impactassessment_pivoted.csv',
-  'tomatoFruit-impactassessment_pivoted.csv'
-];
-
-const keys = [
-  ...cycles,
-  ...impactAssessments,
-  // abyssinianKaleStraw should be ignored as no ref to a cycle
-  'abyssinianKaleStraw-impactassessment_pivoted.csv',
-  // hould be ignored as cycle.id refers to a non-existent cycle
-  'cycle-missing-impactassessment_pivoted.csv'
-];
-
-const fixture = (fileName: string) => loadFixture(`update-aggregated-nodes/${fileName}`);
-
-const getJsonResults = (rows) =>
-  rows
-    .map((r) => r.jsonld_pivoted)
-    .filter(
-      (json) =>
-        ![
-          testSeeds.aggregatedCycles[0].jsonld_pivoted['@id'],
-          testSeeds.aggregatedSites[0].jsonld_pivoted['@id'],
-          testSeeds.aggregatedImpactAssessments[0].jsonld_pivoted['@id']
-        ].includes(json['@id'])
-    );
-
-describe('update-aggregated-nodes > run', () => {
-  beforeEach(async () => {
-    await initDb();
-    stubs.push(sinon.stub(s3, 'listFolder').resolves(keys.map((Key) => ({ Key }))));
-    stubs.push(sinon.stub(s3, 'getObject').callsFake(({ Key }) => of(fixture(Key)) as any));
-  });
-
-  afterEach(() => {
-    stubs.forEach((stub) => stub.restore());
-  });
-
-  it('should save the pivoted csv files in the db', async () => {
-    await run();
-
-    const { rows: siteRows } = await query('SELECT * FROM aggregated_sites');
-    const siteResult = _sortBy(getJsonResults(siteRows), '@id');
-    const jsonPivotedSites = _sortBy(
-      cycles.flatMap((key) => JSON.parse(fixture(`${key}.site.json`))),
-      '@id'
-    );
-
-    expect(siteResult.sort()).to.deep.equal(jsonPivotedSites.sort());
-
-    const { rows: cycleRows } = await query('SELECT * FROM aggregated_cycles');
-    const cycleResult = _sortBy(getJsonResults(cycleRows), '@id');
-    const jsonPivotedCycles = _sortBy(
-      cycles.flatMap((key) => JSON.parse(fixture(`${key}.cycle.json`))),
-      '@id'
-    );
-
-    expect(cycleResult).to.deep.equal(jsonPivotedCycles);
-
-    const { rows: iaRows } = await query('SELECT * FROM aggregated_impact_assessments');
-    const iaResult = _sortBy(getJsonResults(iaRows), '@id');
-    const jsonPivotedIas = _sortBy(
-      impactAssessments.flatMap((key) => JSON.parse(fixture(`${key}.impactAssessment.json`))),
-      '@id'
-    );
-
-    expect(iaResult.sort()).to.deep.equal(jsonPivotedIas.sort());
-  });
-});