@hestia-earth/data-api 0.0.2-2 → 0.0.2-4

package/run-docker.sh

@@ -1,14 +0,0 @@
-#!/bin/sh
-
-docker build --progress=plain \
-  -t hestia-data-api:latest \
-  .
-
-docker run --rm \
-  --name hestia-data-api \
-  --env-file .env \
-  --env PORT=80 \
-  -v ${PWD}/scripts:/app/scripts \
-  -v ${PWD}/src:/app/src \
-  -p 3001:80 \
-  hestia-data-api:latest npm run dev

package/run-test.sh

@@ -1,5 +0,0 @@
-#!/bin/sh
-
-docker-compose -f test/docker-compose.yml build
-docker-compose -f test/docker-compose.yml run test
-docker-compose -f test/docker-compose.yml down

package/scripts/run-lambda.ts

@@ -1,10 +0,0 @@
-/* eslint-disable no-console */
-import * as dotenv from 'dotenv';
-dotenv.config();
-
-import { updateAggregatedNodes } from '../src/lambdas/update-aggregated-nodes/handler';
-
-updateAggregatedNodes().then(() => process.exit(0)).catch(err => {
-  console.error(err);
-  process.exit(1);
-});

package/scripts/run-migrations.ts

@@ -1,18 +0,0 @@
-/* eslint-disable no-console */
-import * as dotenv from 'dotenv';
-dotenv.config();
-
-import { runMigrations } from '../database/migrations';
-
-void (async function () {
-  const { gracefulExit } = await import('exit-hook');
-
-  try {
-    await runMigrations();
-    gracefulExit(0);
-  }
-  catch (error) {
-    console.error(error);
-    gracefulExit(1);
-  }
-})();

package/scripts/run-resetdb.ts

@@ -1,18 +0,0 @@
-/* eslint-disable no-console */
-import * as dotenv from 'dotenv';
-dotenv.config();
-
-import { resetDb } from '../test/utils';
-
-void (async function () {
-  const { gracefulExit } = await import('exit-hook');
-
-  try {
-    await resetDb();
-    gracefulExit(0);
-  }
-  catch (error) {
-    console.error(error);
-    gracefulExit(1);
-  }
-})();

package/scripts/run-seed.ts

@@ -1,18 +0,0 @@
-/* eslint-disable no-console */
-import * as dotenv from 'dotenv';
-dotenv.config();
-
-import { runSeed } from '../database/seed';
-
-void (async function () {
-  const { gracefulExit } = await import('exit-hook');
-
-  try {
-    await runSeed();
-    gracefulExit(0);
-  }
-  catch (error) {
-    console.error(error);
-    gracefulExit(1);
-  }
-})();

package/serverless.yml

@@ -1,76 +0,0 @@
-frameworkVersion: '3'
-service: hestia-data-api
-
-plugins:
-  - serverless-deployment-bucket
-  - serverless-offline
-
-custom:
-  serverless-offline:
-    httpPort: 3002
-  stage: ${env:STAGE, self:provider.stage}
-  securityGroup:
-    staging: sg-0800627978a40527b
-    prod: sg-049204c3b58a8bd08
-  subnetId1:
-    # private 1a
-    staging: subnet-04ad6e355249c757d
-    prod: subnet-00c4a891bce6f48e0
-  subnetId2:
-    # private 1b
-    staging: subnet-01260544aaa3c6810
-    prod: subnet-02365f46c569f5e1a
-  debugging:
-    staging: true
-    prod: false
-  logLevel:
-    staging: DEBUG
-    prod: INFO
-  bucketData: hestia-data-${self:custom.stage}
-  domain:
-    staging: https://www-staging.hestia.earth
-    prod: https://www.hestia.earth
-
-provider:
-  name: aws
-  runtime: nodejs18.x
-  region: us-east-1
-  stage: staging
-  deploymentBucket:
-    name: hestia-serverless-${self:custom.stage}
-  memorySize: 512
-  timeout: 600
-  environment:
-    SERVICE_NAME: ${self:service}
-    STAGE: ${self:custom.stage}
-    DEBUG: ${self:custom.debugging.${self:custom.stage}}
-    LOG_LEVEL: ${self:custom.logLevel.${self:custom.stage}}
-    SENTRY_DSN: https://00ed710e2b664894a3ff3124bfff24cf@o441427.ingest.0.1.0.io/4505186792767488
-    BUCKET_DATA: ${self:custom.bucketData}
-    PGHOST: ${env:PGHOST}
-    PGDATABASE: ${env:PGDATABASE}
-    PGPORT: ${env:PGPORT}
-    PGUSER: ${env:PGUSER}
-    PGPASSWORD: ${env:PGPASSWORD}
-  iamRoleStatements:
-    - Effect: Allow
-      Action:
-        - s3:GetObject
-        - s3:GetObjectAcl
-        - s3:ListObjects
-        - s3:ListBucket
-      Resource:
-        - "arn:aws:s3:::${self:custom.bucketData}"
-        - "arn:aws:s3:::${self:custom.bucketData}/*"
-  vpc:
-    securityGroupIds:
-      - ${self:custom.securityGroup.${self:custom.stage}}
-    subnetIds:
-      - ${self:custom.subnetId1.${self:custom.stage}}
-      - ${self:custom.subnetId2.${self:custom.stage}}
-
-functions:
-  updateAggregatedNodes:
-    handler: build/src/lambdas/update-aggregated-nodes/handler.updateAggregatedNodes
-    events:
-      - schedule: cron(0 2 * * ? *) # 2:00am every day

package/src/aggregated-nodes/model/index.ts

@@ -1,37 +0,0 @@
-import { ISiteJSONLD } from '@hestia-earth/schema';
-
-export interface IFilter {
-  regions?: string[];
-  periods?: string[];
-  products?: string[];
-  defaultMethodClassifications?: string[];
-  practices?: string[];
-  minAggregatedQualityScore?: number;
-}
-
-export const validFilterFields = [
-  'regions',
-  'periods',
-  'products',
-  'defaultMethodClassifications',
-  'practices',
-  'minAggregatedQualityScore'
-];
-
-export interface IFilters {
-  cycleCount: number;
-  impactAssessmentCount: number;
-  products: string[];
-  periods: string[];
-  practices: string[];
-  minAggregatedQualityScore: number[];
-  regions: string[];
-  defaultMethodClassifications: string[];
-}
-
-// TODO: update with pivoted types when available
-export interface AggregatedNodeRow {
-  cycle: any; // pivoted cycle
-  region: ISiteJSONLD['country'];
-  impactAssessments: [any]; // pivoted impact assessments
-}

package/src/aggregated-nodes/routes/pg-get-filters.ts

@@ -1,44 +0,0 @@
-import { Request } from 'express';
-
-import { applyFilters } from '../services/pg-get-filters';
-
-/**
- * @swagger
- *
- * paths:
- *   /aggregated-nodes/pg/filters:
- *     get:
- *       summary: "Get the additional filters available for a filter selection and node counts (uses PG only)"
- *       parameters:
- *        - in: query
- *          name: filter
- *          schema:
- *            type: string
- *            example: "regions=Africa|Albania;products=abyssinianKaleStraw|abyssinianKaleSeedWhole;periods=1970-1973"
- *          description: Set of filters
- *        - in: query
- *          name: query
- *          schema:
- *            type: string
- *            example: Wheat
- *          description: Free search text
- *       responses:
- *         200:
- *           content:
- *             application/json:
- *               schema:
- *                 type: object
- *         400:
- *           $ref: '#/components/responses/BadRequestError'
- *         404:
- *           $ref: '#/components/responses/NotFoundError'
- *         500:
- *           $ref: '#/components/responses/UnexpectedError'
- *         503:
- *           $ref: '#/components/responses/ServiceUnavailable'
- */
-
-export const getFilters = ({ query: qs, filter }: Request) => {
-  const query = qs.query as string;
-  return applyFilters({ filter, query });
-};

package/src/aggregated-nodes/routes/pg-get.ts

@@ -1,50 +0,0 @@
-import { Request } from 'express';
-
-import { applyFilter } from '../services/pg-get';
-
-/**
- * @swagger
- *
- * paths:
- *   /aggregated-nodes/pg:
- *     get:
- *       summary: "Get the aggregated nodes corresponding to a set of filters and search query (uses PG only)"
- *       parameters:
- *        - in: query
- *          name: filter
- *          schema:
- *            type: string
- *            example: "regions=Africa|Albania;products=abyssinianKaleStraw|abyssinianKaleSeedWhole;periods=1970-1973"
- *          description: Set of filters
- *        - in: query
- *          name: page
- *          schema:
- *            type: integer
- *            example: 1
- *        - in: query
- *          name: query
- *          schema:
- *            type: string
- *            example: Wheat
- *          description: Free search text
- *       responses:
- *         200:
- *           content:
- *             application/json:
- *               schema:
- *                 type: object
- *         400:
- *           $ref: '#/components/responses/BadRequestError'
- *         404:
- *           $ref: '#/components/responses/NotFoundError'
- *         500:
- *           $ref: '#/components/responses/UnexpectedError'
- *         503:
- *           $ref: '#/components/responses/ServiceUnavailable'
- */
-
-export const get = ({ query: qs, filter }: Request) => {
-  const page = Number(qs.page) || 1;
-  const query = qs.query as string;
-  return applyFilter({ page, filter, query });
-};

package/src/aggregated-nodes/routes.spec.ts

@@ -1,242 +0,0 @@
-import { expect } from 'chai';
-import * as sinon from 'sinon';
-import 'mocha';
-import express from 'express';
-import supertest from 'supertest';
-import bodyParser from 'body-parser';
-
-import router from './routes';
-import { Errors, errorHandler } from '../errors';
-import { initDb } from '../../test/utils';
-import { seed } from '../../database/seed';
-
-import * as fixtures from '../../test/fixtures/aggregated-nodes/get';
-
-let stubs: sinon.SinonStub[] = [];
-
-const app = express();
-
-const getFixtureNode = (type: 'site' | 'cycle' | 'impactAssessment', id: string) =>
-  fixtures[
-    { cycle: 'aggregatedCycles', site: 'aggregatedSites', impactAssessment: 'aggregatedImpactAssessments' }[type]
-  ].find((node) => node.jsonld_pivoted['@id'] === id).jsonld_pivoted;
-
-app.use(bodyParser.json());
-app.use(router());
-app.use(errorHandler);
-
-describe('routes', () => {
-  beforeEach(() => {
-    stubs = [];
-  });
-
-  afterEach(() => {
-    stubs.forEach((stub) => stub.restore());
-  });
-
-  describe('aggregated-nodes', () => {
-    beforeEach(async () => {
-      await initDb();
-      await seed(fixtures);
-    });
-
-    describe('GET /pg', () => {
-      it('when invalid filter passed returns invalid param error', async () => {
-        const result = await supertest(app)
-          .get('/pg')
-          .query({ filter: 'invalidfilter' })
-          .set('Accept', 'application/json');
-
-        expect(result.body.message).to.equal(Errors.InvalidQueryParam);
-      });
-
-      it('when neither filter nor query passed returns missing param error', async () => {
-        const result = await supertest(app).get('/pg').query({}).set('Accept', 'application/json');
-
-        expect(result.body.message).to.equal(Errors.MissingQueryParam);
-      });
-
-      it('can filter by regions', async () => {
-        const expected = [
-          {
-            cycle: getFixtureNode('cycle', '1'),
-            region: getFixtureNode('site', '1').country,
-            impactAssessments: [getFixtureNode('impactAssessment', '1'), getFixtureNode('impactAssessment', '2')]
-          },
-          {
-            cycle: getFixtureNode('cycle', '2'),
-            region: getFixtureNode('site', '1').country,
-            impactAssessments: [getFixtureNode('impactAssessment', '3')]
-          },
-          {
-            cycle: getFixtureNode('cycle', '3'),
-            region: getFixtureNode('site', '2').country,
-            impactAssessments: [getFixtureNode('impactAssessment', '4')]
-          }
-        ];
-
-        const filter = 'regions=GADM-POL|GADM-DEU';
-        const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
-        expect(result.body.rows).to.deep.equal(expected);
-      });
-
-      it('can filter by practices', async () => {
-        const expected = [
-          {
-            cycle: getFixtureNode('cycle', '1'),
-            region: getFixtureNode('site', '1').country,
-            impactAssessments: [getFixtureNode('impactAssessment', '1'), getFixtureNode('impactAssessment', '2')]
-          },
-          {
-            cycle: getFixtureNode('cycle', '2'),
-            region: getFixtureNode('site', '1').country,
-            impactAssessments: [getFixtureNode('impactAssessment', '3')]
-          }
-        ];
-        const filter = 'practices=earthingUpByHand|deepRipping';
-        const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
-        expect(result.body.rows).to.deep.equal(expected);
-      });
-
-      it('can filter by minimum aggregatedQualityScore', async () => {
-        const expected = [
-          {
-            cycle: getFixtureNode('cycle', '1'),
-            region: getFixtureNode('site', '1').country,
-            impactAssessments: [getFixtureNode('impactAssessment', '1'), getFixtureNode('impactAssessment', '2')]
-          }
-        ];
-        const filter = 'minAggregatedQualityScore=3';
-        const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
-        expect(result.body.rows).to.deep.equal(expected);
-      });
-
-      it('can filter by defaultMethodClassifications', async () => {
-        const expected = [
-          {
-            cycle: getFixtureNode('cycle', '2'),
-            region: getFixtureNode('site', '1').country,
-            impactAssessments: [getFixtureNode('impactAssessment', '3')]
-          },
-          {
-            cycle: getFixtureNode('cycle', '3'),
-            region: getFixtureNode('site', '2').country,
-            impactAssessments: [getFixtureNode('impactAssessment', '4')]
-          }
-        ];
-        const filter = 'defaultMethodClassifications=physical measurement|expert opinion';
-        const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
-        expect(result.body.rows).to.deep.equal(expected);
-      });
-
-      it('can filter by periods', async () => {
-        const expected = [
-          {
-            cycle: getFixtureNode('cycle', '3'),
-            region: getFixtureNode('site', '2').country,
-            impactAssessments: [getFixtureNode('impactAssessment', '4')]
-          },
-          {
-            cycle: getFixtureNode('cycle', '4'),
-            region: getFixtureNode('site', '3').country,
-            impactAssessments: []
-          }
-        ];
-        const filter = 'periods=2010-2019|2000-2009';
-        const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
-        expect(result.body.rows).to.deep.equal(expected);
-      });
-
-      it('can filter by products', async () => {
-        const expected = [
-          {
-            cycle: getFixtureNode('cycle', '1'),
-            region: getFixtureNode('site', '1').country,
-            // impactAssessments have also been filtered
-            impactAssessments: [getFixtureNode('impactAssessment', '1')]
-          },
-          {
-            cycle: getFixtureNode('cycle', '2'),
-            region: getFixtureNode('site', '1').country,
-            impactAssessments: [getFixtureNode('impactAssessment', '3')]
-          }
-        ];
-        const filter = 'products=wheatGrain|fishFingerlings';
-        const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
-        expect(result.body.rows).to.deep.equal(expected);
-      });
-
-      it('can combine multiple filters', async () => {
-        const expected = [
-          {
-            cycle: getFixtureNode('cycle', '5'),
-            region: getFixtureNode('site', '3').country,
-            // impactAssessments have also been filtered
-            impactAssessments: []
-          }
-        ];
-        const filter = [
-          'regions=GADM-POL|GADM-DEU|GADM-BRA',
-          'products=wheatGrain|fishFingerlings|cheese',
-          'periods=2010-2019|2020-2029',
-          'defaultMethodClassifications=modelled|expert opinion',
-          'minAggregatedQualityScore=2',
-          'practices=earthingUpByHand|deepRipping|baggingFruit'
-        ].join(';');
-        const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
-        expect(result.body.rows).to.deep.equal(expected);
-      });
-    });
-
-    describe('GET /pg/filters', () => {
-      it('when invalid filter passed returns invalid param error', async () => {
-        const result = await supertest(app)
-          .get('/pg/filters')
-          .query({ filter: 'invalidfilter' })
-          .set('Accept', 'application/json');
-
-        expect(result.body.message).to.equal(Errors.InvalidQueryParam);
-      });
-
-      describe('returns the filters and node counts available for the current selection', () => {
-        it('returns subset of filters corresponding to selected nodes', async () => {
-          const expected = {
-            cycleCount: 1,
-            impactAssessmentCount: 0,
-            products: ['apples', 'cheese'],
-            periods: ['2020-2029'],
-            practices: ['baggingFruit'],
-            minAggregatedQualityScore: [2],
-            regions: ['GADM-BRA'],
-            defaultMethodClassifications: ['modelled']
-          };
-          const filter = [
-            'regions=GADM-POL|GADM-DEU|GADM-BRA',
-            'products=wheatGrain|fishFingerlings|cheese',
-            'periods=2010-2019|2020-2029',
-            'defaultMethodClassifications=modelled|expert opinion',
-            'minAggregatedQualityScore=2',
-            'practices=earthingUpByHand|deepRipping|baggingFruit'
-          ].join(';');
-          const result = await supertest(app).get('/pg/filters').query({ filter }).set('Accept', 'application/json');
-          expect(result.body).to.deep.equal(expected);
-        });
-
-        it('returns all filters when there is nothing currently selected', async () => {
-          const expected = {
-            cycleCount: 8,
-            impactAssessmentCount: 4,
-            products: ['apples', 'barley', 'cheese', 'fishFingerlings', 'wheatGrain'],
-            periods: ['1990-1999', '2000-2009', '2010-2019', '2020-2029'],
-            practices: ['baggingFruit', 'deepRipping', 'earthingUpByHand'],
-            minAggregatedQualityScore: [1, 2, 3],
-            regions: ['GADM-BRA', 'GADM-DEU', 'GADM-POL'],
-            defaultMethodClassifications: ['expert opinion', 'modelled', 'physical measurement', 'unsourced assumption']
-          };
-          const result = await supertest(app).get('/pg/filters').query({}).set('Accept', 'application/json');
-          expect(result.body).to.deep.equal(expected);
-        });
-      });
-    });
-  });
-});

package/src/aggregated-nodes/routes.ts

@@ -1,56 +0,0 @@
-import { Request, Response, NextFunction, Router } from 'express';
-import { endpointWrapper } from '../utils/endpoint-wrapper';
-import { get as pgGet } from './routes/pg-get';
-import { HestiaError, Errors } from '../errors';
-import { validFilterFields } from './model';
-import { getFilters as pgGetFilters } from './routes/pg-get-filters';
-import { IFilter } from './model';
-
-const isValidFilterString = (filter: string) =>
-  filter.split(';').every((field) => {
-    const [key] = field.split('=');
-    return validFilterFields.includes(key);
-  });
-
-const validateGet = (req: Request, _res: Response, next: NextFunction) => {
-  const missingParam = !req.query.filter && !req.query.query;
-  const invalidParam = req.query.filter && !isValidFilterString(req.query.filter as string);
-  return missingParam
-    ? next(new HestiaError(Errors.MissingQueryParam))
-    : invalidParam
-      ? next(new HestiaError(Errors.InvalidQueryParam))
-      : next();
-};
-
-const validateGetFilters = (req: Request, _res: Response, next: NextFunction) => {
-  const invalidParam = req.query.filter && !isValidFilterString(req.query.filter as string);
-  return invalidParam ? next(new HestiaError(Errors.InvalidQueryParam)) : next();
-};
-
-const parseArrayField = (val: string) => val.split('|');
-
-const filterFieldParsers: Record<keyof IFilter, (val: string) => any> = {
-  regions: parseArrayField,
-  periods: parseArrayField,
-  products: parseArrayField,
-  defaultMethodClassifications: parseArrayField,
-  minAggregatedQualityScore: (val: string) => Number(val),
-  practices: parseArrayField
-};
-
-const parseFilterString = (req: Request, _res: Response, next: NextFunction) => {
-  req.filter = ((req.query.filter || '') as string).split(';').reduce((acc, entry) => {
-    const [key, val] = entry.split('=');
-    return filterFieldParsers[key] ? { ...acc, [key]: filterFieldParsers[key](val) } : acc;
-  }, {});
-  next();
-};
-
-export default () => {
-  const router = Router();
-
-  router.get('/pg/filters', validateGetFilters, parseFilterString, endpointWrapper(pgGetFilters));
-  router.get('/pg', validateGet, parseFilterString, endpointWrapper(pgGet));
-
-  return router;
-};

package/src/aggregated-nodes/services/pg-get-filters.ts

@@ -1,52 +0,0 @@
-import { getFilterClauses } from './pg-get';
-import { query as doQuery } from '../../../database';
-import { IFilter, IFilters } from '../model';
-
-const getFiltersQuery = (filter: IFilter): [string, any[]] => {
-  const { filterClauses, params } = getFilterClauses(filter);
-  return [
-    `
-SELECT
-  COUNT(DISTINCT c) as "cycleCount",
-  COUNT(DISTINCT ia) AS "impactAssessmentCount",
-  ARRAY_AGG(DISTINCT product ORDER BY product) as products,
-  ARRAY_AGG(DISTINCT c.generated_period ORDER BY c.generated_period) as periods,
-  ARRAY_AGG(DISTINCT practice ORDER BY practice) as practices,
-  ARRAY_AGG(
-    DISTINCT (c.jsonld_pivoted->>'aggregatedQualityScore')::integer
-    ORDER BY (c.jsonld_pivoted->>'aggregatedQualityScore')::integer
-  ) AS "minAggregatedQualityScore",
-  ARRAY_AGG(
-    DISTINCT s.jsonld_pivoted['country']->>'@id'
-    ORDER BY s.jsonld_pivoted['country']->>'@id'
-  ) AS regions,
-  ARRAY_AGG(
-    DISTINCT c.jsonld_pivoted->>'defaultMethodClassification'
-    ORDER BY c.jsonld_pivoted->>'defaultMethodClassification'
-  ) AS "defaultMethodClassifications"
-FROM aggregated_cycles AS c
-LEFT JOIN aggregated_impact_assessments AS ia
-  ON c.hestia_id = ia.cycle_hestia_id
-  AND ${filterClauses.products.iaFilter}
-JOIN aggregated_sites AS s
-  ON c.site_hestia_id = s.hestia_id
-  AND ${filterClauses.regions}
-CROSS JOIN LATERAL
-  jsonb_object_keys(c.jsonld_pivoted['products']) as products(product),
-  jsonb_object_keys(c.jsonld_pivoted['practices']) as practices(practice)
-WHERE ${filterClauses.practices}
-  AND ${filterClauses.defaultMethodClassifications}
-  AND ${filterClauses.products.cycleFilter}
-  AND ${filterClauses.minAggregatedQualityScore}
-  AND ${filterClauses.periods}
-  `,
-    params
-  ];
-};
-
-export const applyFilters = async ({ filter }: { filter: IFilter; query?: string }) => {
-  // TODO: handle query later
-  const [pgQuery, params] = getFiltersQuery(filter);
-  const res = await doQuery<IFilters>(pgQuery, params);
-  return res?.rows[0];
-};