@hestia-earth/data-api 0.0.2-1

package/envs/.staging.env

@@ -0,0 +1,7 @@
+export ENVIRONMENT='staging'
+export STAGE='staging'
+export PGHOST=$PG_HOST_STAGING
+export PGPASSWORD=$PG_PASSWORD_STAGING
+export PGDATABASE=$PG_DATABASE
+export PGPORT=$PG_PORT
+export PGUSER=$PG_USER

package/index.js

@@ -0,0 +1,3 @@
+require('dotenv').config();
+
+require('./build/src').run();

package/package.json

@@ -0,0 +1,105 @@
+{
+  "name": "@hestia-earth/data-api",
+  "version": "0.0.2-1",
+  "description": "Hestia Data API definitions",
+  "main": "dist/models.js",
+  "typings": "dist/models.d.ts",
+  "scripts": {
+    "build": "rm -rf build && tsc -p tsconfig.build.json",
+    "build:module": "rm -rf dist && tsc -p tsconfig.dist.json",
+    "build:lambdas": "rm -rf build-lambdas && tsc -p tsconfig.lambdas.json",
+    "start": "node index.js",
+    "dev": "ts-node-dev --respawn --rs dev.ts --files --ignore-watch node_modules",
+    "lint": "eslint .",
+    "lint:fix": "npm run lint -- --fix",
+    "test": "nyc mocha \"./**/*.spec.ts\"",
+    "migrate": "ts-node scripts/run-migrations.ts",
+    "seed": "ts-node scripts/run-seed.ts",
+    "resetdb": "ts-node scripts/run-resetdb.ts && npm run migrate && npm run seed",
+    "pre-release": "standard-version --prerelease && git push origin develop --follow-tags",
+    "release": "standard-version",
+    "postrelease": "git push origin master --follow-tags"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@gitlab.com/hestia-earth/hestia-data-api.git"
+  },
+  "author": "Guillaume Royer <guillaume@hestia.earth>",
+  "license": "UNLICENSED",
+  "keywords": [
+    "hestia",
+    "data",
+    "api"
+  ],
+  "bugs": {
+    "url": "https://gitlab.com/hestia-earth/hestia-data-api/issues"
+  },
+  "homepage": "https://gitlab.com/hestia-earth/hestia-data-api#readme",
+  "dependencies": {
+    "@hestia-earth/schema": "^22.1.0"
+  },
+  "devDependencies": {
+    "@hestia-earth/pipeline-utils": "^0.10.4",
+    "@sentry/node": "^7.51.2",
+    "@sentry/serverless": "^7.55.2",
+    "@sentry/tracing": "^7.51.2",
+    "@slack/web-api": "^6.8.1",
+    "compression": "^1.7.4",
+    "cors": "^2.8.5",
+    "csvtojson": "^2.0.10",
+    "dotenv": "^16.0.3",
+    "exit-hook": "^3.2.0",
+    "express": "^4.17.3",
+    "express-async-errors": "^3.1.1",
+    "http2-express-bridge": "^1.0.7",
+    "lodash.chunk": "^4.2.0",
+    "lodash.uniqby": "^4.7.0",
+    "map-obj": "^5.0.2",
+    "morgan": "^1.10.0",
+    "pg": "^8.11.0",
+    "rxjs": "^7.8.1",
+    "swagger-jsdoc": "^6.2.8",
+    "swagger-ui-express": "^4.6.3",
+    "winston": "^3.8.2",
+    "@commitlint/cli": "^17.6.5",
+    "@commitlint/config-conventional": "^17.6.5",
+    "@hestia-earth/eslint-config": "^0.0.5",
+    "@types/chai": "^4.3.5",
+    "@types/chai-as-promised": "^7.1.5",
+    "@types/compression": "^1.7.2",
+    "@types/cors": "^2.8.13",
+    "@types/debug": "^4.1.8",
+    "@types/express": "^4.17.6",
+    "@types/lodash.chunk": "^4.2.7",
+    "@types/lodash.uniqby": "^4.7.7",
+    "@types/mocha-prepare": "^0.1.0",
+    "@types/morgan": "^1.9.4",
+    "@types/node-cleanup": "^2.1.2",
+    "@types/pg": "^8.10.1",
+    "@types/sinon": "^10.0.15",
+    "@types/supertest": "^2.0.12",
+    "@types/swagger-jsdoc": "^6.0.1",
+    "@types/swagger-ui-express": "^4.1.3",
+    "chai": "^4.3.7",
+    "chai-as-promised": "^7.1.1",
+    "eslint": "^7.32.0",
+    "husky": "^4.3.8",
+    "mocha-prepare": "^0.1.0",
+    "nyc": "^15.1.0",
+    "postgrator": "^7.1.1",
+    "serverless": "^3.32.2",
+    "serverless-deployment-bucket": "^1.6.0",
+    "serverless-offline": "^12.0.4",
+    "sinon": "^15.1.0",
+    "standard-version": "^9.5.0",
+    "supertest": "^6.3.3",
+    "ts-node": "^10.9.1",
+    "ts-node-dev": "^2.0.0",
+    "typescript": "^5.0.0"
+  },
+  "husky": {
+    "hooks": {
+      "commit-msg": "commitlint -E HUSKY_GIT_PARAMS"
+    }
+  }
+}

package/run-docker.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+docker build --progress=plain \
+  -t hestia-data-api:latest \
+  .
+
+docker run --rm \
+  --name hestia-data-api \
+  --env-file .env \
+  --env PORT=80 \
+  -v ${PWD}/scripts:/app/scripts \
+  -v ${PWD}/src:/app/src \
+  -p 3001:80 \
+  hestia-data-api:latest npm run dev

package/run-test.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+docker-compose -f test/docker-compose.yml build
+docker-compose -f test/docker-compose.yml run test
+docker-compose -f test/docker-compose.yml down

package/scripts/run-lambda.ts

@@ -0,0 +1,10 @@
+/* eslint-disable no-console */
+import * as dotenv from 'dotenv';
+dotenv.config();
+
+import { updateAggregatedNodes } from '../src/lambdas/update-aggregated-nodes/handler';
+
+updateAggregatedNodes().then(() => process.exit(0)).catch(err => {
+  console.error(err);
+  process.exit(1);
+});

package/scripts/run-migrations.ts

@@ -0,0 +1,18 @@
+/* eslint-disable no-console */
+import * as dotenv from 'dotenv';
+dotenv.config();
+
+import { runMigrations } from '../database/migrations';
+
+void (async function () {
+  const { gracefulExit } = await import('exit-hook');
+
+  try {
+    await runMigrations();
+    gracefulExit(0);
+  }
+  catch (error) {
+    console.error(error);
+    gracefulExit(1);
+  }
+})();

package/scripts/run-resetdb.ts

@@ -0,0 +1,18 @@
+/* eslint-disable no-console */
+import * as dotenv from 'dotenv';
+dotenv.config();
+
+import { resetDb } from '../test/utils';
+
+void (async function () {
+  const { gracefulExit } = await import('exit-hook');
+
+  try {
+    await resetDb();
+    gracefulExit(0);
+  }
+  catch (error) {
+    console.error(error);
+    gracefulExit(1);
+  }
+})();

package/scripts/run-seed.ts

@@ -0,0 +1,18 @@
+/* eslint-disable no-console */
+import * as dotenv from 'dotenv';
+dotenv.config();
+
+import { runSeed } from '../database/seed';
+
+void (async function () {
+  const { gracefulExit } = await import('exit-hook');
+
+  try {
+    await runSeed();
+    gracefulExit(0);
+  }
+  catch (error) {
+    console.error(error);
+    gracefulExit(1);
+  }
+})();

package/serverless.yml

@@ -0,0 +1,76 @@
+frameworkVersion: '3'
+service: hestia-data-api
+
+plugins:
+  - serverless-deployment-bucket
+  - serverless-offline
+
+custom:
+  serverless-offline:
+    httpPort: 3002
+  stage: ${env:STAGE, self:provider.stage}
+  securityGroup:
+    staging: sg-0800627978a40527b
+    prod: sg-049204c3b58a8bd08
+  subnetId1:
+    # private 1a
+    staging: subnet-04ad6e355249c757d
+    prod: subnet-00c4a891bce6f48e0
+  subnetId2:
+    # private 1b
+    staging: subnet-01260544aaa3c6810
+    prod: subnet-02365f46c569f5e1a
+  debugging:
+    staging: true
+    prod: false
+  logLevel:
+    staging: DEBUG
+    prod: INFO
+  bucketData: hestia-data-${self:custom.stage}
+  domain:
+    staging: https://www-staging.hestia.earth
+    prod: https://www.hestia.earth
+
+provider:
+  name: aws
+  runtime: nodejs18.x
+  region: us-east-1
+  stage: staging
+  deploymentBucket:
+    name: hestia-serverless-${self:custom.stage}
+  memorySize: 512
+  timeout: 600
+  environment:
+    SERVICE_NAME: ${self:service}
+    STAGE: ${self:custom.stage}
+    DEBUG: ${self:custom.debugging.${self:custom.stage}}
+    LOG_LEVEL: ${self:custom.logLevel.${self:custom.stage}}
+    SENTRY_DSN: https://00ed710e2b664894a3ff3124bfff24cf@o441427.ingest.0.1.0.io/4505186792767488
+    BUCKET_DATA: ${self:custom.bucketData}
+    PGHOST: ${env:PGHOST}
+    PGDATABASE: ${env:PGDATABASE}
+    PGPORT: ${env:PGPORT}
+    PGUSER: ${env:PGUSER}
+    PGPASSWORD: ${env:PGPASSWORD}
+  iamRoleStatements:
+    - Effect: Allow
+      Action:
+        - s3:GetObject
+        - s3:GetObjectAcl
+        - s3:ListObjects
+        - s3:ListBucket
+      Resource:
+        - "arn:aws:s3:::${self:custom.bucketData}"
+        - "arn:aws:s3:::${self:custom.bucketData}/*"
+  vpc:
+    securityGroupIds:
+      - ${self:custom.securityGroup.${self:custom.stage}}
+    subnetIds:
+      - ${self:custom.subnetId1.${self:custom.stage}}
+      - ${self:custom.subnetId2.${self:custom.stage}}
+
+functions:
+  updateAggregatedNodes:
+    handler: build/src/lambdas/update-aggregated-nodes/handler.updateAggregatedNodes
+    events:
+      - schedule: cron(0 2 * * ? *) # 2:00am every day

package/src/aggregated-nodes/model/index.ts

@@ -0,0 +1,37 @@
+import { ISiteJSONLD } from '@hestia-earth/schema';
+
+export interface IFilter {
+  regions?: string[];
+  periods?: string[];
+  products?: string[];
+  defaultMethodClassifications?: string[];
+  practices?: string[];
+  minAggregatedQualityScore?: number;
+}
+
+export const validFilterFields = [
+  'regions',
+  'periods',
+  'products',
+  'defaultMethodClassifications',
+  'practices',
+  'minAggregatedQualityScore'
+];
+
+export interface IFilters {
+  cycleCount: number;
+  impactAssessmentCount: number;
+  products: string[];
+  periods: string[];
+  practices: string[];
+  minAggregatedQualityScore: number[];
+  regions: string[];
+  defaultMethodClassifications: string[];
+}
+
+// TODO: update with pivoted types when available
+export interface AggregatedNodeRow {
+  cycle: any; // pivoted cycle
+  region: ISiteJSONLD['country'];
+  impactAssessments: [any]; // pivoted impact assessments
+}

package/src/aggregated-nodes/routes/pg-get-filters.ts

@@ -0,0 +1,44 @@
+import { Request } from 'express';
+
+import { applyFilters } from '../services/pg-get-filters';
+
+/**
+ * @swagger
+ *
+ * paths:
+ *   /aggregated-nodes/pg/filters:
+ *     get:
+ *       summary: "Get the additional filters available for a filter selection and node counts (uses PG only)"
+ *       parameters:
+ *        - in: query
+ *          name: filter
+ *          schema:
+ *            type: string
+ *            example: "regions=Africa|Albania;products=abyssinianKaleStraw|abyssinianKaleSeedWhole;periods=1970-1973"
+ *          description: Set of filters
+ *        - in: query
+ *          name: query
+ *          schema:
+ *            type: string
+ *            example: Wheat
+ *          description: Free search text
+ *       responses:
+ *         200:
+ *           content:
+ *             application/json:
+ *               schema:
+ *                 type: object
+ *         400:
+ *           $ref: '#/components/responses/BadRequestError'
+ *         404:
+ *           $ref: '#/components/responses/NotFoundError'
+ *         500:
+ *           $ref: '#/components/responses/UnexpectedError'
+ *         503:
+ *           $ref: '#/components/responses/ServiceUnavailable'
+ */
+
+export const getFilters = ({ query: qs, filter }: Request) => {
+  const query = qs.query as string;
+  return applyFilters({ filter, query });
+};

package/src/aggregated-nodes/routes/pg-get.ts

@@ -0,0 +1,50 @@
+import { Request } from 'express';
+
+import { applyFilter } from '../services/pg-get';
+
+/**
+ * @swagger
+ *
+ * paths:
+ *   /aggregated-nodes/pg:
+ *     get:
+ *       summary: "Get the aggregated nodes corresponding to a set of filters and search query (uses PG only)"
+ *       parameters:
+ *        - in: query
+ *          name: filter
+ *          schema:
+ *            type: string
+ *            example: "regions=Africa|Albania;products=abyssinianKaleStraw|abyssinianKaleSeedWhole;periods=1970-1973"
+ *          description: Set of filters
+ *        - in: query
+ *          name: page
+ *          schema:
+ *            type: integer
+ *            example: 1
+ *        - in: query
+ *          name: query
+ *          schema:
+ *            type: string
+ *            example: Wheat
+ *          description: Free search text
+ *       responses:
+ *         200:
+ *           content:
+ *             application/json:
+ *               schema:
+ *                 type: object
+ *         400:
+ *           $ref: '#/components/responses/BadRequestError'
+ *         404:
+ *           $ref: '#/components/responses/NotFoundError'
+ *         500:
+ *           $ref: '#/components/responses/UnexpectedError'
+ *         503:
+ *           $ref: '#/components/responses/ServiceUnavailable'
+ */
+
+export const get = ({ query: qs, filter }: Request) => {
+  const page = Number(qs.page) || 1;
+  const query = qs.query as string;
+  return applyFilter({ page, filter, query });
+};

package/src/aggregated-nodes/routes.spec.ts

@@ -0,0 +1,242 @@
+import { expect } from 'chai';
+import * as sinon from 'sinon';
+import 'mocha';
+import express from 'express';
+import supertest from 'supertest';
+import bodyParser from 'body-parser';
+
+import router from './routes';
+import { Errors, errorHandler } from '../errors';
+import { initDb } from '../../test/utils';
+import { seed } from '../../database/seed';
+
+import * as fixtures from '../../test/fixtures/aggregated-nodes/get';
+
+let stubs: sinon.SinonStub[] = [];
+
+const app = express();
+
+const getFixtureNode = (type: 'site' | 'cycle' | 'impactAssessment', id: string) =>
+  fixtures[
+    { cycle: 'aggregatedCycles', site: 'aggregatedSites', impactAssessment: 'aggregatedImpactAssessments' }[type]
+  ].find((node) => node.jsonld_pivoted['@id'] === id).jsonld_pivoted;
+
+app.use(bodyParser.json());
+app.use(router());
+app.use(errorHandler);
+
+describe('routes', () => {
+  beforeEach(() => {
+    stubs = [];
+  });
+
+  afterEach(() => {
+    stubs.forEach((stub) => stub.restore());
+  });
+
+  describe('aggregated-nodes', () => {
+    beforeEach(async () => {
+      await initDb();
+      await seed(fixtures);
+    });
+
+    describe('GET /pg', () => {
+      it('when invalid filter passed returns invalid param error', async () => {
+        const result = await supertest(app)
+          .get('/pg')
+          .query({ filter: 'invalidfilter' })
+          .set('Accept', 'application/json');
+
+        expect(result.body.message).to.equal(Errors.InvalidQueryParam);
+      });
+
+      it('when neither filter nor query passed returns missing param error', async () => {
+        const result = await supertest(app).get('/pg').query({}).set('Accept', 'application/json');
+
+        expect(result.body.message).to.equal(Errors.MissingQueryParam);
+      });
+
+      it('can filter by regions', async () => {
+        const expected = [
+          {
+            cycle: getFixtureNode('cycle', '1'),
+            region: getFixtureNode('site', '1').country,
+            impactAssessments: [getFixtureNode('impactAssessment', '1'), getFixtureNode('impactAssessment', '2')]
+          },
+          {
+            cycle: getFixtureNode('cycle', '2'),
+            region: getFixtureNode('site', '1').country,
+            impactAssessments: [getFixtureNode('impactAssessment', '3')]
+          },
+          {
+            cycle: getFixtureNode('cycle', '3'),
+            region: getFixtureNode('site', '2').country,
+            impactAssessments: [getFixtureNode('impactAssessment', '4')]
+          }
+        ];
+
+        const filter = 'regions=GADM-POL|GADM-DEU';
+        const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
+        expect(result.body.rows).to.deep.equal(expected);
+      });
+
+      it('can filter by practices', async () => {
+        const expected = [
+          {
+            cycle: getFixtureNode('cycle', '1'),
+            region: getFixtureNode('site', '1').country,
+            impactAssessments: [getFixtureNode('impactAssessment', '1'), getFixtureNode('impactAssessment', '2')]
+          },
+          {
+            cycle: getFixtureNode('cycle', '2'),
+            region: getFixtureNode('site', '1').country,
+            impactAssessments: [getFixtureNode('impactAssessment', '3')]
+          }
+        ];
+        const filter = 'practices=earthingUpByHand|deepRipping';
+        const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
+        expect(result.body.rows).to.deep.equal(expected);
+      });
+
+      it('can filter by minimum aggregatedQualityScore', async () => {
+        const expected = [
+          {
+            cycle: getFixtureNode('cycle', '1'),
+            region: getFixtureNode('site', '1').country,
+            impactAssessments: [getFixtureNode('impactAssessment', '1'), getFixtureNode('impactAssessment', '2')]
+          }
+        ];
+        const filter = 'minAggregatedQualityScore=3';
+        const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
+        expect(result.body.rows).to.deep.equal(expected);
+      });
+
+      it('can filter by defaultMethodClassifications', async () => {
+        const expected = [
+          {
+            cycle: getFixtureNode('cycle', '2'),
+            region: getFixtureNode('site', '1').country,
+            impactAssessments: [getFixtureNode('impactAssessment', '3')]
+          },
+          {
+            cycle: getFixtureNode('cycle', '3'),
+            region: getFixtureNode('site', '2').country,
+            impactAssessments: [getFixtureNode('impactAssessment', '4')]
+          }
+        ];
+        const filter = 'defaultMethodClassifications=physical measurement|expert opinion';
+        const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
+        expect(result.body.rows).to.deep.equal(expected);
+      });
+
+      it('can filter by periods', async () => {
+        const expected = [
+          {
+            cycle: getFixtureNode('cycle', '3'),
+            region: getFixtureNode('site', '2').country,
+            impactAssessments: [getFixtureNode('impactAssessment', '4')]
+          },
+          {
+            cycle: getFixtureNode('cycle', '4'),
+            region: getFixtureNode('site', '3').country,
+            impactAssessments: []
+          }
+        ];
+        const filter = 'periods=2010-2019|2000-2009';
+        const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
+        expect(result.body.rows).to.deep.equal(expected);
+      });
+
+      it('can filter by products', async () => {
+        const expected = [
+          {
+            cycle: getFixtureNode('cycle', '1'),
+            region: getFixtureNode('site', '1').country,
+            // impactAssessments have also been filtered
+            impactAssessments: [getFixtureNode('impactAssessment', '1')]
+          },
+          {
+            cycle: getFixtureNode('cycle', '2'),
+            region: getFixtureNode('site', '1').country,
+            impactAssessments: [getFixtureNode('impactAssessment', '3')]
+          }
+        ];
+        const filter = 'products=wheatGrain|fishFingerlings';
+        const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
+        expect(result.body.rows).to.deep.equal(expected);
+      });
+
+      it('can combine multiple filters', async () => {
+        const expected = [
+          {
+            cycle: getFixtureNode('cycle', '5'),
+            region: getFixtureNode('site', '3').country,
+            // impactAssessments have also been filtered
+            impactAssessments: []
+          }
+        ];
+        const filter = [
+          'regions=GADM-POL|GADM-DEU|GADM-BRA',
+          'products=wheatGrain|fishFingerlings|cheese',
+          'periods=2010-2019|2020-2029',
+          'defaultMethodClassifications=modelled|expert opinion',
+          'minAggregatedQualityScore=2',
+          'practices=earthingUpByHand|deepRipping|baggingFruit'
+        ].join(';');
+        const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
+        expect(result.body.rows).to.deep.equal(expected);
+      });
+    });
+
+    describe('GET /pg/filters', () => {
+      it('when invalid filter passed returns invalid param error', async () => {
+        const result = await supertest(app)
+          .get('/pg/filters')
+          .query({ filter: 'invalidfilter' })
+          .set('Accept', 'application/json');
+
+        expect(result.body.message).to.equal(Errors.InvalidQueryParam);
+      });
+
+      describe('returns the filters and node counts available for the current selection', () => {
+        it('returns subset of filters corresponding to selected nodes', async () => {
+          const expected = {
+            cycleCount: 1,
+            impactAssessmentCount: 0,
+            products: ['apples', 'cheese'],
+            periods: ['2020-2029'],
+            practices: ['baggingFruit'],
+            minAggregatedQualityScore: [2],
+            regions: ['GADM-BRA'],
+            defaultMethodClassifications: ['modelled']
+          };
+          const filter = [
+            'regions=GADM-POL|GADM-DEU|GADM-BRA',
+            'products=wheatGrain|fishFingerlings|cheese',
+            'periods=2010-2019|2020-2029',
+            'defaultMethodClassifications=modelled|expert opinion',
+            'minAggregatedQualityScore=2',
+            'practices=earthingUpByHand|deepRipping|baggingFruit'
+          ].join(';');
+          const result = await supertest(app).get('/pg/filters').query({ filter }).set('Accept', 'application/json');
+          expect(result.body).to.deep.equal(expected);
+        });
+
+        it('returns all filters when there is nothing currently selected', async () => {
+          const expected = {
+            cycleCount: 8,
+            impactAssessmentCount: 4,
+            products: ['apples', 'barley', 'cheese', 'fishFingerlings', 'wheatGrain'],
+            periods: ['1990-1999', '2000-2009', '2010-2019', '2020-2029'],
+            practices: ['baggingFruit', 'deepRipping', 'earthingUpByHand'],
+            minAggregatedQualityScore: [1, 2, 3],
+            regions: ['GADM-BRA', 'GADM-DEU', 'GADM-POL'],
+            defaultMethodClassifications: ['expert opinion', 'modelled', 'physical measurement', 'unsourced assumption']
+          };
+          const result = await supertest(app).get('/pg/filters').query({}).set('Accept', 'application/json');
+          expect(result.body).to.deep.equal(expected);
+        });
+      });
+    });
+  });
+});