@hestia-earth/data-api 0.0.2-3 → 0.0.2-4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hestia-earth/data-api",
-  "version": "0.0.2-3",
+  "version": "0.0.2-4",
   "description": "Hestia Data API definitions",
   "main": "dist/models.js",
   "typings": "dist/models.d.ts",

package/.dockerignore

@@ -1,25 +0,0 @@
-*.DS_Store
-.gitignore
-.env
-.git
-.gitlab*
-.npmignore
-*.md
-*.sh
-.nvmrc
-.eslintrc.js
-.nycrc
-*.spec.ts
-.nyc_output/
-build/
-coverage/
-dist/
-logs/
-node_modules/
-test/
-docker-compose*
-Dockerfile
-commitlint.config.js
-build-lambdas/
-tsconfig.lambdas.json
-envs/

package/.env.test

@@ -1,7 +0,0 @@
-PGHOST=localhost
-PGUSER=postgres
-PGDATABASE=postgres
-PGPASSWORD=password
-PGPORT=5432
-
-WRITE_SETTINGS_SECRET=dummy

package/.eslintignore

@@ -1,7 +0,0 @@
-.eslintrc.js
-coverage
-build
-build-lambdas
-commitlint.config.js
-.mocharc.js
-dist

package/.eslintrc.js

@@ -1,11 +0,0 @@
-module.exports = {
-  env: {
-    node: true
-  },
-  extends: ['@hestia-earth/eslint-config'],
-  parser: '@typescript-eslint/parser',
-  parserOptions: {
-    project: 'tsconfig.json',
-    sourceType: 'module'
-  }
-};

package/.gitlab-ci.yml

@@ -1,128 +0,0 @@
-stages:
-  - test
-  - publish
-  - deploy
-
-default:
-  image: node:18
-  cache:
-    paths:
-      - node_modules/
-  before_script:
-    - npm ci --include=dev
-
-variables:
-  CONTAINER_IMAGE: hestia-data-api
-  CONTAINER_IMAGE_BUILD: $CONTAINER_IMAGE:$CI_COMMIT_REF_NAME
-
-lint:
-  stage: test
-  script:
-    - npm run lint
-  except:
-    - tags
-
-lint-commit:
-  stage: test
-  script:
-    - git fetch origin develop
-    - npx commitlint --from=origin/develop
-  except:
-    - tags
-    - develop
-    - master
-
-test:
-  stage: test
-  services:
-    - postgres:14
-  coverage: /^Statements\s*:\s*([^%]+)/
-  artifacts:
-    paths:
-      - coverage/
-    reports:
-      coverage_report:
-        coverage_format: cobertura
-        path: coverage/cobertura-coverage.xml
-  variables:
-    PGHOST: postgres
-    PGUSER: postgres
-    PGDATABASE: "${CI_COMMIT_SHORT_SHA}"
-    POSTGRES_DB: "${CI_COMMIT_SHORT_SHA}"
-    PGPORT: 5432
-    PGPASSWORD: password
-    POSTGRES_PASSWORD: password
-    WRITE_SETTINGS_SECRET: dummy
-    ENVIRONMENT: test
-  script:
-    - npm test
-  except:
-    - tags
-
-publish-sources:
-  stage: publish
-  artifacts:
-    paths:
-      - build/
-  script:
-    - npm run build
-  only:
-    - tags
-
-publish-npm:
-  stage: publish
-  script:
-    - npm run build:module
-    - npm config set //registry.npmjs.org/:_authToken=${NPM_TOKEN}
-    - rm *.md
-    - npm publish --access public --verbose
-  only:
-    - tags
-
-publish-docker:
-  image: docker:stable
-  stage: publish
-  services:
-    - docker:dind
-  before_script:
-    - apk add --no-cache curl jq python3 py3-pip
-    # fix error PyYAML
-    - pip install pyyaml==5.3.1
-    - pip install awscli
-    - $(aws ecr get-login --no-include-email --region us-east-1)
-  script:
-    - docker build -f Dockerfile -t $CONTAINER_REGISTRY/$CONTAINER_IMAGE_BUILD .
-    - docker push $CONTAINER_REGISTRY/$CONTAINER_IMAGE_BUILD
-    # cleanup
-    - ./cleanup-docker.sh $CONTAINER_IMAGE
-  only:
-    - tags
-
-deploy-lambdas:
-  stage: deploy
-  script:
-    - source envs/.${CI_COMMIT_REF_NAME}.env
-    - npm run build:lambdas
-    # copy serverless package to include correct dependencies
-    - rm -rf package.json node_modules
-    - mv package.serverless.json package.json && npm ci --include=dev
-    - npx serverless deploy --verbose
-  only:
-    - master
-    - staging
-
-sentry:
-  stage: deploy
-  image: getsentry/sentry-cli
-  before_script:
-    - export SENTRY_PROJECT=hestia-data-api
-    - export RELEASE=$(echo $CI_COMMIT_REF_NAME| cut -c2-)
-  script:
-    - echo "Create a new release $RELEASE..."
-    - sentry-cli releases new $RELEASE
-    - sentry-cli releases set-commits --auto $RELEASE
-    - sentry-cli releases files $RELEASE upload-sourcemaps $CI_PROJECT_DIR/build -x .js -x .map --validate --verbose --rewrite --strip-common-prefix
-    - sentry-cli releases finalize $RELEASE
-    - echo "Create a new release $RELEASE DONE"
-  only:
-    - tags

package/.mocharc.js

@@ -1,8 +0,0 @@
-'use strict';
-
-module.exports = {
-  require: ['ts-node/register', 'source-map-support/register', 'test/prepare.ts'],
-  'full-trace': true,
-  'watch-files': 'src',
-  timeout: 10000
-};

package/.nvmrc

@@ -1 +0,0 @@
-18

package/.nycrc

@@ -1,15 +0,0 @@
-{
-  "check-coverage": true,
-  "per-file": false,
-  "statements": 95,
-  "branches": 85,
-  "functions": 95,
-  "lines": 95,
-  "include": ["src/**/*.ts"],
-  "exclude": ["src/swagger/*.ts", "src/**/*.spec.ts"],
-  "extension": [".ts"],
-  "require": ["ts-node/register"],
-  "reporter": ["text-summary", "html", "lcov", "cobertura"],
-  "sourceMap": true,
-  "instrument": true
-}

package/Dockerfile

@@ -1,17 +0,0 @@
-FROM node:18-slim
-
-WORKDIR /app
-
-COPY package.json .
-COPY package-lock.json .
-RUN npm ci --include=dev
-
-# copy source from context
-ADD . .
-# build
-RUN npm run build
-
-RUN mkdir -p logs
-
-COPY index.js ./
-CMD npm start

package/cleanup-docker.sh

@@ -1,4 +0,0 @@
-#!/bin/sh
-
-IMAGES_TO_DELETE=$( aws ecr list-images --region us-east-1 --repository-name $1 --filter "tagStatus=UNTAGGED" --query 'imageIds[*]' --output json )
-aws ecr batch-delete-image --region us-east-1 --repository-name $1 --image-ids "$IMAGES_TO_DELETE" || true

package/commitlint.config.js

@@ -1 +0,0 @@
-module.exports = { extends: ['@commitlint/config-conventional'] };

package/database/index.ts

@@ -1,76 +0,0 @@
-import Debug from 'debug';
-import pg from 'pg';
-pg.defaults.parseInt8 = true;
-import { logger } from '../src/logger';
-
-const debug = Debug('@hestia/data-api:db');
-
-logger.info('Creating pg pool');
-
-const pool = new pg.Pool({
-  connectionTimeoutMillis: 10000,
-  max: 4
-});
-
-pool.on('error', (err) => logger.error('Pool error:', err));
-
-export const endPool = () => pool.end();
-
-export const query = async <R extends pg.QueryResultRow = any, I extends any[] = any[]>(
-  queryTextOrConfig: string | pg.QueryConfig<I>,
-  params?: I | undefined
-): Promise<pg.QueryResult<R>> => {
-  const start = Date.now();
-  try {
-    const res = await pool.query<R, I>(queryTextOrConfig, params);
-    const duration = Date.now() - start;
-    debug('Executed query %O', { queryTextOrConfig, duration, rows: res.rowCount });
-    return res;
-  }
-  catch (e) {
-    logger.error('Query failed', { queryTextOrConfig, params });
-    throw e;
-  }
-};
-
-export const getClient = () => pool.connect();
-
-type OperationsFn<R extends pg.QueryResultRow> = (client: pg.PoolClient) => Promise<pg.QueryResult<R>>;
-
-export async function transaction<R extends pg.QueryResultRow>(operations: OperationsFn<R>) {
-  const start = Date.now();
-  const client = await pool.connect();
-  const gotClient = Date.now() - start;
-  let committed: number | null = null;
-  let released: number;
-  try {
-    await client.query('BEGIN');
-    const result = await operations(client);
-    await client.query('COMMIT');
-    committed = Date.now() - start;
-    return result;
-  }
-  catch (e) {
-    await client.query('ROLLBACK');
-    throw e;
-  }
-  finally {
-    client.release();
-    released = Date.now() - start;
-    debug('Executed transaction', { gotClient, committed, released });
-  }
-}
-
-void (async () => {
-  const { asyncExitHook } = await import('exit-hook');
-
-  asyncExitHook(
-    async () => {
-      logger.info('Closing pool...');
-      await endPool();
-    },
-    {
-      minimumWait: 300
-    }
-  );
-})();

package/database/migrations/001.do.init.sql

@@ -1,53 +0,0 @@
-CREATE TABLE
-  aggregated_cycles (
-    id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
-    hestia_id TEXT UNIQUE NOT NULL GENERATED ALWAYS AS (jsonld_pivoted->>'@id') STORED,
-    jsonld_pivoted JSONB NOT NULL,
-    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
-  );
-
-CREATE TABLE
-  aggregated_impact_assessments (
-    id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
-    cycle_hestia_id TEXT REFERENCES aggregated_cycles(hestia_id) NOT NULL GENERATED ALWAYS AS (jsonld_pivoted->'cycle'->>'@id') STORED,
-    hestia_id TEXT UNIQUE NOT NULL GENERATED ALWAYS AS (jsonld_pivoted->>'@id') STORED,
-    jsonld_pivoted JSONB NOT NULL,
-    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
-  );
-
-CREATE TYPE setting_type AS ENUM ('maintenanceEnabled');
-
-CREATE TABLE
-  settings (
-    id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
-    setting setting_type NOT NULL UNIQUE,
-    active BOOLEAN NOT NULL DEFAULT FALSE,
-    metadata JSONB NOT NULL DEFAULT '{}',
-    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
-  );
-
-CREATE OR REPLACE FUNCTION trigger_set_timestamp()
-RETURNS TRIGGER AS $$
-BEGIN
-  NEW.updated_at = NOW();
-  RETURN NEW;
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE TRIGGER set_timestamp
-BEFORE UPDATE ON aggregated_cycles
-FOR EACH ROW
-EXECUTE PROCEDURE trigger_set_timestamp();
-
-CREATE TRIGGER set_timestamp
-BEFORE UPDATE ON aggregated_impact_assessments
-FOR EACH ROW
-EXECUTE PROCEDURE trigger_set_timestamp();
-
-CREATE TRIGGER set_timestamp
-BEFORE UPDATE ON settings
-FOR EACH ROW
-EXECUTE PROCEDURE trigger_set_timestamp();

package/database/migrations/002.do.add-aggregated-sites.sql

@@ -1,16 +0,0 @@
-CREATE TABLE
-  aggregated_sites (
-    id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
-    hestia_id TEXT UNIQUE NOT NULL GENERATED ALWAYS AS (jsonld_pivoted->>'@id') STORED,
-    jsonld_pivoted JSONB NOT NULL,
-    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
-  );
-
-CREATE TRIGGER set_timestamp
-BEFORE UPDATE ON aggregated_sites
-FOR EACH ROW
-EXECUTE PROCEDURE trigger_set_timestamp();
-
-ALTER TABLE aggregated_cycles
-ADD COLUMN site_hestia_id TEXT REFERENCES aggregated_sites(hestia_id) NOT NULL GENERATED ALWAYS AS (jsonld_pivoted->'site'->>'@id') STORED;

package/database/migrations/003.do.add-generated-period-cols.sql

@@ -1,7 +0,0 @@
-ALTER TABLE aggregated_impact_assessments
-ADD COLUMN generated_period TEXT
-GENERATED ALWAYS AS ((jsonld_pivoted::jsonb->>'startDate') || '-' || (jsonld_pivoted::jsonb->>'endDate')) STORED;
-
-ALTER TABLE aggregated_cycles
-ADD COLUMN generated_period TEXT
-GENERATED ALWAYS AS ((jsonld_pivoted::jsonb->>'startDate') || '-' || (jsonld_pivoted::jsonb->>'endDate')) STORED;

package/database/migrations/index.ts

@@ -1,36 +0,0 @@
-/* eslint-disable no-console */
-import { join } from 'path';
-import Debug from 'debug';
-
-import { query } from '../index';
-
-const debug = Debug('@hestia/data-api:db:migrate');
-
-export const runMigrations = async () => {
-  const { default: Postgrator } = await import('postgrator');
-
-  try {
-    const postgrator = new Postgrator({
-      migrationPattern: join(__dirname, '/*.sql'),
-      driver: 'pg',
-      execQuery: query
-    });
-
-    postgrator.on('validation-started', (migration) => debug('Started validation:', migration.filename));
-    postgrator.on('validation-finished', (migration) => debug('Finished validation:', migration.filename));
-    postgrator.on('migration-started', (migration) => debug('Started migration:', migration.filename));
-    postgrator.on('migration-finished', (migration) => debug('Finished migration:', migration.filename));
-
-    // Migrate to specific version -> postgrator.migrate('002')
-    const appliedMigrations = await postgrator.migrate();
-
-    debug(
-      'Applied:',
-      appliedMigrations.map((m) => m.filename)
-    );
-  }
-  catch (error) {
-    console.log('Applied:', error.appliedMigrations);
-    throw error;
-  }
-};

package/database/seed/common.ts

@@ -1,7 +0,0 @@
-export const settings = [
-  {
-    setting: 'maintenanceEnabled',
-    active: false,
-    metadata: {}
-  }
-];

package/database/seed/index.ts

@@ -1,60 +0,0 @@
-import Debug from 'debug';
-
-import { query } from '..';
-import * as commonSeeds from './common';
-
-const debug = Debug('@hestia/data-api:db:seed');
-
-export const seed = async ({
-  aggregatedCycles = [],
-  aggregatedImpactAssessments = [],
-  aggregatedSites = [],
-  settings = []
-}) => {
-  for (const site of aggregatedSites) {
-    const { rows } = await query(
-      `INSERT INTO aggregated_sites (jsonld_pivoted) VALUES ($1)
-       ON CONFLICT DO NOTHING
-       RETURNING id`,
-      [site.jsonld_pivoted]
-    );
-    const rowId = rows[0]?.id;
-    debug('Inserted aggregated site with ID:', rowId);
-  }
-  for (const cycle of aggregatedCycles) {
-    const { rows } = await query(
-      `INSERT INTO aggregated_cycles (jsonld_pivoted) VALUES ($1)
-       ON CONFLICT DO NOTHING
-       RETURNING id`,
-      [cycle.jsonld_pivoted]
-    );
-    const rowId = rows[0]?.id;
-    debug('Inserted aggregated cycle with ID:', rowId);
-  }
-  for (const ia of aggregatedImpactAssessments) {
-    const { rows } = await query(
-      `INSERT INTO aggregated_impact_assessments (jsonld_pivoted) VALUES ($1)
-       ON CONFLICT DO NOTHING
-       RETURNING id`,
-      [ia.jsonld_pivoted]
-    );
-    const rowId = rows[0]?.id;
-    debug('Inserted aggregated impact assessment with ID:', rowId);
-  }
-  for (const setting of settings) {
-    const { rows } = await query(
-      `INSERT INTO settings (setting, active, metadata) VALUES ($1, $2, $3)
-       ON CONFLICT DO NOTHING
-       RETURNING id`,
-      [setting.setting, setting.active, setting.metadata]
-    );
-    const rowId = rows[0]?.id;
-    debug('Inserted setting with ID:', rowId);
-  }
-};
-
-export const runSeed = async () => {
-  const seedFile = `./${process.env.ENVIRONMENT || 'local'}`;
-  const envSeeds = require(seedFile);
-  await seed({ ...commonSeeds, ...envSeeds });
-};

package/database/seed/local/index.ts

@@ -1,28 +0,0 @@
-export const aggregatedCycles = [
-  {
-    jsonld_pivoted: {
-      '@id': 'test_cycle_id',
-      '@type': 'Cycle',
-      site: { '@id': 'test_site_id' }
-    }
-  }
-];
-
-export const aggregatedImpactAssessments = [
-  {
-    jsonld_pivoted: {
-      '@id': 'test_impact_assessment_id',
-      '@type': 'ImpactAssessment',
-      cycle: { '@id': 'test_cycle_id' }
-    }
-  }
-];
-
-export const aggregatedSites = [
-  {
-    jsonld_pivoted: {
-      '@id': 'test_site_id',
-      '@type': 'Site'
-    }
-  }
-];

package/database/seed/production/index.ts

@@ -1,3 +0,0 @@
-export const aggregatedCycles = [];
-
-export const aggregatedImpactAssessments = [];

package/database/seed/staging/index.ts

@@ -1,5 +0,0 @@
-export const aggregatedCycles = [];
-
-export const aggregatedImpactAssessments = [];
-
-export const aggregatedSites = [];

package/dev.ts

@@ -1,3 +0,0 @@
-require('dotenv').config();
-
-require('./src').run();

package/dist/aggregated-nodes/model/index.js

@@ -1,11 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.validFilterFields = void 0;
-exports.validFilterFields = [
-    'regions',
-    'periods',
-    'products',
-    'defaultMethodClassifications',
-    'practices',
-    'minAggregatedQualityScore'
-];

package/docker-compose.yml

@@ -1,42 +0,0 @@
-version: '3'
-
-services:
-  api:
-    image: hestia-data-api
-    build:
-      context: ./
-    ports:
-      - '${PORT}:${PORT}'
-    env_file: .env
-    command: sh -c 'npm run migrate && npm run seed && npm run dev'
-    environment:
-      - PGHOST=hestia-data-api-local-db
-      - PGUSER=postgres
-      - PGDATABASE=postgres
-      - PGPASSWORD=password
-    volumes:
-      - ./logs:/app/logs
-      - ./scripts:/app/scripts
-      - ./src:/app/src
-    depends_on:
-      db:
-        condition: service_healthy
-
-  db:
-    image: postgres:14
-    container_name: hestia-data-api-local-db
-    ports:
-      - 5432:5432
-    volumes:
-      - postgres:/data/postgres
-    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=password
-      - POSTGRES_DB=postgres
-    healthcheck:
-      test: ['CMD', 'pg_isready', '-q', '-d', 'postgres', '-U', 'postgres']
-      interval: 1s
-      timeout: 5s
-      retries: 5
-volumes:
-  postgres:

package/envs/.master.env

@@ -1,7 +0,0 @@
-export ENVIRONMENT='production'
-export STAGE='prod'
-export PGHOST=$PG_HOST_PROD
-export PGPASSWORD=$PG_PASSWORD_PROD
-export PGDATABASE=$PG_DATABASE
-export PGPORT=$PG_PORT
-export PGUSER=$PG_USER

package/envs/.staging.env

@@ -1,7 +0,0 @@
-export ENVIRONMENT='staging'
-export STAGE='staging'
-export PGHOST=$PG_HOST_STAGING
-export PGPASSWORD=$PG_PASSWORD_STAGING
-export PGDATABASE=$PG_DATABASE
-export PGPORT=$PG_PORT
-export PGUSER=$PG_USER

package/index.js

@@ -1,3 +0,0 @@
-require('dotenv').config();
-
-require('./build/src').run();

package/package.serverless.json

@@ -1,21 +0,0 @@
-{
-  "name": "@hestia-earth/data-api-serverless",
-  "dependencies": {
-    "@hestia-earth/schema": "*",
-    "@hestia-earth/pipeline-utils": "*",
-    "@sentry/serverless": "*",
-    "csvtojson": "*",
-    "debug": "*",
-    "exit-hook": "*",
-    "lodash.chunk": "*",
-    "lodash.uniqby": "*",
-    "map-obj": "*",
-    "pg": "*",
-    "winston": "*"
-  },
-  "devDependencies": {
-    "serverless": "^3.32.2",
-    "serverless-deployment-bucket": "^1.6.0",
-    "serverless-offline": "^12.0.4"
-  }
-}

package/run-docker.sh

@@ -1,14 +0,0 @@
-#!/bin/sh
-
-docker build --progress=plain \
-  -t hestia-data-api:latest \
-  .
-
-docker run --rm \
-  --name hestia-data-api \
-  --env-file .env \
-  --env PORT=80 \
-  -v ${PWD}/scripts:/app/scripts \
-  -v ${PWD}/src:/app/src \
-  -p 3001:80 \
-  hestia-data-api:latest npm run dev

package/run-test.sh

@@ -1,5 +0,0 @@
-#!/bin/sh
-
-docker-compose -f test/docker-compose.yml build
-docker-compose -f test/docker-compose.yml run test
-docker-compose -f test/docker-compose.yml down