@hestia-earth/data-api 0.0.2-1 → 0.0.2-3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/.gitlab-ci.yml CHANGED
@@ -9,7 +9,7 @@ default:
9
9
  paths:
10
10
  - node_modules/
11
11
  before_script:
12
- - npm install
12
+ - npm ci --include=dev
13
13
 
14
14
  variables:
15
15
  CONTAINER_IMAGE: hestia-data-api
@@ -103,6 +103,9 @@ deploy-lambdas:
103
103
  script:
104
104
  - source envs/.${CI_COMMIT_REF_NAME}.env
105
105
  - npm run build:lambdas
106
+ # copy serverless package to include correct dependencies
107
+ - rm -rf package.json node_modules
108
+ - mv package.serverless.json package.json && npm ci --include=dev
106
109
  - npx serverless deploy --verbose
107
110
  only:
108
111
  - master
package/.mocharc.js CHANGED
@@ -4,5 +4,5 @@ module.exports = {
4
4
  require: ['ts-node/register', 'source-map-support/register', 'test/prepare.ts'],
5
5
  'full-trace': true,
6
6
  'watch-files': 'src',
7
- timeout: 5000
7
+ timeout: 10000
8
8
  };
package/Dockerfile CHANGED
@@ -4,7 +4,7 @@ WORKDIR /app
4
4
 
5
5
  COPY package.json .
6
6
  COPY package-lock.json .
7
- RUN npm ci
7
+ RUN npm ci --include=dev
8
8
 
9
9
  # copy source from context
10
10
  ADD . .
@@ -5,7 +5,12 @@ import * as commonSeeds from './common';
5
5
 
6
6
  const debug = Debug('@hestia/data-api:db:seed');
7
7
 
8
- export const seed = async ({ aggregatedCycles, aggregatedImpactAssessments, aggregatedSites, settings }) => {
8
+ export const seed = async ({
9
+ aggregatedCycles = [],
10
+ aggregatedImpactAssessments = [],
11
+ aggregatedSites = [],
12
+ settings = []
13
+ }) => {
9
14
  for (const site of aggregatedSites) {
10
15
  const { rows } = await query(
11
16
  `INSERT INTO aggregated_sites (jsonld_pivoted) VALUES ($1)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@hestia-earth/data-api",
3
- "version": "0.0.2-1",
3
+ "version": "0.0.2-3",
4
4
  "description": "Hestia Data API definitions",
5
5
  "main": "dist/models.js",
6
6
  "typings": "dist/models.d.ts",
@@ -10,7 +10,7 @@
10
10
  "build:lambdas": "rm -rf build-lambdas && tsc -p tsconfig.lambdas.json",
11
11
  "start": "node index.js",
12
12
  "dev": "ts-node-dev --respawn --rs dev.ts --files --ignore-watch node_modules",
13
- "lint": "eslint .",
13
+ "lint": "tsc --noEmit && eslint .",
14
14
  "lint:fix": "npm run lint -- --fix",
15
15
  "test": "nyc mocha \"./**/*.spec.ts\"",
16
16
  "migrate": "ts-node scripts/run-migrations.ts",
@@ -39,31 +39,14 @@
39
39
  "@hestia-earth/schema": "^22.1.0"
40
40
  },
41
41
  "devDependencies": {
42
- "@hestia-earth/pipeline-utils": "^0.10.4",
42
+ "@commitlint/cli": "^17.6.5",
43
+ "@commitlint/config-conventional": "^17.6.5",
44
+ "@hestia-earth/eslint-config": "^0.0.5",
45
+ "@hestia-earth/pipeline-utils": "^0.12.1",
43
46
  "@sentry/node": "^7.51.2",
44
47
  "@sentry/serverless": "^7.55.2",
45
48
  "@sentry/tracing": "^7.51.2",
46
49
  "@slack/web-api": "^6.8.1",
47
- "compression": "^1.7.4",
48
- "cors": "^2.8.5",
49
- "csvtojson": "^2.0.10",
50
- "dotenv": "^16.0.3",
51
- "exit-hook": "^3.2.0",
52
- "express": "^4.17.3",
53
- "express-async-errors": "^3.1.1",
54
- "http2-express-bridge": "^1.0.7",
55
- "lodash.chunk": "^4.2.0",
56
- "lodash.uniqby": "^4.7.0",
57
- "map-obj": "^5.0.2",
58
- "morgan": "^1.10.0",
59
- "pg": "^8.11.0",
60
- "rxjs": "^7.8.1",
61
- "swagger-jsdoc": "^6.2.8",
62
- "swagger-ui-express": "^4.6.3",
63
- "winston": "^3.8.2",
64
- "@commitlint/cli": "^17.6.5",
65
- "@commitlint/config-conventional": "^17.6.5",
66
- "@hestia-earth/eslint-config": "^0.0.5",
67
50
  "@types/chai": "^4.3.5",
68
51
  "@types/chai-as-promised": "^7.1.5",
69
52
  "@types/compression": "^1.7.2",
@@ -82,20 +65,38 @@
82
65
  "@types/swagger-ui-express": "^4.1.3",
83
66
  "chai": "^4.3.7",
84
67
  "chai-as-promised": "^7.1.1",
68
+ "compression": "^1.7.4",
69
+ "cors": "^2.8.5",
70
+ "csvtojson": "^2.0.10",
71
+ "debug": "^4.3.4",
72
+ "dotenv": "^16.0.3",
85
73
  "eslint": "^7.32.0",
74
+ "exit-hook": "^3.2.0",
75
+ "express": "^4.17.3",
76
+ "express-async-errors": "^3.1.1",
77
+ "http2-express-bridge": "^1.0.7",
86
78
  "husky": "^4.3.8",
87
- "mocha-prepare": "^0.1.0",
79
+ "lodash.chunk": "^4.2.0",
80
+ "lodash.uniqby": "^4.7.0",
81
+ "map-obj": "^5.0.2",
82
+ "mocha": "^10.2.0",
83
+ "morgan": "^1.10.0",
88
84
  "nyc": "^15.1.0",
85
+ "pg": "^8.11.0",
89
86
  "postgrator": "^7.1.1",
87
+ "rxjs": "^7.8.1",
90
88
  "serverless": "^3.32.2",
91
89
  "serverless-deployment-bucket": "^1.6.0",
92
90
  "serverless-offline": "^12.0.4",
93
91
  "sinon": "^15.1.0",
94
92
  "standard-version": "^9.5.0",
95
93
  "supertest": "^6.3.3",
94
+ "swagger-jsdoc": "^6.2.8",
95
+ "swagger-ui-express": "^4.6.3",
96
96
  "ts-node": "^10.9.1",
97
97
  "ts-node-dev": "^2.0.0",
98
- "typescript": "^5.0.0"
98
+ "typescript": "^5.0.0",
99
+ "winston": "^3.8.2"
99
100
  },
100
101
  "husky": {
101
102
  "hooks": {
@@ -0,0 +1,21 @@
1
+ {
2
+ "name": "@hestia-earth/data-api-serverless",
3
+ "dependencies": {
4
+ "@hestia-earth/schema": "*",
5
+ "@hestia-earth/pipeline-utils": "*",
6
+ "@sentry/serverless": "*",
7
+ "csvtojson": "*",
8
+ "debug": "*",
9
+ "exit-hook": "*",
10
+ "lodash.chunk": "*",
11
+ "lodash.uniqby": "*",
12
+ "map-obj": "*",
13
+ "pg": "*",
14
+ "winston": "*"
15
+ },
16
+ "devDependencies": {
17
+ "serverless": "^3.32.2",
18
+ "serverless-deployment-bucket": "^1.6.0",
19
+ "serverless-offline": "^12.0.4"
20
+ }
21
+ }
package/serverless.yml CHANGED
@@ -1,10 +1,38 @@
1
1
  frameworkVersion: '3'
2
+
2
3
  service: hestia-data-api
3
4
 
4
5
  plugins:
5
6
  - serverless-deployment-bucket
6
7
  - serverless-offline
7
8
 
9
+ package:
10
+ exclude:
11
+ - node_modules/@types/**
12
+ # cannot be removed as used in migrations
13
+ # - node_modules/typescript/**
14
+ - node_modules/**/*.map
15
+ - node_modules/**/*.d.ts
16
+ - node_modules/**/coverage/**
17
+ - node_modules/**/tests/**
18
+ - node_modules/**/test/**
19
+ - .nyc_output/**
20
+ - coverage/**
21
+ - data/**
22
+ - docs/**
23
+ - envs/**
24
+ - samples/**
25
+ - scripts/**
26
+ - test/**
27
+ - tmp/**
28
+ - tsconfig*
29
+ - "*.yml"
30
+ - "*.md"
31
+ - "*.sh"
32
+ - ".dockerignore"
33
+ - "Dockerfile"
34
+ - "gulpfile.js"
35
+
8
36
  custom:
9
37
  serverless-offline:
10
38
  httpPort: 3002
@@ -26,10 +54,7 @@ custom:
26
54
  logLevel:
27
55
  staging: DEBUG
28
56
  prod: INFO
29
- bucketData: hestia-data-${self:custom.stage}
30
- domain:
31
- staging: https://www-staging.hestia.earth
32
- prod: https://www.hestia.earth
57
+ bucket: hestia-data-${self:custom.stage}
33
58
 
34
59
  provider:
35
60
  name: aws
@@ -45,8 +70,8 @@ provider:
45
70
  STAGE: ${self:custom.stage}
46
71
  DEBUG: ${self:custom.debugging.${self:custom.stage}}
47
72
  LOG_LEVEL: ${self:custom.logLevel.${self:custom.stage}}
48
- SENTRY_DSN: https://00ed710e2b664894a3ff3124bfff24cf@o441427.ingest.0.1.0.io/4505186792767488
49
- BUCKET_DATA: ${self:custom.bucketData}
73
+ SENTRY_DSN: https://00ed710e2b664894a3ff3124bfff24cf@o441427.ingest.sentry.io/4505186792767488
74
+ BUCKET: ${self:custom.bucket}
50
75
  PGHOST: ${env:PGHOST}
51
76
  PGDATABASE: ${env:PGDATABASE}
52
77
  PGPORT: ${env:PGPORT}
@@ -60,8 +85,8 @@ provider:
60
85
  - s3:ListObjects
61
86
  - s3:ListBucket
62
87
  Resource:
63
- - "arn:aws:s3:::${self:custom.bucketData}"
64
- - "arn:aws:s3:::${self:custom.bucketData}/*"
88
+ - "arn:aws:s3:::${self:custom.bucket}"
89
+ - "arn:aws:s3:::${self:custom.bucket}/*"
65
90
  vpc:
66
91
  securityGroupIds:
67
92
  - ${self:custom.securityGroup.${self:custom.stage}}
@@ -14,7 +14,17 @@ import { applyFilters } from '../services/pg-get-filters';
14
14
  * name: filter
15
15
  * schema:
16
16
  * type: string
17
- * example: "regions=Africa|Albania;products=abyssinianKaleStraw|abyssinianKaleSeedWhole;periods=1970-1973"
17
+ * examples:
18
+ * example1:
19
+ * summary: Barley in Australia and India between 2010 and 2019.
20
+ * value: "regions=GADM-AUS|GADM-IND;\
21
+ * products=barleyGrainWhole|barleyStraw;\
22
+ * periods=2010-2019"
23
+ * example2:
24
+ * summary: Wheat and Maize in France, United Kingdom and Worldwide between 2000 and 2019.
25
+ * value: "regions=region-world|GADM-GBR|GADM-FRA;\
26
+ * products=wheatGrain|maizeGrain;\
27
+ * periods=2000-2009|2010-2019"
18
28
  * description: Set of filters
19
29
  * - in: query
20
30
  * name: query
@@ -1,3 +1,4 @@
1
+ /* eslint-disable max-len */
1
2
  import { Request } from 'express';
2
3
 
3
4
  import { applyFilter } from '../services/pg-get';
@@ -14,7 +15,17 @@ import { applyFilter } from '../services/pg-get';
14
15
  * name: filter
15
16
  * schema:
16
17
  * type: string
17
- * example: "regions=Africa|Albania;products=abyssinianKaleStraw|abyssinianKaleSeedWhole;periods=1970-1973"
18
+ * examples:
19
+ * example1:
20
+ * summary: Barley in Australia and India between 2010 and 2019.
21
+ * value: "regions=GADM-AUS|GADM-IND;\
22
+ * products=barleyGrainWhole|barleyStraw;\
23
+ * periods=2010-2019"
24
+ * example2:
25
+ * summary: Wheat and Maize in France, United Kingdom and Worldwide between 2000 and 2019.
26
+ * value: "regions=region-world|GADM-GBR|GADM-FRA;\
27
+ * products=wheatGrain|maizeGrain;\
28
+ * periods=2000-2009|2010-2019"
18
29
  * description: Set of filters
19
30
  * - in: query
20
31
  * name: page
@@ -77,7 +77,7 @@ describe('routes', () => {
77
77
 
78
78
  const filter = 'regions=GADM-POL|GADM-DEU';
79
79
  const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
80
- expect(result.body.rows).to.deep.equal(expected);
80
+ expect(result.body.results).to.deep.equal(expected);
81
81
  });
82
82
 
83
83
  it('can filter by practices', async () => {
@@ -95,7 +95,7 @@ describe('routes', () => {
95
95
  ];
96
96
  const filter = 'practices=earthingUpByHand|deepRipping';
97
97
  const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
98
- expect(result.body.rows).to.deep.equal(expected);
98
+ expect(result.body.results).to.deep.equal(expected);
99
99
  });
100
100
 
101
101
  it('can filter by minimum aggregatedQualityScore', async () => {
@@ -108,7 +108,7 @@ describe('routes', () => {
108
108
  ];
109
109
  const filter = 'minAggregatedQualityScore=3';
110
110
  const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
111
- expect(result.body.rows).to.deep.equal(expected);
111
+ expect(result.body.results).to.deep.equal(expected);
112
112
  });
113
113
 
114
114
  it('can filter by defaultMethodClassifications', async () => {
@@ -126,7 +126,7 @@ describe('routes', () => {
126
126
  ];
127
127
  const filter = 'defaultMethodClassifications=physical measurement|expert opinion';
128
128
  const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
129
- expect(result.body.rows).to.deep.equal(expected);
129
+ expect(result.body.results).to.deep.equal(expected);
130
130
  });
131
131
 
132
132
  it('can filter by periods', async () => {
@@ -144,7 +144,7 @@ describe('routes', () => {
144
144
  ];
145
145
  const filter = 'periods=2010-2019|2000-2009';
146
146
  const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
147
- expect(result.body.rows).to.deep.equal(expected);
147
+ expect(result.body.results).to.deep.equal(expected);
148
148
  });
149
149
 
150
150
  it('can filter by products', async () => {
@@ -163,7 +163,7 @@ describe('routes', () => {
163
163
  ];
164
164
  const filter = 'products=wheatGrain|fishFingerlings';
165
165
  const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
166
- expect(result.body.rows).to.deep.equal(expected);
166
+ expect(result.body.results).to.deep.equal(expected);
167
167
  });
168
168
 
169
169
  it('can combine multiple filters', async () => {
@@ -184,7 +184,7 @@ describe('routes', () => {
184
184
  'practices=earthingUpByHand|deepRipping|baggingFruit'
185
185
  ].join(';');
186
186
  const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
187
- expect(result.body.rows).to.deep.equal(expected);
187
+ expect(result.body.results).to.deep.equal(expected);
188
188
  });
189
189
  });
190
190
 
@@ -224,10 +224,10 @@ describe('routes', () => {
224
224
 
225
225
  it('returns all filters when there is nothing currently selected', async () => {
226
226
  const expected = {
227
- cycleCount: 8,
227
+ cycleCount: 9,
228
228
  impactAssessmentCount: 4,
229
229
  products: ['apples', 'barley', 'cheese', 'fishFingerlings', 'wheatGrain'],
230
- periods: ['1990-1999', '2000-2009', '2010-2019', '2020-2029'],
230
+ periods: ['1980-1989', '1990-1999', '2000-2009', '2010-2019', '2020-2029'],
231
231
  practices: ['baggingFruit', 'deepRipping', 'earthingUpByHand'],
232
232
  minAggregatedQualityScore: [1, 2, 3],
233
233
  regions: ['GADM-BRA', 'GADM-DEU', 'GADM-POL'],
@@ -236,6 +236,38 @@ describe('routes', () => {
236
236
  const result = await supertest(app).get('/pg/filters').query({}).set('Accept', 'application/json');
237
237
  expect(result.body).to.deep.equal(expected);
238
238
  });
239
+
240
+ it('does not exclude nodes with missing practices key', async () => {
241
+ const expected = {
242
+ cycleCount: 1,
243
+ impactAssessmentCount: 0,
244
+ products: ['apples', 'cheese'],
245
+ periods: ['1980-1989'],
246
+ practices: [],
247
+ minAggregatedQualityScore: [2],
248
+ regions: ['GADM-BRA'],
249
+ defaultMethodClassifications: ['modelled']
250
+ };
251
+ const filter = ['regions=GADM-BRA', 'periods=1980-1989'].join(';');
252
+ const result = await supertest(app).get('/pg/filters').query({ filter }).set('Accept', 'application/json');
253
+ expect(result.body).to.deep.equal(expected);
254
+ });
255
+
256
+ it('returns empty arrays when no filters available', async () => {
257
+ const expected = {
258
+ cycleCount: 0,
259
+ impactAssessmentCount: 0,
260
+ products: [],
261
+ periods: [],
262
+ practices: [],
263
+ minAggregatedQualityScore: [],
264
+ regions: [],
265
+ defaultMethodClassifications: []
266
+ };
267
+ const filter = ['products=grapes'].join(';');
268
+ const result = await supertest(app).get('/pg/filters').query({ filter }).set('Accept', 'application/json');
269
+ expect(result.body).to.deep.equal(expected);
270
+ });
239
271
  });
240
272
  });
241
273
  });
@@ -9,31 +9,41 @@ const getFiltersQuery = (filter: IFilter): [string, any[]] => {
9
9
  SELECT
10
10
  COUNT(DISTINCT c) as "cycleCount",
11
11
  COUNT(DISTINCT ia) AS "impactAssessmentCount",
12
- ARRAY_AGG(DISTINCT product ORDER BY product) as products,
13
- ARRAY_AGG(DISTINCT c.generated_period ORDER BY c.generated_period) as periods,
14
- ARRAY_AGG(DISTINCT practice ORDER BY practice) as practices,
15
- ARRAY_AGG(
16
- DISTINCT (c.jsonld_pivoted->>'aggregatedQualityScore')::integer
17
- ORDER BY (c.jsonld_pivoted->>'aggregatedQualityScore')::integer
12
+ COALESCE(ARRAY_AGG(DISTINCT products ORDER BY products) FILTER (WHERE products IS NOT NULL), '{}') as products,
13
+ COALESCE(
14
+ ARRAY_AGG(DISTINCT c.generated_period ORDER BY c.generated_period) FILTER (WHERE c.generated_period IS NOT NULL),
15
+ '{}'
16
+ ) as periods,
17
+ COALESCE(ARRAY_AGG(DISTINCT practices ORDER BY practices) FILTER (WHERE practices IS NOT NULL), '{}') as practices,
18
+ COALESCE(
19
+ ARRAY_AGG(
20
+ DISTINCT (c.jsonld_pivoted->>'aggregatedQualityScore')::integer
21
+ ORDER BY (c.jsonld_pivoted->>'aggregatedQualityScore')::integer
22
+ ) FILTER (WHERE c.jsonld_pivoted->>'aggregatedQualityScore' is NOT NULL),
23
+ '{}'
18
24
  ) AS "minAggregatedQualityScore",
19
- ARRAY_AGG(
25
+ COALESCE(ARRAY_AGG(
20
26
  DISTINCT s.jsonld_pivoted['country']->>'@id'
21
27
  ORDER BY s.jsonld_pivoted['country']->>'@id'
22
- ) AS regions,
23
- ARRAY_AGG(
24
- DISTINCT c.jsonld_pivoted->>'defaultMethodClassification'
25
- ORDER BY c.jsonld_pivoted->>'defaultMethodClassification'
28
+ ) FILTER (WHERE s.jsonld_pivoted['country']->>'@id' IS NOT NULL), '{}') AS regions,
29
+ COALESCE(
30
+ ARRAY_AGG(
31
+ DISTINCT c.jsonld_pivoted->>'defaultMethodClassification'
32
+ ORDER BY c.jsonld_pivoted->>'defaultMethodClassification'
33
+ ) FILTER (WHERE c.jsonld_pivoted->>'defaultMethodClassification' IS NOT NULL),
34
+ '{}'
26
35
  ) AS "defaultMethodClassifications"
27
36
  FROM aggregated_cycles AS c
37
+ LEFT JOIN
38
+ jsonb_object_keys(c.jsonld_pivoted['products']) as products on TRUE
39
+ LEFT JOIN
40
+ jsonb_object_keys(c.jsonld_pivoted['practices']) as practices on TRUE
28
41
  LEFT JOIN aggregated_impact_assessments AS ia
29
42
  ON c.hestia_id = ia.cycle_hestia_id
30
43
  AND ${filterClauses.products.iaFilter}
31
44
  JOIN aggregated_sites AS s
32
45
  ON c.site_hestia_id = s.hestia_id
33
46
  AND ${filterClauses.regions}
34
- CROSS JOIN LATERAL
35
- jsonb_object_keys(c.jsonld_pivoted['products']) as products(product),
36
- jsonb_object_keys(c.jsonld_pivoted['practices']) as practices(practice)
37
47
  WHERE ${filterClauses.practices}
38
48
  AND ${filterClauses.defaultMethodClassifications}
39
49
  AND ${filterClauses.products.cycleFilter}
@@ -72,6 +72,6 @@ LIMIT 500
72
72
  export const applyFilter = async ({ filter }: { page?: number; filter: IFilter; query?: string }) => {
73
73
  // TODO: handle page and query later
74
74
  const [pgQuery, params] = getNodesQuery(filter);
75
- const rows = await doQuery<AggregatedNodeRow>(pgQuery, params);
76
- return rows;
75
+ const result = await doQuery<AggregatedNodeRow>(pgQuery, params);
76
+ return { count: result.rowCount, results: result.rows };
77
77
  };
@@ -8,7 +8,6 @@ import { sortBy as _sortBy } from 'lodash';
8
8
  import { loadFixture, initDb } from '../../../test/utils';
9
9
  import { run } from './handler';
10
10
  import { query } from '../../../database';
11
- import * as testSeeds from '../../../database/seed/test';
12
11
 
13
12
  const stubs: sinon.SinonStub[] = [];
14
13
 
@@ -32,15 +31,7 @@ const fixture = (fileName: string) => loadFixture(`update-aggregated-nodes/${fil
32
31
 
33
32
  const getJsonResults = (rows) =>
34
33
  rows
35
- .map((r) => r.jsonld_pivoted)
36
- .filter(
37
- (json) =>
38
- ![
39
- testSeeds.aggregatedCycles[0].jsonld_pivoted['@id'],
40
- testSeeds.aggregatedSites[0].jsonld_pivoted['@id'],
41
- testSeeds.aggregatedImpactAssessments[0].jsonld_pivoted['@id']
42
- ].includes(json['@id'])
43
- );
34
+ .map((r) => r.jsonld_pivoted);
44
35
 
45
36
  describe('update-aggregated-nodes > run', () => {
46
37
  beforeEach(async () => {
@@ -1,5 +1,4 @@
1
- import { info, debug, error, timeSpent, listFolder, getObject } from '@hestia-earth/pipeline-utils';
2
- import { aggregationFolder } from '@hestia-earth/api';
1
+ import { info, debug, timeSpent, listFolder, getObject, bucket } from '@hestia-earth/pipeline-utils';
3
2
  import csvtojson from 'csvtojson';
4
3
  import uniqBy from 'lodash.uniqby';
5
4
  import chunk from 'lodash.chunk';
@@ -7,8 +6,6 @@ import chunk from 'lodash.chunk';
7
6
  import { wrapHandler } from '../sentry';
8
7
  import { query } from '../../../database';
9
8
 
10
- const bucket = process.env.BUCKET_DATA;
11
-
12
9
  // If a cycle or IA is missing its referenced site/cycle, it is out of date: omit
13
10
  // JOIN ensures only records with a valid fkey will be selected for insert
14
11
  const upsertCyclesQuery = `
@@ -99,8 +96,8 @@ const getFormattedNodes = async (keys: string[]): Promise<{ [key in AggregationT
99
96
  };
100
97
 
101
98
  export const run = async () => {
102
- const folder = `backups/${aggregationFolder}`;
103
- const allKeys = (await listFolder(folder, bucket)).map(({ Key }) => Key);
99
+ const folder = 'backups/aggregation';
100
+ const allKeys = (await listFolder(folder)).map(({ Key }) => Key);
104
101
  debug('Found', allKeys.length, 'files in', folder, 'folder on bucket', bucket);
105
102
 
106
103
  // cycles must be processed/inserted before impact assessments because of fkey constraints
@@ -125,17 +122,8 @@ export const updateAggregatedNodes = async () => {
125
122
  const functionName = 'updateAggregatedNodes';
126
123
  info(functionName);
127
124
  const now = new Date();
128
-
129
- try {
130
- await run();
131
- }
132
- catch (err) {
133
- error(err);
134
- throw err;
135
- }
136
- finally {
137
- info(`Finished ${functionName} in ${timeSpent(now)}ms`);
138
- }
125
+ await run();
126
+ info(`Finished ${functionName} in ${timeSpent(now)}ms`);
139
127
  };
140
128
 
141
129
  export const updateAggregatedNodesHandler = wrapHandler(updateAggregatedNodes);
package/src/routes.ts CHANGED
@@ -1,9 +1,8 @@
1
1
  import { Express } from 'express';
2
+ import aggregatedNodes from './aggregated-nodes/routes';
2
3
 
3
4
  const routes = (app: Express) => {
4
- app.use('/', (req, res) => {
5
- res.send('Hello World!');
6
- });
5
+ app.use('/aggregated-nodes', aggregatedNodes());
7
6
  };
8
7
 
9
8
  export default routes;
@@ -102,6 +102,18 @@ export const aggregatedCycles = [
102
102
  practices: { baggingFruit: {} },
103
103
  products: { apples: {}, cheese: {} }
104
104
  }
105
+ },
106
+ {
107
+ jsonld_pivoted: {
108
+ '@id': '9',
109
+ '@type': 'Cycle',
110
+ site: { '@id': '3' },
111
+ defaultMethodClassification: 'modelled',
112
+ startDate: '1980',
113
+ endDate: '1989',
114
+ aggregatedQualityScore: 2,
115
+ products: { apples: {}, cheese: {} }
116
+ }
105
117
  }
106
118
  ];
107
119
 
package/test/prepare.ts CHANGED
@@ -1,15 +1,13 @@
1
- import prepare from 'mocha-prepare';
2
1
  import * as dotenv from 'dotenv';
3
2
  dotenv.config({ path: '.env.test' });
4
3
 
5
4
  let endPool: () => Promise<void>;
6
5
 
7
- prepare(
8
- (done) => {
6
+ export const mochaHooks = {
7
+ beforeAll() {
9
8
  ({ endPool } = require('../database/'));
10
- done();
11
9
  },
12
- (done) => {
10
+ afterAll(done) {
13
11
  endPool().finally(() => done());
14
12
  }
15
- );
13
+ };
package/test/utils.ts CHANGED
@@ -2,11 +2,10 @@ import { resolve, join } from 'path';
2
2
  import { readFileSync } from 'fs';
3
3
  import { query } from '../database';
4
4
  import { seed } from '../database/seed';
5
- import * as testSeeds from '../database/seed/test';
6
5
  import * as commonSeeds from '../database/seed/common';
7
6
  import { runMigrations } from '../database/migrations';
8
7
 
9
- export const fixtures = Object.freeze({ ...commonSeeds, ...testSeeds });
8
+ export const fixtures = Object.freeze({ ...commonSeeds });
10
9
 
11
10
  export const FIXTURES_DIR = resolve(join(__dirname, 'fixtures'));
12
11
 
@@ -29,5 +28,5 @@ export const resetDb = async () => {
29
28
  export const initDb = async () => {
30
29
  await resetDb();
31
30
  await runMigrations();
32
- await seed({ ...commonSeeds, ...testSeeds });
31
+ await seed({ ...commonSeeds });
33
32
  };
package/tsconfig.json CHANGED
@@ -1,4 +1,9 @@
1
1
  {
2
+ "ts-node": {
3
+ "files": true,
4
+ "typeCheck": false,
5
+ "transpileOnly": true
6
+ },
2
7
  "compilerOptions": {
3
8
  "target": "es2017",
4
9
  "module": "node16",
@@ -1,28 +0,0 @@
1
- export const aggregatedCycles = [
2
- {
3
- jsonld_pivoted: {
4
- '@id': 'test_cycle_id',
5
- '@type': 'Cycle',
6
- site: { '@id': 'test_site_id' }
7
- }
8
- }
9
- ];
10
-
11
- export const aggregatedImpactAssessments = [
12
- {
13
- jsonld_pivoted: {
14
- '@id': 'test_impact_assessment_id',
15
- '@type': 'ImpactAssessment',
16
- cycle: { '@id': 'test_cycle_id' }
17
- }
18
- }
19
- ];
20
-
21
- export const aggregatedSites = [
22
- {
23
- jsonld_pivoted: {
24
- '@id': 'test_site_id',
25
- '@type': 'Site'
26
- }
27
- }
28
- ];
@@ -1,33 +0,0 @@
1
- import { expect } from 'chai';
2
- import * as sinon from 'sinon';
3
- import 'mocha';
4
- import express from 'express';
5
- import supertest from 'supertest';
6
- import bodyParser from 'body-parser';
7
-
8
- import routes from './routes';
9
- import { errorHandler } from './errors';
10
-
11
- let stubs: sinon.SinonStub[] = [];
12
-
13
- const app = express();
14
- app.use(bodyParser.json());
15
- routes(app);
16
- app.use(errorHandler);
17
-
18
- describe('routes', () => {
19
- beforeEach(() => {
20
- stubs = [];
21
- });
22
-
23
- afterEach(() => {
24
- stubs.forEach((stub) => stub.restore());
25
- });
26
-
27
- describe('GET /', () => {
28
- it('returns hello world', async () => {
29
- const result = await supertest(app).get('/');
30
- expect(result.text).to.equal('Hello World!');
31
- });
32
- });
33
- });
File without changes