@hestia-earth/data-api 0.0.2-1 → 0.0.2-3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitlab-ci.yml +4 -1
- package/.mocharc.js +1 -1
- package/Dockerfile +1 -1
- package/database/seed/index.ts +6 -1
- package/package.json +26 -25
- package/package.serverless.json +21 -0
- package/serverless.yml +33 -8
- package/src/aggregated-nodes/routes/pg-get-filters.ts +11 -1
- package/src/aggregated-nodes/routes/pg-get.ts +12 -1
- package/src/aggregated-nodes/routes.spec.ts +41 -9
- package/src/aggregated-nodes/services/pg-get-filters.ts +24 -14
- package/src/aggregated-nodes/services/pg-get.ts +2 -2
- package/src/lambdas/update-aggregated-nodes/handler.spec.ts +1 -10
- package/src/lambdas/update-aggregated-nodes/handler.ts +5 -17
- package/src/routes.ts +2 -3
- package/test/fixtures/aggregated-nodes/get.ts +12 -0
- package/test/prepare.ts +4 -6
- package/test/utils.ts +2 -3
- package/tsconfig.json +5 -0
- package/database/seed/test/index.ts +0 -28
- package/src/routes.spec.ts +0 -33
- /package/{.nvrm → .nvmrc} +0 -0
package/.gitlab-ci.yml
CHANGED
|
@@ -9,7 +9,7 @@ default:
|
|
|
9
9
|
paths:
|
|
10
10
|
- node_modules/
|
|
11
11
|
before_script:
|
|
12
|
-
- npm
|
|
12
|
+
- npm ci --include=dev
|
|
13
13
|
|
|
14
14
|
variables:
|
|
15
15
|
CONTAINER_IMAGE: hestia-data-api
|
|
@@ -103,6 +103,9 @@ deploy-lambdas:
|
|
|
103
103
|
script:
|
|
104
104
|
- source envs/.${CI_COMMIT_REF_NAME}.env
|
|
105
105
|
- npm run build:lambdas
|
|
106
|
+
# copy serverless package to include correct dependencies
|
|
107
|
+
- rm -rf package.json node_modules
|
|
108
|
+
- mv package.serverless.json package.json && npm ci --include=dev
|
|
106
109
|
- npx serverless deploy --verbose
|
|
107
110
|
only:
|
|
108
111
|
- master
|
package/.mocharc.js
CHANGED
package/Dockerfile
CHANGED
package/database/seed/index.ts
CHANGED
|
@@ -5,7 +5,12 @@ import * as commonSeeds from './common';
|
|
|
5
5
|
|
|
6
6
|
const debug = Debug('@hestia/data-api:db:seed');
|
|
7
7
|
|
|
8
|
-
export const seed = async ({
|
|
8
|
+
export const seed = async ({
|
|
9
|
+
aggregatedCycles = [],
|
|
10
|
+
aggregatedImpactAssessments = [],
|
|
11
|
+
aggregatedSites = [],
|
|
12
|
+
settings = []
|
|
13
|
+
}) => {
|
|
9
14
|
for (const site of aggregatedSites) {
|
|
10
15
|
const { rows } = await query(
|
|
11
16
|
`INSERT INTO aggregated_sites (jsonld_pivoted) VALUES ($1)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hestia-earth/data-api",
|
|
3
|
-
"version": "0.0.2-
|
|
3
|
+
"version": "0.0.2-3",
|
|
4
4
|
"description": "Hestia Data API definitions",
|
|
5
5
|
"main": "dist/models.js",
|
|
6
6
|
"typings": "dist/models.d.ts",
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"build:lambdas": "rm -rf build-lambdas && tsc -p tsconfig.lambdas.json",
|
|
11
11
|
"start": "node index.js",
|
|
12
12
|
"dev": "ts-node-dev --respawn --rs dev.ts --files --ignore-watch node_modules",
|
|
13
|
-
"lint": "eslint .",
|
|
13
|
+
"lint": "tsc --noEmit && eslint .",
|
|
14
14
|
"lint:fix": "npm run lint -- --fix",
|
|
15
15
|
"test": "nyc mocha \"./**/*.spec.ts\"",
|
|
16
16
|
"migrate": "ts-node scripts/run-migrations.ts",
|
|
@@ -39,31 +39,14 @@
|
|
|
39
39
|
"@hestia-earth/schema": "^22.1.0"
|
|
40
40
|
},
|
|
41
41
|
"devDependencies": {
|
|
42
|
-
"@
|
|
42
|
+
"@commitlint/cli": "^17.6.5",
|
|
43
|
+
"@commitlint/config-conventional": "^17.6.5",
|
|
44
|
+
"@hestia-earth/eslint-config": "^0.0.5",
|
|
45
|
+
"@hestia-earth/pipeline-utils": "^0.12.1",
|
|
43
46
|
"@sentry/node": "^7.51.2",
|
|
44
47
|
"@sentry/serverless": "^7.55.2",
|
|
45
48
|
"@sentry/tracing": "^7.51.2",
|
|
46
49
|
"@slack/web-api": "^6.8.1",
|
|
47
|
-
"compression": "^1.7.4",
|
|
48
|
-
"cors": "^2.8.5",
|
|
49
|
-
"csvtojson": "^2.0.10",
|
|
50
|
-
"dotenv": "^16.0.3",
|
|
51
|
-
"exit-hook": "^3.2.0",
|
|
52
|
-
"express": "^4.17.3",
|
|
53
|
-
"express-async-errors": "^3.1.1",
|
|
54
|
-
"http2-express-bridge": "^1.0.7",
|
|
55
|
-
"lodash.chunk": "^4.2.0",
|
|
56
|
-
"lodash.uniqby": "^4.7.0",
|
|
57
|
-
"map-obj": "^5.0.2",
|
|
58
|
-
"morgan": "^1.10.0",
|
|
59
|
-
"pg": "^8.11.0",
|
|
60
|
-
"rxjs": "^7.8.1",
|
|
61
|
-
"swagger-jsdoc": "^6.2.8",
|
|
62
|
-
"swagger-ui-express": "^4.6.3",
|
|
63
|
-
"winston": "^3.8.2",
|
|
64
|
-
"@commitlint/cli": "^17.6.5",
|
|
65
|
-
"@commitlint/config-conventional": "^17.6.5",
|
|
66
|
-
"@hestia-earth/eslint-config": "^0.0.5",
|
|
67
50
|
"@types/chai": "^4.3.5",
|
|
68
51
|
"@types/chai-as-promised": "^7.1.5",
|
|
69
52
|
"@types/compression": "^1.7.2",
|
|
@@ -82,20 +65,38 @@
|
|
|
82
65
|
"@types/swagger-ui-express": "^4.1.3",
|
|
83
66
|
"chai": "^4.3.7",
|
|
84
67
|
"chai-as-promised": "^7.1.1",
|
|
68
|
+
"compression": "^1.7.4",
|
|
69
|
+
"cors": "^2.8.5",
|
|
70
|
+
"csvtojson": "^2.0.10",
|
|
71
|
+
"debug": "^4.3.4",
|
|
72
|
+
"dotenv": "^16.0.3",
|
|
85
73
|
"eslint": "^7.32.0",
|
|
74
|
+
"exit-hook": "^3.2.0",
|
|
75
|
+
"express": "^4.17.3",
|
|
76
|
+
"express-async-errors": "^3.1.1",
|
|
77
|
+
"http2-express-bridge": "^1.0.7",
|
|
86
78
|
"husky": "^4.3.8",
|
|
87
|
-
"
|
|
79
|
+
"lodash.chunk": "^4.2.0",
|
|
80
|
+
"lodash.uniqby": "^4.7.0",
|
|
81
|
+
"map-obj": "^5.0.2",
|
|
82
|
+
"mocha": "^10.2.0",
|
|
83
|
+
"morgan": "^1.10.0",
|
|
88
84
|
"nyc": "^15.1.0",
|
|
85
|
+
"pg": "^8.11.0",
|
|
89
86
|
"postgrator": "^7.1.1",
|
|
87
|
+
"rxjs": "^7.8.1",
|
|
90
88
|
"serverless": "^3.32.2",
|
|
91
89
|
"serverless-deployment-bucket": "^1.6.0",
|
|
92
90
|
"serverless-offline": "^12.0.4",
|
|
93
91
|
"sinon": "^15.1.0",
|
|
94
92
|
"standard-version": "^9.5.0",
|
|
95
93
|
"supertest": "^6.3.3",
|
|
94
|
+
"swagger-jsdoc": "^6.2.8",
|
|
95
|
+
"swagger-ui-express": "^4.6.3",
|
|
96
96
|
"ts-node": "^10.9.1",
|
|
97
97
|
"ts-node-dev": "^2.0.0",
|
|
98
|
-
"typescript": "^5.0.0"
|
|
98
|
+
"typescript": "^5.0.0",
|
|
99
|
+
"winston": "^3.8.2"
|
|
99
100
|
},
|
|
100
101
|
"husky": {
|
|
101
102
|
"hooks": {
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@hestia-earth/data-api-serverless",
|
|
3
|
+
"dependencies": {
|
|
4
|
+
"@hestia-earth/schema": "*",
|
|
5
|
+
"@hestia-earth/pipeline-utils": "*",
|
|
6
|
+
"@sentry/serverless": "*",
|
|
7
|
+
"csvtojson": "*",
|
|
8
|
+
"debug": "*",
|
|
9
|
+
"exit-hook": "*",
|
|
10
|
+
"lodash.chunk": "*",
|
|
11
|
+
"lodash.uniqby": "*",
|
|
12
|
+
"map-obj": "*",
|
|
13
|
+
"pg": "*",
|
|
14
|
+
"winston": "*"
|
|
15
|
+
},
|
|
16
|
+
"devDependencies": {
|
|
17
|
+
"serverless": "^3.32.2",
|
|
18
|
+
"serverless-deployment-bucket": "^1.6.0",
|
|
19
|
+
"serverless-offline": "^12.0.4"
|
|
20
|
+
}
|
|
21
|
+
}
|
package/serverless.yml
CHANGED
|
@@ -1,10 +1,38 @@
|
|
|
1
1
|
frameworkVersion: '3'
|
|
2
|
+
|
|
2
3
|
service: hestia-data-api
|
|
3
4
|
|
|
4
5
|
plugins:
|
|
5
6
|
- serverless-deployment-bucket
|
|
6
7
|
- serverless-offline
|
|
7
8
|
|
|
9
|
+
package:
|
|
10
|
+
exclude:
|
|
11
|
+
- node_modules/@types/**
|
|
12
|
+
# cannot be removed as used in migrations
|
|
13
|
+
# - node_modules/typescript/**
|
|
14
|
+
- node_modules/**/*.map
|
|
15
|
+
- node_modules/**/*.d.ts
|
|
16
|
+
- node_modules/**/coverage/**
|
|
17
|
+
- node_modules/**/tests/**
|
|
18
|
+
- node_modules/**/test/**
|
|
19
|
+
- .nyc_output/**
|
|
20
|
+
- coverage/**
|
|
21
|
+
- data/**
|
|
22
|
+
- docs/**
|
|
23
|
+
- envs/**
|
|
24
|
+
- samples/**
|
|
25
|
+
- scripts/**
|
|
26
|
+
- test/**
|
|
27
|
+
- tmp/**
|
|
28
|
+
- tsconfig*
|
|
29
|
+
- "*.yml"
|
|
30
|
+
- "*.md"
|
|
31
|
+
- "*.sh"
|
|
32
|
+
- ".dockerignore"
|
|
33
|
+
- "Dockerfile"
|
|
34
|
+
- "gulpfile.js"
|
|
35
|
+
|
|
8
36
|
custom:
|
|
9
37
|
serverless-offline:
|
|
10
38
|
httpPort: 3002
|
|
@@ -26,10 +54,7 @@ custom:
|
|
|
26
54
|
logLevel:
|
|
27
55
|
staging: DEBUG
|
|
28
56
|
prod: INFO
|
|
29
|
-
|
|
30
|
-
domain:
|
|
31
|
-
staging: https://www-staging.hestia.earth
|
|
32
|
-
prod: https://www.hestia.earth
|
|
57
|
+
bucket: hestia-data-${self:custom.stage}
|
|
33
58
|
|
|
34
59
|
provider:
|
|
35
60
|
name: aws
|
|
@@ -45,8 +70,8 @@ provider:
|
|
|
45
70
|
STAGE: ${self:custom.stage}
|
|
46
71
|
DEBUG: ${self:custom.debugging.${self:custom.stage}}
|
|
47
72
|
LOG_LEVEL: ${self:custom.logLevel.${self:custom.stage}}
|
|
48
|
-
SENTRY_DSN: https://00ed710e2b664894a3ff3124bfff24cf@o441427.ingest.
|
|
49
|
-
|
|
73
|
+
SENTRY_DSN: https://00ed710e2b664894a3ff3124bfff24cf@o441427.ingest.sentry.io/4505186792767488
|
|
74
|
+
BUCKET: ${self:custom.bucket}
|
|
50
75
|
PGHOST: ${env:PGHOST}
|
|
51
76
|
PGDATABASE: ${env:PGDATABASE}
|
|
52
77
|
PGPORT: ${env:PGPORT}
|
|
@@ -60,8 +85,8 @@ provider:
|
|
|
60
85
|
- s3:ListObjects
|
|
61
86
|
- s3:ListBucket
|
|
62
87
|
Resource:
|
|
63
|
-
- "arn:aws:s3:::${self:custom.
|
|
64
|
-
- "arn:aws:s3:::${self:custom.
|
|
88
|
+
- "arn:aws:s3:::${self:custom.bucket}"
|
|
89
|
+
- "arn:aws:s3:::${self:custom.bucket}/*"
|
|
65
90
|
vpc:
|
|
66
91
|
securityGroupIds:
|
|
67
92
|
- ${self:custom.securityGroup.${self:custom.stage}}
|
|
@@ -14,7 +14,17 @@ import { applyFilters } from '../services/pg-get-filters';
|
|
|
14
14
|
* name: filter
|
|
15
15
|
* schema:
|
|
16
16
|
* type: string
|
|
17
|
-
*
|
|
17
|
+
* examples:
|
|
18
|
+
* example1:
|
|
19
|
+
* summary: Barley in Australia and India between 2010 and 2019.
|
|
20
|
+
* value: "regions=GADM-AUS|GADM-IND;\
|
|
21
|
+
* products=barleyGrainWhole|barleyStraw;\
|
|
22
|
+
* periods=2010-2019"
|
|
23
|
+
* example2:
|
|
24
|
+
* summary: Wheat and Maize in France, United Kingdom and Worldwide between 2000 and 2019.
|
|
25
|
+
* value: "regions=region-world|GADM-GBR|GADM-FRA;\
|
|
26
|
+
* products=wheatGrain|maizeGrain;\
|
|
27
|
+
* periods=2000-2009|2010-2019"
|
|
18
28
|
* description: Set of filters
|
|
19
29
|
* - in: query
|
|
20
30
|
* name: query
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
/* eslint-disable max-len */
|
|
1
2
|
import { Request } from 'express';
|
|
2
3
|
|
|
3
4
|
import { applyFilter } from '../services/pg-get';
|
|
@@ -14,7 +15,17 @@ import { applyFilter } from '../services/pg-get';
|
|
|
14
15
|
* name: filter
|
|
15
16
|
* schema:
|
|
16
17
|
* type: string
|
|
17
|
-
*
|
|
18
|
+
* examples:
|
|
19
|
+
* example1:
|
|
20
|
+
* summary: Barley in Australia and India between 2010 and 2019.
|
|
21
|
+
* value: "regions=GADM-AUS|GADM-IND;\
|
|
22
|
+
* products=barleyGrainWhole|barleyStraw;\
|
|
23
|
+
* periods=2010-2019"
|
|
24
|
+
* example2:
|
|
25
|
+
* summary: Wheat and Maize in France, United Kingdom and Worldwide between 2000 and 2019.
|
|
26
|
+
* value: "regions=region-world|GADM-GBR|GADM-FRA;\
|
|
27
|
+
* products=wheatGrain|maizeGrain;\
|
|
28
|
+
* periods=2000-2009|2010-2019"
|
|
18
29
|
* description: Set of filters
|
|
19
30
|
* - in: query
|
|
20
31
|
* name: page
|
|
@@ -77,7 +77,7 @@ describe('routes', () => {
|
|
|
77
77
|
|
|
78
78
|
const filter = 'regions=GADM-POL|GADM-DEU';
|
|
79
79
|
const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
|
|
80
|
-
expect(result.body.
|
|
80
|
+
expect(result.body.results).to.deep.equal(expected);
|
|
81
81
|
});
|
|
82
82
|
|
|
83
83
|
it('can filter by practices', async () => {
|
|
@@ -95,7 +95,7 @@ describe('routes', () => {
|
|
|
95
95
|
];
|
|
96
96
|
const filter = 'practices=earthingUpByHand|deepRipping';
|
|
97
97
|
const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
|
|
98
|
-
expect(result.body.
|
|
98
|
+
expect(result.body.results).to.deep.equal(expected);
|
|
99
99
|
});
|
|
100
100
|
|
|
101
101
|
it('can filter by minimum aggregatedQualityScore', async () => {
|
|
@@ -108,7 +108,7 @@ describe('routes', () => {
|
|
|
108
108
|
];
|
|
109
109
|
const filter = 'minAggregatedQualityScore=3';
|
|
110
110
|
const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
|
|
111
|
-
expect(result.body.
|
|
111
|
+
expect(result.body.results).to.deep.equal(expected);
|
|
112
112
|
});
|
|
113
113
|
|
|
114
114
|
it('can filter by defaultMethodClassifications', async () => {
|
|
@@ -126,7 +126,7 @@ describe('routes', () => {
|
|
|
126
126
|
];
|
|
127
127
|
const filter = 'defaultMethodClassifications=physical measurement|expert opinion';
|
|
128
128
|
const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
|
|
129
|
-
expect(result.body.
|
|
129
|
+
expect(result.body.results).to.deep.equal(expected);
|
|
130
130
|
});
|
|
131
131
|
|
|
132
132
|
it('can filter by periods', async () => {
|
|
@@ -144,7 +144,7 @@ describe('routes', () => {
|
|
|
144
144
|
];
|
|
145
145
|
const filter = 'periods=2010-2019|2000-2009';
|
|
146
146
|
const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
|
|
147
|
-
expect(result.body.
|
|
147
|
+
expect(result.body.results).to.deep.equal(expected);
|
|
148
148
|
});
|
|
149
149
|
|
|
150
150
|
it('can filter by products', async () => {
|
|
@@ -163,7 +163,7 @@ describe('routes', () => {
|
|
|
163
163
|
];
|
|
164
164
|
const filter = 'products=wheatGrain|fishFingerlings';
|
|
165
165
|
const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
|
|
166
|
-
expect(result.body.
|
|
166
|
+
expect(result.body.results).to.deep.equal(expected);
|
|
167
167
|
});
|
|
168
168
|
|
|
169
169
|
it('can combine multiple filters', async () => {
|
|
@@ -184,7 +184,7 @@ describe('routes', () => {
|
|
|
184
184
|
'practices=earthingUpByHand|deepRipping|baggingFruit'
|
|
185
185
|
].join(';');
|
|
186
186
|
const result = await supertest(app).get('/pg').query({ filter }).set('Accept', 'application/json');
|
|
187
|
-
expect(result.body.
|
|
187
|
+
expect(result.body.results).to.deep.equal(expected);
|
|
188
188
|
});
|
|
189
189
|
});
|
|
190
190
|
|
|
@@ -224,10 +224,10 @@ describe('routes', () => {
|
|
|
224
224
|
|
|
225
225
|
it('returns all filters when there is nothing currently selected', async () => {
|
|
226
226
|
const expected = {
|
|
227
|
-
cycleCount:
|
|
227
|
+
cycleCount: 9,
|
|
228
228
|
impactAssessmentCount: 4,
|
|
229
229
|
products: ['apples', 'barley', 'cheese', 'fishFingerlings', 'wheatGrain'],
|
|
230
|
-
periods: ['1990-1999', '2000-2009', '2010-2019', '2020-2029'],
|
|
230
|
+
periods: ['1980-1989', '1990-1999', '2000-2009', '2010-2019', '2020-2029'],
|
|
231
231
|
practices: ['baggingFruit', 'deepRipping', 'earthingUpByHand'],
|
|
232
232
|
minAggregatedQualityScore: [1, 2, 3],
|
|
233
233
|
regions: ['GADM-BRA', 'GADM-DEU', 'GADM-POL'],
|
|
@@ -236,6 +236,38 @@ describe('routes', () => {
|
|
|
236
236
|
const result = await supertest(app).get('/pg/filters').query({}).set('Accept', 'application/json');
|
|
237
237
|
expect(result.body).to.deep.equal(expected);
|
|
238
238
|
});
|
|
239
|
+
|
|
240
|
+
it('does not exclude nodes with missing practices key', async () => {
|
|
241
|
+
const expected = {
|
|
242
|
+
cycleCount: 1,
|
|
243
|
+
impactAssessmentCount: 0,
|
|
244
|
+
products: ['apples', 'cheese'],
|
|
245
|
+
periods: ['1980-1989'],
|
|
246
|
+
practices: [],
|
|
247
|
+
minAggregatedQualityScore: [2],
|
|
248
|
+
regions: ['GADM-BRA'],
|
|
249
|
+
defaultMethodClassifications: ['modelled']
|
|
250
|
+
};
|
|
251
|
+
const filter = ['regions=GADM-BRA', 'periods=1980-1989'].join(';');
|
|
252
|
+
const result = await supertest(app).get('/pg/filters').query({ filter }).set('Accept', 'application/json');
|
|
253
|
+
expect(result.body).to.deep.equal(expected);
|
|
254
|
+
});
|
|
255
|
+
|
|
256
|
+
it('returns empty arrays when no filters available', async () => {
|
|
257
|
+
const expected = {
|
|
258
|
+
cycleCount: 0,
|
|
259
|
+
impactAssessmentCount: 0,
|
|
260
|
+
products: [],
|
|
261
|
+
periods: [],
|
|
262
|
+
practices: [],
|
|
263
|
+
minAggregatedQualityScore: [],
|
|
264
|
+
regions: [],
|
|
265
|
+
defaultMethodClassifications: []
|
|
266
|
+
};
|
|
267
|
+
const filter = ['products=grapes'].join(';');
|
|
268
|
+
const result = await supertest(app).get('/pg/filters').query({ filter }).set('Accept', 'application/json');
|
|
269
|
+
expect(result.body).to.deep.equal(expected);
|
|
270
|
+
});
|
|
239
271
|
});
|
|
240
272
|
});
|
|
241
273
|
});
|
|
@@ -9,31 +9,41 @@ const getFiltersQuery = (filter: IFilter): [string, any[]] => {
|
|
|
9
9
|
SELECT
|
|
10
10
|
COUNT(DISTINCT c) as "cycleCount",
|
|
11
11
|
COUNT(DISTINCT ia) AS "impactAssessmentCount",
|
|
12
|
-
ARRAY_AGG(DISTINCT
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
12
|
+
COALESCE(ARRAY_AGG(DISTINCT products ORDER BY products) FILTER (WHERE products IS NOT NULL), '{}') as products,
|
|
13
|
+
COALESCE(
|
|
14
|
+
ARRAY_AGG(DISTINCT c.generated_period ORDER BY c.generated_period) FILTER (WHERE c.generated_period IS NOT NULL),
|
|
15
|
+
'{}'
|
|
16
|
+
) as periods,
|
|
17
|
+
COALESCE(ARRAY_AGG(DISTINCT practices ORDER BY practices) FILTER (WHERE practices IS NOT NULL), '{}') as practices,
|
|
18
|
+
COALESCE(
|
|
19
|
+
ARRAY_AGG(
|
|
20
|
+
DISTINCT (c.jsonld_pivoted->>'aggregatedQualityScore')::integer
|
|
21
|
+
ORDER BY (c.jsonld_pivoted->>'aggregatedQualityScore')::integer
|
|
22
|
+
) FILTER (WHERE c.jsonld_pivoted->>'aggregatedQualityScore' is NOT NULL),
|
|
23
|
+
'{}'
|
|
18
24
|
) AS "minAggregatedQualityScore",
|
|
19
|
-
ARRAY_AGG(
|
|
25
|
+
COALESCE(ARRAY_AGG(
|
|
20
26
|
DISTINCT s.jsonld_pivoted['country']->>'@id'
|
|
21
27
|
ORDER BY s.jsonld_pivoted['country']->>'@id'
|
|
22
|
-
) AS regions,
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
28
|
+
) FILTER (WHERE s.jsonld_pivoted['country']->>'@id' IS NOT NULL), '{}') AS regions,
|
|
29
|
+
COALESCE(
|
|
30
|
+
ARRAY_AGG(
|
|
31
|
+
DISTINCT c.jsonld_pivoted->>'defaultMethodClassification'
|
|
32
|
+
ORDER BY c.jsonld_pivoted->>'defaultMethodClassification'
|
|
33
|
+
) FILTER (WHERE c.jsonld_pivoted->>'defaultMethodClassification' IS NOT NULL),
|
|
34
|
+
'{}'
|
|
26
35
|
) AS "defaultMethodClassifications"
|
|
27
36
|
FROM aggregated_cycles AS c
|
|
37
|
+
LEFT JOIN
|
|
38
|
+
jsonb_object_keys(c.jsonld_pivoted['products']) as products on TRUE
|
|
39
|
+
LEFT JOIN
|
|
40
|
+
jsonb_object_keys(c.jsonld_pivoted['practices']) as practices on TRUE
|
|
28
41
|
LEFT JOIN aggregated_impact_assessments AS ia
|
|
29
42
|
ON c.hestia_id = ia.cycle_hestia_id
|
|
30
43
|
AND ${filterClauses.products.iaFilter}
|
|
31
44
|
JOIN aggregated_sites AS s
|
|
32
45
|
ON c.site_hestia_id = s.hestia_id
|
|
33
46
|
AND ${filterClauses.regions}
|
|
34
|
-
CROSS JOIN LATERAL
|
|
35
|
-
jsonb_object_keys(c.jsonld_pivoted['products']) as products(product),
|
|
36
|
-
jsonb_object_keys(c.jsonld_pivoted['practices']) as practices(practice)
|
|
37
47
|
WHERE ${filterClauses.practices}
|
|
38
48
|
AND ${filterClauses.defaultMethodClassifications}
|
|
39
49
|
AND ${filterClauses.products.cycleFilter}
|
|
@@ -72,6 +72,6 @@ LIMIT 500
|
|
|
72
72
|
export const applyFilter = async ({ filter }: { page?: number; filter: IFilter; query?: string }) => {
|
|
73
73
|
// TODO: handle page and query later
|
|
74
74
|
const [pgQuery, params] = getNodesQuery(filter);
|
|
75
|
-
const
|
|
76
|
-
return rows;
|
|
75
|
+
const result = await doQuery<AggregatedNodeRow>(pgQuery, params);
|
|
76
|
+
return { count: result.rowCount, results: result.rows };
|
|
77
77
|
};
|
|
@@ -8,7 +8,6 @@ import { sortBy as _sortBy } from 'lodash';
|
|
|
8
8
|
import { loadFixture, initDb } from '../../../test/utils';
|
|
9
9
|
import { run } from './handler';
|
|
10
10
|
import { query } from '../../../database';
|
|
11
|
-
import * as testSeeds from '../../../database/seed/test';
|
|
12
11
|
|
|
13
12
|
const stubs: sinon.SinonStub[] = [];
|
|
14
13
|
|
|
@@ -32,15 +31,7 @@ const fixture = (fileName: string) => loadFixture(`update-aggregated-nodes/${fil
|
|
|
32
31
|
|
|
33
32
|
const getJsonResults = (rows) =>
|
|
34
33
|
rows
|
|
35
|
-
.map((r) => r.jsonld_pivoted)
|
|
36
|
-
.filter(
|
|
37
|
-
(json) =>
|
|
38
|
-
![
|
|
39
|
-
testSeeds.aggregatedCycles[0].jsonld_pivoted['@id'],
|
|
40
|
-
testSeeds.aggregatedSites[0].jsonld_pivoted['@id'],
|
|
41
|
-
testSeeds.aggregatedImpactAssessments[0].jsonld_pivoted['@id']
|
|
42
|
-
].includes(json['@id'])
|
|
43
|
-
);
|
|
34
|
+
.map((r) => r.jsonld_pivoted);
|
|
44
35
|
|
|
45
36
|
describe('update-aggregated-nodes > run', () => {
|
|
46
37
|
beforeEach(async () => {
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import { info, debug,
|
|
2
|
-
import { aggregationFolder } from '@hestia-earth/api';
|
|
1
|
+
import { info, debug, timeSpent, listFolder, getObject, bucket } from '@hestia-earth/pipeline-utils';
|
|
3
2
|
import csvtojson from 'csvtojson';
|
|
4
3
|
import uniqBy from 'lodash.uniqby';
|
|
5
4
|
import chunk from 'lodash.chunk';
|
|
@@ -7,8 +6,6 @@ import chunk from 'lodash.chunk';
|
|
|
7
6
|
import { wrapHandler } from '../sentry';
|
|
8
7
|
import { query } from '../../../database';
|
|
9
8
|
|
|
10
|
-
const bucket = process.env.BUCKET_DATA;
|
|
11
|
-
|
|
12
9
|
// If a cycle or IA is missing its referenced site/cycle, it is out of date: omit
|
|
13
10
|
// JOIN ensures only records with a valid fkey will be selected for insert
|
|
14
11
|
const upsertCyclesQuery = `
|
|
@@ -99,8 +96,8 @@ const getFormattedNodes = async (keys: string[]): Promise<{ [key in AggregationT
|
|
|
99
96
|
};
|
|
100
97
|
|
|
101
98
|
export const run = async () => {
|
|
102
|
-
const folder =
|
|
103
|
-
const allKeys = (await listFolder(folder
|
|
99
|
+
const folder = 'backups/aggregation';
|
|
100
|
+
const allKeys = (await listFolder(folder)).map(({ Key }) => Key);
|
|
104
101
|
debug('Found', allKeys.length, 'files in', folder, 'folder on bucket', bucket);
|
|
105
102
|
|
|
106
103
|
// cycles must be processed/inserted before impact assessments because of fkey constraints
|
|
@@ -125,17 +122,8 @@ export const updateAggregatedNodes = async () => {
|
|
|
125
122
|
const functionName = 'updateAggregatedNodes';
|
|
126
123
|
info(functionName);
|
|
127
124
|
const now = new Date();
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
await run();
|
|
131
|
-
}
|
|
132
|
-
catch (err) {
|
|
133
|
-
error(err);
|
|
134
|
-
throw err;
|
|
135
|
-
}
|
|
136
|
-
finally {
|
|
137
|
-
info(`Finished ${functionName} in ${timeSpent(now)}ms`);
|
|
138
|
-
}
|
|
125
|
+
await run();
|
|
126
|
+
info(`Finished ${functionName} in ${timeSpent(now)}ms`);
|
|
139
127
|
};
|
|
140
128
|
|
|
141
129
|
export const updateAggregatedNodesHandler = wrapHandler(updateAggregatedNodes);
|
package/src/routes.ts
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import { Express } from 'express';
|
|
2
|
+
import aggregatedNodes from './aggregated-nodes/routes';
|
|
2
3
|
|
|
3
4
|
const routes = (app: Express) => {
|
|
4
|
-
app.use('/', (
|
|
5
|
-
res.send('Hello World!');
|
|
6
|
-
});
|
|
5
|
+
app.use('/aggregated-nodes', aggregatedNodes());
|
|
7
6
|
};
|
|
8
7
|
|
|
9
8
|
export default routes;
|
|
@@ -102,6 +102,18 @@ export const aggregatedCycles = [
|
|
|
102
102
|
practices: { baggingFruit: {} },
|
|
103
103
|
products: { apples: {}, cheese: {} }
|
|
104
104
|
}
|
|
105
|
+
},
|
|
106
|
+
{
|
|
107
|
+
jsonld_pivoted: {
|
|
108
|
+
'@id': '9',
|
|
109
|
+
'@type': 'Cycle',
|
|
110
|
+
site: { '@id': '3' },
|
|
111
|
+
defaultMethodClassification: 'modelled',
|
|
112
|
+
startDate: '1980',
|
|
113
|
+
endDate: '1989',
|
|
114
|
+
aggregatedQualityScore: 2,
|
|
115
|
+
products: { apples: {}, cheese: {} }
|
|
116
|
+
}
|
|
105
117
|
}
|
|
106
118
|
];
|
|
107
119
|
|
package/test/prepare.ts
CHANGED
|
@@ -1,15 +1,13 @@
|
|
|
1
|
-
import prepare from 'mocha-prepare';
|
|
2
1
|
import * as dotenv from 'dotenv';
|
|
3
2
|
dotenv.config({ path: '.env.test' });
|
|
4
3
|
|
|
5
4
|
let endPool: () => Promise<void>;
|
|
6
5
|
|
|
7
|
-
|
|
8
|
-
(
|
|
6
|
+
export const mochaHooks = {
|
|
7
|
+
beforeAll() {
|
|
9
8
|
({ endPool } = require('../database/'));
|
|
10
|
-
done();
|
|
11
9
|
},
|
|
12
|
-
(done)
|
|
10
|
+
afterAll(done) {
|
|
13
11
|
endPool().finally(() => done());
|
|
14
12
|
}
|
|
15
|
-
|
|
13
|
+
};
|
package/test/utils.ts
CHANGED
|
@@ -2,11 +2,10 @@ import { resolve, join } from 'path';
|
|
|
2
2
|
import { readFileSync } from 'fs';
|
|
3
3
|
import { query } from '../database';
|
|
4
4
|
import { seed } from '../database/seed';
|
|
5
|
-
import * as testSeeds from '../database/seed/test';
|
|
6
5
|
import * as commonSeeds from '../database/seed/common';
|
|
7
6
|
import { runMigrations } from '../database/migrations';
|
|
8
7
|
|
|
9
|
-
export const fixtures = Object.freeze({ ...commonSeeds
|
|
8
|
+
export const fixtures = Object.freeze({ ...commonSeeds });
|
|
10
9
|
|
|
11
10
|
export const FIXTURES_DIR = resolve(join(__dirname, 'fixtures'));
|
|
12
11
|
|
|
@@ -29,5 +28,5 @@ export const resetDb = async () => {
|
|
|
29
28
|
export const initDb = async () => {
|
|
30
29
|
await resetDb();
|
|
31
30
|
await runMigrations();
|
|
32
|
-
await seed({ ...commonSeeds
|
|
31
|
+
await seed({ ...commonSeeds });
|
|
33
32
|
};
|
package/tsconfig.json
CHANGED
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
export const aggregatedCycles = [
|
|
2
|
-
{
|
|
3
|
-
jsonld_pivoted: {
|
|
4
|
-
'@id': 'test_cycle_id',
|
|
5
|
-
'@type': 'Cycle',
|
|
6
|
-
site: { '@id': 'test_site_id' }
|
|
7
|
-
}
|
|
8
|
-
}
|
|
9
|
-
];
|
|
10
|
-
|
|
11
|
-
export const aggregatedImpactAssessments = [
|
|
12
|
-
{
|
|
13
|
-
jsonld_pivoted: {
|
|
14
|
-
'@id': 'test_impact_assessment_id',
|
|
15
|
-
'@type': 'ImpactAssessment',
|
|
16
|
-
cycle: { '@id': 'test_cycle_id' }
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
];
|
|
20
|
-
|
|
21
|
-
export const aggregatedSites = [
|
|
22
|
-
{
|
|
23
|
-
jsonld_pivoted: {
|
|
24
|
-
'@id': 'test_site_id',
|
|
25
|
-
'@type': 'Site'
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
];
|
package/src/routes.spec.ts
DELETED
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
import { expect } from 'chai';
|
|
2
|
-
import * as sinon from 'sinon';
|
|
3
|
-
import 'mocha';
|
|
4
|
-
import express from 'express';
|
|
5
|
-
import supertest from 'supertest';
|
|
6
|
-
import bodyParser from 'body-parser';
|
|
7
|
-
|
|
8
|
-
import routes from './routes';
|
|
9
|
-
import { errorHandler } from './errors';
|
|
10
|
-
|
|
11
|
-
let stubs: sinon.SinonStub[] = [];
|
|
12
|
-
|
|
13
|
-
const app = express();
|
|
14
|
-
app.use(bodyParser.json());
|
|
15
|
-
routes(app);
|
|
16
|
-
app.use(errorHandler);
|
|
17
|
-
|
|
18
|
-
describe('routes', () => {
|
|
19
|
-
beforeEach(() => {
|
|
20
|
-
stubs = [];
|
|
21
|
-
});
|
|
22
|
-
|
|
23
|
-
afterEach(() => {
|
|
24
|
-
stubs.forEach((stub) => stub.restore());
|
|
25
|
-
});
|
|
26
|
-
|
|
27
|
-
describe('GET /', () => {
|
|
28
|
-
it('returns hello world', async () => {
|
|
29
|
-
const result = await supertest(app).get('/');
|
|
30
|
-
expect(result.text).to.equal('Hello World!');
|
|
31
|
-
});
|
|
32
|
-
});
|
|
33
|
-
});
|
/package/{.nvrm → .nvmrc}
RENAMED
|
File without changes
|