@bcgov/plugin-catalog-backend-module-bc-data-catalogue 0.1.0 → 0.2.5-feature.aps-4105-connect-pipelines.cbf325cb
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -27
- package/dist/BcDataCatalogueApisProvider.cjs.js +16 -454
- package/dist/BcDataCatalogueApisProvider.cjs.js.map +1 -1
- package/dist/BcDataCatalogueClient.cjs.js +196 -0
- package/dist/BcDataCatalogueClient.cjs.js.map +1 -0
- package/dist/BcDataCatalogueDatasetProcessor.cjs.js +35 -0
- package/dist/BcDataCatalogueDatasetProcessor.cjs.js.map +1 -0
- package/dist/BcDataCatalogueEntityFactory.cjs.js +510 -0
- package/dist/BcDataCatalogueEntityFactory.cjs.js.map +1 -0
- package/dist/BcDataCatalogueNaming.cjs.js +59 -0
- package/dist/BcDataCatalogueNaming.cjs.js.map +1 -0
- package/dist/module.cjs.js +8 -4
- package/dist/module.cjs.js.map +1 -1
- package/package.json +11 -4
- package/dist/BcDataCatalogueModel.cjs.js +0 -146
- package/dist/BcDataCatalogueModel.cjs.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,42 +1,43 @@
|
|
|
1
|
-
# plugin-catalog-backend-module-bc-data-catalogue
|
|
1
|
+
# @bcgov/plugin-catalog-backend-module-bc-data-catalogue
|
|
2
2
|
|
|
3
|
-
This
|
|
3
|
+
This package is part of the BC Data Catalogue Backstage integration.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
👉 **Refer to the main project README at the root of this repository for installation and configuration instructions.**
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
---
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
## Purpose
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
The `@bcgov/plugin-catalog-backend-module-bc-data-catalogue` package provides the backend integration with the BC Data Catalogue.
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
It is responsible for:
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
- fetching datasets from the BC Data Catalogue
|
|
16
|
+
- transforming them into Backstage entities
|
|
17
|
+
- registering entities such as:
|
|
18
|
+
- `Dataset`
|
|
19
|
+
- `API`
|
|
20
|
+
- `System`
|
|
21
|
+
- `Group`
|
|
22
|
+
- `User`
|
|
16
23
|
|
|
17
|
-
|
|
24
|
+
---
|
|
18
25
|
|
|
19
|
-
|
|
20
|
-
import { createBackend } from '@backstage/backend-defaults';
|
|
26
|
+
## Usage
|
|
21
27
|
|
|
22
|
-
|
|
23
|
-
// ... other backend.add() calls ...
|
|
28
|
+
This package is intended to be registered as a backend module in Backstage:
|
|
24
29
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
backend.start();
|
|
30
|
+
```ts
|
|
31
|
+
backend.add(import('@bcgov/plugin-catalog-backend-module-bc-data-catalogue'));
|
|
28
32
|
```
|
|
29
33
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
Add the following configuration to your `app-config.yaml` to allow the backend to access the BC Data Catalogue API:
|
|
34
|
+
---
|
|
33
35
|
|
|
34
|
-
|
|
35
|
-
backend:
|
|
36
|
-
reading:
|
|
37
|
-
allow:
|
|
38
|
-
- host: catalogue.data.gov.bc.ca
|
|
39
|
-
scheme: https
|
|
40
|
-
```
|
|
36
|
+
## Notes
|
|
41
37
|
|
|
42
|
-
|
|
38
|
+
- This package is **not intended to be used in isolation**
|
|
39
|
+
- It depends on:
|
|
40
|
+
- `@bcgov/plugin-catalog-common-bc-data-catalogue`
|
|
41
|
+
- It is designed to be used together with:
|
|
42
|
+
- `@bcgov/catalog-dataset` (frontend UI)
|
|
43
|
+
- All setup, configuration, and environment details are documented in the root project README
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
var
|
|
3
|
+
var BcDataCatalogueClient = require('./BcDataCatalogueClient.cjs.js');
|
|
4
|
+
var BcDataCatalogueEntityFactory = require('./BcDataCatalogueEntityFactory.cjs.js');
|
|
4
5
|
|
|
5
6
|
class BcDataCatalogueApisProvider {
|
|
6
7
|
logger;
|
|
@@ -9,7 +10,6 @@ class BcDataCatalogueApisProvider {
|
|
|
9
10
|
connection;
|
|
10
11
|
taskRunner;
|
|
11
12
|
allowedHosts;
|
|
12
|
-
/** [1] */
|
|
13
13
|
constructor(env, reader, taskRunner, allowedHosts, logger) {
|
|
14
14
|
this.env = env;
|
|
15
15
|
this.reader = reader;
|
|
@@ -17,13 +17,11 @@ class BcDataCatalogueApisProvider {
|
|
|
17
17
|
this.allowedHosts = allowedHosts;
|
|
18
18
|
this.logger = logger;
|
|
19
19
|
}
|
|
20
|
-
/** [2] */
|
|
21
20
|
getProviderName() {
|
|
22
21
|
return `bc-data-catalogue-${this.env}`;
|
|
23
22
|
}
|
|
24
|
-
/** [3] */
|
|
25
23
|
async connect(connection) {
|
|
26
|
-
this.logger.info("<connect");
|
|
24
|
+
this.logger.info("[BCDC Entity Provider] <connect");
|
|
27
25
|
this.connection = connection;
|
|
28
26
|
await this.taskRunner.run({
|
|
29
27
|
id: this.getProviderName(),
|
|
@@ -31,468 +29,32 @@ class BcDataCatalogueApisProvider {
|
|
|
31
29
|
await this.run();
|
|
32
30
|
}
|
|
33
31
|
});
|
|
34
|
-
this.logger.info(">connect");
|
|
32
|
+
this.logger.info("[BCDC Entity Provider] >connect");
|
|
35
33
|
}
|
|
36
|
-
/** [4] */
|
|
37
34
|
async run() {
|
|
38
|
-
this.logger.info("<run");
|
|
35
|
+
this.logger.info("[BCDC Entity Provider] <run");
|
|
39
36
|
if (!this.connection) {
|
|
40
37
|
throw new Error("Not initialized");
|
|
41
38
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
let allPackages = [];
|
|
46
|
-
do {
|
|
47
|
-
const start = 1e3 * page;
|
|
48
|
-
const url = `https://catalogue.data.gov.bc.ca/api/3/action/package_search?start=${start}&rows=1000`;
|
|
49
|
-
const response = await this.reader.readUrl(url);
|
|
50
|
-
const data = JSON.parse((await response.buffer()).toString());
|
|
51
|
-
if (!data?.success || !Array.isArray(data.result?.results)) {
|
|
52
|
-
this.logger.warn("Invalid or unsuccessful response", { success: data?.success });
|
|
53
|
-
++retries;
|
|
54
|
-
continue;
|
|
55
|
-
}
|
|
56
|
-
retries = 0;
|
|
57
|
-
const results = data.result.results;
|
|
58
|
-
if (results.length > 0) {
|
|
59
|
-
const packages = this.getBcDataCataloguePackages(results);
|
|
60
|
-
allPackages.push(...packages);
|
|
61
|
-
++page;
|
|
62
|
-
} else {
|
|
63
|
-
page = -1;
|
|
64
|
-
}
|
|
65
|
-
} while (page > 0 && retries < 3);
|
|
66
|
-
this.logger.info(`Total packages ${allPackages.length}`);
|
|
67
|
-
const allGroups = /* @__PURE__ */ new Map();
|
|
68
|
-
const bcGovGroupId = this.getGroupId("gov.bc.ca");
|
|
69
|
-
allGroups.set(bcGovGroupId, this.createGroupEntity("gov.bc.ca", "Governent of British Columbia", void 0));
|
|
70
|
-
const allOrganizations = /* @__PURE__ */ new Map();
|
|
71
|
-
allPackages.forEach((pkg) => {
|
|
72
|
-
const organization = pkg.organization;
|
|
73
|
-
allOrganizations.set(organization.id, organization);
|
|
74
|
-
});
|
|
75
|
-
const allSystems = /* @__PURE__ */ new Map();
|
|
76
|
-
this.logger.info(`Organizations ${allOrganizations.size}`);
|
|
77
|
-
allOrganizations.forEach((organization) => {
|
|
78
|
-
const systemEntity = {
|
|
79
|
-
apiVersion: "backstage.io/v1alpha1",
|
|
80
|
-
kind: "System",
|
|
81
|
-
spec: {
|
|
82
|
-
owner: bcGovGroupId,
|
|
83
|
-
type: "government"
|
|
84
|
-
},
|
|
85
|
-
metadata: {
|
|
86
|
-
name: this.toSafeName(organization.name),
|
|
87
|
-
title: organization.title,
|
|
88
|
-
description: organization.description,
|
|
89
|
-
annotations: {
|
|
90
|
-
"backstage.io/managed-by-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search",
|
|
91
|
-
"backstage.io/managed-by-origin-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search",
|
|
92
|
-
"bcdata.gov.bc.ca/organization-id": organization.id,
|
|
93
|
-
"bcdata.gov.bc.ca/organization-type": organization.type,
|
|
94
|
-
"bcdata.gov.bc.ca/organization-created": organization.created,
|
|
95
|
-
"bcdata.gov.bc.ca/organization-approval-status": organization.approval_status,
|
|
96
|
-
"bcdata.gov.bc.ca/organization-state": organization.state
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
};
|
|
100
|
-
allSystems.set(this.getSystemId(organization.name), systemEntity);
|
|
39
|
+
const client = new BcDataCatalogueClient.BcDataCatalogueClient({
|
|
40
|
+
reader: this.reader,
|
|
41
|
+
logger: this.logger
|
|
101
42
|
});
|
|
102
|
-
const
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
let user = allUsers.get(email);
|
|
107
|
-
if (user == void 0) {
|
|
108
|
-
user = {
|
|
109
|
-
apiVersion: "backstage.io/v1alpha1",
|
|
110
|
-
kind: "User",
|
|
111
|
-
spec: {
|
|
112
|
-
profile: {
|
|
113
|
-
email
|
|
114
|
-
},
|
|
115
|
-
memberOf: []
|
|
116
|
-
},
|
|
117
|
-
metadata: {
|
|
118
|
-
name: this.toSafeName(email),
|
|
119
|
-
annotations: {
|
|
120
|
-
"backstage.io/managed-by-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search",
|
|
121
|
-
"backstage.io/managed-by-origin-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search"
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
};
|
|
125
|
-
allUsers.set(this.getUserId(email), user);
|
|
126
|
-
}
|
|
127
|
-
if (user?.spec.profile?.displayName == void 0) {
|
|
128
|
-
user.spec.profile.displayName = contact.name;
|
|
129
|
-
}
|
|
130
|
-
const hostName = this.getEmailHostname(email);
|
|
131
|
-
if (hostName == void 0) {
|
|
132
|
-
this.logger.warn(`Failed to extract hostname from email address ${email}`);
|
|
133
|
-
} else {
|
|
134
|
-
const groupId = this.getGroupId(hostName);
|
|
135
|
-
let group = allGroups.get(groupId);
|
|
136
|
-
if (group == void 0) {
|
|
137
|
-
allGroups.set(groupId, this.createGroupEntity(hostName, hostName, bcGovGroupId));
|
|
138
|
-
}
|
|
139
|
-
if (!user.spec.memberOf?.includes(groupId)) {
|
|
140
|
-
user.spec.memberOf?.push(groupId);
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
});
|
|
43
|
+
const factory = new BcDataCatalogueEntityFactory.BcDataCatalogueEntityFactory({
|
|
44
|
+
reader: this.reader,
|
|
45
|
+
logger: this.logger,
|
|
46
|
+
allowedHosts: this.allowedHosts
|
|
144
47
|
});
|
|
145
|
-
const
|
|
146
|
-
const
|
|
147
|
-
for (const pkg of allPackages) {
|
|
148
|
-
const safeName = this.toSafeName(pkg.name);
|
|
149
|
-
const systemId = this.getSystemId(pkg.organization.name);
|
|
150
|
-
const entityLinks = [];
|
|
151
|
-
pkg.contacts.forEach((contact) => {
|
|
152
|
-
const email = contact.email.toLowerCase();
|
|
153
|
-
const entityLink = {
|
|
154
|
-
url: `mailto:${email}`,
|
|
155
|
-
title: `Contact: ${contact.name}`,
|
|
156
|
-
icon: "email",
|
|
157
|
-
type: "contact"
|
|
158
|
-
};
|
|
159
|
-
entityLinks.push(entityLink);
|
|
160
|
-
});
|
|
161
|
-
pkg.more_info?.forEach((more_info) => {
|
|
162
|
-
if (more_info.url.length > 0) {
|
|
163
|
-
const entityLink = {
|
|
164
|
-
url: more_info.url,
|
|
165
|
-
title: more_info.description || more_info.url,
|
|
166
|
-
icon: "externalLink",
|
|
167
|
-
type: "more_info"
|
|
168
|
-
};
|
|
169
|
-
entityLinks.push(entityLink);
|
|
170
|
-
}
|
|
171
|
-
});
|
|
172
|
-
const apiResources = [];
|
|
173
|
-
pkg.resources?.forEach((resource) => {
|
|
174
|
-
if (resource.bcdc_type == "webservice" && resource.format != "kml" || resource.format == "arcgis_rest" || resource.format == "openapi-json") {
|
|
175
|
-
apiResources.push(resource);
|
|
176
|
-
} else if (resource.bcdc_type == "geographic") ; else if (resource.url.length > 0) {
|
|
177
|
-
const entityLink = {
|
|
178
|
-
url: resource.url,
|
|
179
|
-
title: resource.name,
|
|
180
|
-
icon: "catalog",
|
|
181
|
-
type: resource.bcdc_type
|
|
182
|
-
};
|
|
183
|
-
entityLinks.push(entityLink);
|
|
184
|
-
} else {
|
|
185
|
-
this.logger.info(`Missing URL ${resource.bcdc_type} ${resource.format} ${pkg.name}`);
|
|
186
|
-
}
|
|
187
|
-
});
|
|
188
|
-
pkg.dates.forEach((date) => {
|
|
189
|
-
({ type: date.type, date: date.type });
|
|
190
|
-
});
|
|
191
|
-
const tags = [];
|
|
192
|
-
pkg.tags?.forEach((tag) => {
|
|
193
|
-
tags.push(this.toSafeName(tag.display_name));
|
|
194
|
-
});
|
|
195
|
-
const componentEntity = {
|
|
196
|
-
apiVersion: "backstage.io/v1alpha1",
|
|
197
|
-
kind: "Component",
|
|
198
|
-
spec: {
|
|
199
|
-
type: pkg.type,
|
|
200
|
-
lifecycle: "production",
|
|
201
|
-
owner: bcGovGroupId,
|
|
202
|
-
subcomponentOf: void 0,
|
|
203
|
-
providesApis: [],
|
|
204
|
-
consumesApis: void 0,
|
|
205
|
-
dependsOn: void 0,
|
|
206
|
-
dependencyOf: void 0,
|
|
207
|
-
system: systemId
|
|
208
|
-
},
|
|
209
|
-
metadata: {
|
|
210
|
-
name: safeName,
|
|
211
|
-
description: pkg.notes || "No description available",
|
|
212
|
-
annotations: {
|
|
213
|
-
"backstage.io/managed-by-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search",
|
|
214
|
-
"backstage.io/managed-by-origin-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search",
|
|
215
|
-
"bcdata.gov.bc.ca/package-author": pkg.author || "Unknown",
|
|
216
|
-
"bcdata.gov.bc.ca/package-author_email": pkg.author_email || "Unknown",
|
|
217
|
-
"bcdata.gov.bc.ca/package-creator_user_id": pkg.creator_user_id,
|
|
218
|
-
"bcdata.gov.bc.ca/package-download_audience": pkg.download_audience,
|
|
219
|
-
"bcdata.gov.bc.ca/package-id": pkg.id,
|
|
220
|
-
"bcdata.gov.bc.ca/package-isopen": `${pkg.isopen}`,
|
|
221
|
-
"bcdata.gov.bc.ca/package-license_id": pkg.license_id,
|
|
222
|
-
"bcdata.gov.bc.ca/package-license_title": pkg.license_title || "Unknown",
|
|
223
|
-
"bcdata.gov.bc.ca/package-license_url": pkg.license_url,
|
|
224
|
-
"bcdata.gov.bc.ca/package-maintainer": pkg.maintainer || "Unknown",
|
|
225
|
-
"bcdata.gov.bc.ca/package-maintainer_email": pkg.maintainer_email || "Unknown",
|
|
226
|
-
"bcdata.gov.bc.ca/package-metadata_created": pkg.metadata_created,
|
|
227
|
-
"bcdata.gov.bc.ca/package-metadata_modified": pkg.metadata_modified,
|
|
228
|
-
"bcdata.gov.bc.ca/package-metadata_visibility": pkg.metadata_visibility,
|
|
229
|
-
"bcdata.gov.bc.ca/package-name": pkg.name,
|
|
230
|
-
"bcdata.gov.bc.ca/package-notes": pkg.notes || "Unknown",
|
|
231
|
-
"bcdata.gov.bc.ca/package-owner_org": pkg.owner_org,
|
|
232
|
-
"bcdata.gov.bc.ca/package-private": `${pkg.private}`,
|
|
233
|
-
"bcdata.gov.bc.ca/package-publish_state": pkg.publish_state,
|
|
234
|
-
"bcdata.gov.bc.ca/package-record_create_date": pkg.record_create_date || "Unknown",
|
|
235
|
-
"bcdata.gov.bc.ca/package-record_last_modified": pkg.record_last_modified,
|
|
236
|
-
"bcdata.gov.bc.ca/package-record_publish_date": pkg.record_publish_date,
|
|
237
|
-
"bcdata.gov.bc.ca/package-resource_status": pkg.resource_status,
|
|
238
|
-
"bcdata.gov.bc.ca/package-security_class": pkg.security_class,
|
|
239
|
-
"bcdata.gov.bc.ca/package-state": pkg.state,
|
|
240
|
-
"bcdata.gov.bc.ca/package-title": pkg.title || "Unknown",
|
|
241
|
-
"bcdata.gov.bc.ca/package-type": pkg.type,
|
|
242
|
-
"bcdata.gov.bc.ca/package-url": pkg.url || "Unknown",
|
|
243
|
-
"bcdata.gov.bc.ca/package-version": pkg.version || "Unknown",
|
|
244
|
-
"bcdata.gov.bc.ca/package-view_audience": pkg.view_audience
|
|
245
|
-
},
|
|
246
|
-
links: entityLinks,
|
|
247
|
-
tags
|
|
248
|
-
}
|
|
249
|
-
};
|
|
250
|
-
allComponents.set(this.getComponentId(safeName), componentEntity);
|
|
251
|
-
for (const apiResource of apiResources) {
|
|
252
|
-
const name = apiResource.name;
|
|
253
|
-
let resourceSafeName = this.toSafeName(name);
|
|
254
|
-
let apiSafeName = resourceSafeName;
|
|
255
|
-
if (apiResource.bcdc_type === "webservice" || apiResource.format === "arcgis_rest") {
|
|
256
|
-
const prefix = apiResource.format === "openapi-json" ? "api" : apiResource.format;
|
|
257
|
-
const formatSafeName = this.toSafeName(prefix);
|
|
258
|
-
let baseName = this.toSafeName(`${formatSafeName}-${safeName}`);
|
|
259
|
-
let candidateName = baseName;
|
|
260
|
-
let candidateApiId = this.getApiId(candidateName);
|
|
261
|
-
if (allApis.has(candidateApiId)) {
|
|
262
|
-
const distinguishingSuffix = this.extractDistinguishingSuffix(name);
|
|
263
|
-
const suffixWithSeparator = `-${distinguishingSuffix}`;
|
|
264
|
-
const maxBaseLength = 63 - suffixWithSeparator.length;
|
|
265
|
-
let truncatedBase = baseName;
|
|
266
|
-
if (baseName.length > maxBaseLength) {
|
|
267
|
-
truncatedBase = baseName.slice(0, maxBaseLength).replace(/[-_.]+$/, "");
|
|
268
|
-
}
|
|
269
|
-
candidateName = this.toSafeName(`${truncatedBase}${suffixWithSeparator}`);
|
|
270
|
-
candidateApiId = this.getApiId(candidateName);
|
|
271
|
-
let counter = 1;
|
|
272
|
-
while (allApis.has(candidateApiId)) {
|
|
273
|
-
const counterSuffix = `-${counter}`;
|
|
274
|
-
const maxBaseWithCounter = 63 - suffixWithSeparator.length - counterSuffix.length;
|
|
275
|
-
let truncatedBaseForCounter = baseName;
|
|
276
|
-
if (baseName.length > maxBaseWithCounter) {
|
|
277
|
-
truncatedBaseForCounter = baseName.slice(0, maxBaseWithCounter).replace(/[-_.]+$/, "");
|
|
278
|
-
}
|
|
279
|
-
candidateName = this.toSafeName(`${truncatedBaseForCounter}${suffixWithSeparator}${counterSuffix}`);
|
|
280
|
-
candidateApiId = this.getApiId(candidateName);
|
|
281
|
-
counter++;
|
|
282
|
-
}
|
|
283
|
-
}
|
|
284
|
-
apiSafeName = candidateName;
|
|
285
|
-
}
|
|
286
|
-
const definition = apiResource.url;
|
|
287
|
-
const url = new URL(definition);
|
|
288
|
-
const host = url.host.toLowerCase();
|
|
289
|
-
if (!this.allowedHosts.includes(host)) {
|
|
290
|
-
this.logger.warn(`API definition host is NOT allowed: "${host}"`);
|
|
291
|
-
}
|
|
292
|
-
const apiEntityLinks = [];
|
|
293
|
-
if (apiResource.url && apiResource.url.length > 0) {
|
|
294
|
-
const apiEntityLink = {
|
|
295
|
-
url: apiResource.url,
|
|
296
|
-
title: apiResource.name,
|
|
297
|
-
icon: "api",
|
|
298
|
-
type: apiResource.bcdc_type
|
|
299
|
-
};
|
|
300
|
-
apiEntityLinks.push(apiEntityLink);
|
|
301
|
-
}
|
|
302
|
-
let definitionContent = apiResource.url;
|
|
303
|
-
if (apiResource.format === "openapi-json") {
|
|
304
|
-
try {
|
|
305
|
-
const response = await this.reader.readUrl(apiResource.url);
|
|
306
|
-
const content = (await response.buffer()).toString();
|
|
307
|
-
definitionContent = content;
|
|
308
|
-
} catch (error) {
|
|
309
|
-
this.logger.warn(`Failed to fetch OpenAPI definition from ${apiResource.url}: ${error}`);
|
|
310
|
-
definitionContent = apiResource.url;
|
|
311
|
-
}
|
|
312
|
-
}
|
|
313
|
-
const apiEntity = {
|
|
314
|
-
apiVersion: "backstage.io/v1alpha1",
|
|
315
|
-
kind: "API",
|
|
316
|
-
spec: {
|
|
317
|
-
type: apiResource.format === "openapi-json" ? "openapi" : apiResource.bcdc_type,
|
|
318
|
-
lifecycle: "production",
|
|
319
|
-
owner: bcGovGroupId,
|
|
320
|
-
definition: definitionContent,
|
|
321
|
-
system: systemId
|
|
322
|
-
},
|
|
323
|
-
metadata: {
|
|
324
|
-
name: apiSafeName,
|
|
325
|
-
description: apiResource.description || "No description available",
|
|
326
|
-
links: apiEntityLinks,
|
|
327
|
-
// tags: [this.toSafeName(apiResource.format)],
|
|
328
|
-
tags: [this.toSafeName(apiResource.format)],
|
|
329
|
-
annotations: {
|
|
330
|
-
"backstage.io/managed-by-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search",
|
|
331
|
-
"backstage.io/managed-by-origin-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search",
|
|
332
|
-
"bcdata.gov.bc.ca/resource-bcdc_type": apiResource.bcdc_type,
|
|
333
|
-
"bcdata.gov.bc.ca/resource-cache_last_updated": apiResource.cache_last_updated || "Undefined",
|
|
334
|
-
"bcdata.gov.bc.ca/resource-cache_url": apiResource.cache_url || "Undefined",
|
|
335
|
-
"bcdata.gov.bc.ca/resource-created": apiResource.created,
|
|
336
|
-
"bcdata.gov.bc.ca/resource-datastore_active": `${apiResource.datastore_active}`,
|
|
337
|
-
"bcdata.gov.bc.ca/resource-description": apiResource.description || "Undefined",
|
|
338
|
-
// 'bcdata.gov.bc.ca/resource-details': apiResource.details,
|
|
339
|
-
"bcdata.gov.bc.ca/resource-format": apiResource.format,
|
|
340
|
-
// 'bcdata.gov.bc.ca/resource-geographic_extent': apiResource.geographic_extent,
|
|
341
|
-
"bcdata.gov.bc.ca/resource-hash": apiResource.hash,
|
|
342
|
-
"bcdata.gov.bc.ca/resource-id": apiResource.id,
|
|
343
|
-
// 'bcdata.gov.bc.ca/resource-iso_topic_category': apiResource.iso_topic_category || 'Undefined',
|
|
344
|
-
"bcdata.gov.bc.ca/resource-metadata_modified": apiResource.metadata_modified,
|
|
345
|
-
"bcdata.gov.bc.ca/resource-mimetype": apiResource.mimetype || "Undefined",
|
|
346
|
-
"bcdata.gov.bc.ca/resource-name": apiResource.name,
|
|
347
|
-
"bcdata.gov.bc.ca/resource-package_id": apiResource.package_id,
|
|
348
|
-
"bcdata.gov.bc.ca/resource-position": `${apiResource.position}`,
|
|
349
|
-
// 'bcdata.gov.bc.ca/resource-preview_info': apiResource.preview_info,
|
|
350
|
-
"bcdata.gov.bc.ca/resource-projection_name": apiResource.projection_name || "Undefined",
|
|
351
|
-
"bcdata.gov.bc.ca/resource-resource_access_method": apiResource.resource_access_method,
|
|
352
|
-
"bcdata.gov.bc.ca/resource-resource_storage_location": apiResource.resource_storage_location,
|
|
353
|
-
"bcdata.gov.bc.ca/resource-resource_type": apiResource.resource_type,
|
|
354
|
-
"bcdata.gov.bc.ca/resource-resource_update_cycle": apiResource.resource_update_cycle,
|
|
355
|
-
"bcdata.gov.bc.ca/resource-size": `${apiResource.size}`,
|
|
356
|
-
"bcdata.gov.bc.ca/resource-spatial_datatype": apiResource.spatial_datatype || "Undefined",
|
|
357
|
-
"bcdata.gov.bc.ca/resource-state": apiResource.state,
|
|
358
|
-
"bcdata.gov.bc.ca/resource-url": apiResource.url,
|
|
359
|
-
"bcdata.gov.bc.ca/resource-url_type": apiResource.url_type || "Undefined"
|
|
360
|
-
}
|
|
361
|
-
}
|
|
362
|
-
};
|
|
363
|
-
const apiId = this.getApiId(apiSafeName);
|
|
364
|
-
if (allApis.has(apiId)) {
|
|
365
|
-
const existingApi = allApis.get(apiId);
|
|
366
|
-
this.logger.warn(
|
|
367
|
-
`Duplicate API name detected: "${apiSafeName}" (ID: ${apiId}). Existing API: name="${existingApi.metadata.name}", Format: "${existingApi.metadata.annotations?.["bcdata.gov.bc.ca/resource-format"]}", resource-id="${existingApi.metadata.annotations?.["bcdata.gov.bc.ca/resource-id"]}", package-id="${existingApi.metadata.annotations?.["bcdata.gov.bc.ca/resource-package_id"]}", url="${existingApi.metadata.annotations?.["bcdata.gov.bc.ca/resource-url"]}". New API: name="${apiEntity.metadata.name}", resource-id="${apiResource.id}", package-id="${apiResource.package_id}", url="${apiResource.url}". The new API will overwrite the existing one.`
|
|
368
|
-
);
|
|
369
|
-
}
|
|
370
|
-
allApis.set(apiId, apiEntity);
|
|
371
|
-
componentEntity.spec.providesApis?.push(apiId);
|
|
372
|
-
}
|
|
373
|
-
}
|
|
374
|
-
const userEntities = Array.from(allUsers.values());
|
|
375
|
-
const groupEntities = Array.from(allGroups.values());
|
|
376
|
-
const systemEntities = Array.from(allSystems.values());
|
|
377
|
-
const componentEntities = Array.from(allComponents.values());
|
|
378
|
-
const apiEntities = Array.from(allApis.values());
|
|
379
|
-
allEntities.push(...userEntities);
|
|
380
|
-
allEntities.push(...groupEntities);
|
|
381
|
-
allEntities.push(...systemEntities);
|
|
382
|
-
allEntities.push(...componentEntities);
|
|
383
|
-
allEntities.push(...apiEntities);
|
|
48
|
+
const packages = await client.getAllPackages();
|
|
49
|
+
const entities = await factory.createEntities(packages);
|
|
384
50
|
await this.connection.applyMutation({
|
|
385
51
|
type: "full",
|
|
386
|
-
entities:
|
|
52
|
+
entities: entities.map((entity) => ({
|
|
387
53
|
entity,
|
|
388
54
|
locationKey: `bc-data-catalogue:${this.env}`
|
|
389
55
|
}))
|
|
390
56
|
});
|
|
391
|
-
this.logger.info(">run");
|
|
392
|
-
}
|
|
393
|
-
getEmailHostname(email) {
|
|
394
|
-
const match = email.trim().toLowerCase().match(/@([\w.-]+)/);
|
|
395
|
-
return match ? match[1] : void 0;
|
|
396
|
-
}
|
|
397
|
-
getSystemId(name) {
|
|
398
|
-
return `system:default/${this.toSafeName(name)}`;
|
|
399
|
-
}
|
|
400
|
-
getGroupId(hostName) {
|
|
401
|
-
return `group:default/${this.toSafeName(hostName)}`;
|
|
402
|
-
}
|
|
403
|
-
getUserId(email) {
|
|
404
|
-
return `user:default/${email.toLowerCase()}`;
|
|
405
|
-
}
|
|
406
|
-
getComponentId(name) {
|
|
407
|
-
return `component:default/${name.toLowerCase()}`;
|
|
408
|
-
}
|
|
409
|
-
getApiId(name) {
|
|
410
|
-
return `api:default/${name.toLowerCase()}`;
|
|
411
|
-
}
|
|
412
|
-
createGroupEntity(hostName, displayName, parentGroup) {
|
|
413
|
-
const groupEntity = {
|
|
414
|
-
apiVersion: "backstage.io/v1alpha1",
|
|
415
|
-
kind: "Group",
|
|
416
|
-
metadata: {
|
|
417
|
-
name: this.toSafeName(hostName),
|
|
418
|
-
annotations: {
|
|
419
|
-
"backstage.io/managed-by-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search",
|
|
420
|
-
"backstage.io/managed-by-origin-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search"
|
|
421
|
-
}
|
|
422
|
-
},
|
|
423
|
-
spec: {
|
|
424
|
-
type: "government",
|
|
425
|
-
profile: {
|
|
426
|
-
displayName
|
|
427
|
-
},
|
|
428
|
-
parent: parentGroup,
|
|
429
|
-
children: [],
|
|
430
|
-
members: []
|
|
431
|
-
}
|
|
432
|
-
};
|
|
433
|
-
return groupEntity;
|
|
434
|
-
}
|
|
435
|
-
/**
|
|
436
|
-
* Converts BC Data Catalogue (CKAN package_search) results into Backstage Entities
|
|
437
|
-
*/
|
|
438
|
-
getBcDataCataloguePackages(results) {
|
|
439
|
-
this.logger.info("<getBcDataCataloguePackages", {
|
|
440
|
-
resultCount: Array.isArray(results) ? results.length : "unknown"
|
|
441
|
-
});
|
|
442
|
-
const rawPackages = results;
|
|
443
|
-
this.logger.info(`rawPackageCount ${rawPackages.length}`);
|
|
444
|
-
const validPackages = [];
|
|
445
|
-
for (const item of rawPackages) {
|
|
446
|
-
const parseResult = BcDataCatalogueModel.BcDataCataloguePackageSchema.safeParse(item);
|
|
447
|
-
if (parseResult.success) {
|
|
448
|
-
const pkg = parseResult.data;
|
|
449
|
-
if (pkg.type === "bcdc_dataset" && pkg.state === "active") {
|
|
450
|
-
validPackages.push(pkg);
|
|
451
|
-
}
|
|
452
|
-
} else {
|
|
453
|
-
this.logger.warn(JSON.stringify(item));
|
|
454
|
-
this.logger.warn("Invalid package skipped", {
|
|
455
|
-
error: parseResult.error.message
|
|
456
|
-
});
|
|
457
|
-
throw new Error(
|
|
458
|
-
`BC Data Catalogue package validation failedSee logs for details. First issue: ${parseResult.error.issues[0]?.message}`
|
|
459
|
-
);
|
|
460
|
-
}
|
|
461
|
-
}
|
|
462
|
-
this.logger.info(`>getBcDataCataloguePackages ${validPackages.length}`);
|
|
463
|
-
return validPackages;
|
|
464
|
-
}
|
|
465
|
-
toSafeName(name) {
|
|
466
|
-
return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/(?:^[-_.]+|[-_.]+$)/g, "").slice(0, 63).replace(/[-_.]+$/, "");
|
|
467
|
-
}
|
|
468
|
-
/**
|
|
469
|
-
* Extracts distinguishing information from a resource name to help make API names unique.
|
|
470
|
-
* Tries to extract meaningful parts like years, version numbers, or other identifiers.
|
|
471
|
-
*/
|
|
472
|
-
extractDistinguishingSuffix(resourceName) {
|
|
473
|
-
const safeName = this.toSafeName(resourceName);
|
|
474
|
-
const yearMatch = resourceName.match(/\b(19|20)\d{2}\b/);
|
|
475
|
-
if (yearMatch) {
|
|
476
|
-
return yearMatch[0];
|
|
477
|
-
}
|
|
478
|
-
const versionMatch = resourceName.match(/\b(?:v|version)[\s_-]?(\d+)\b/i);
|
|
479
|
-
if (versionMatch) {
|
|
480
|
-
return `v${versionMatch[1]}`;
|
|
481
|
-
}
|
|
482
|
-
const trailingNumberMatch = resourceName.match(/\b(\d{2,})\b/);
|
|
483
|
-
if (trailingNumberMatch) {
|
|
484
|
-
return trailingNumberMatch[0];
|
|
485
|
-
}
|
|
486
|
-
const words = safeName.split("-").filter(
|
|
487
|
-
(w) => w.length > 0 && !["service", "request", "getcapabilities", "wms", "kml", "arcgis", "rest", "online"].includes(w)
|
|
488
|
-
);
|
|
489
|
-
if (words.length >= 2) {
|
|
490
|
-
return words.slice(-2).join("-");
|
|
491
|
-
} else if (words.length === 1) {
|
|
492
|
-
return words[0];
|
|
493
|
-
} else {
|
|
494
|
-
return safeName.split("-").slice(-1)[0].slice(0, 10);
|
|
495
|
-
}
|
|
57
|
+
this.logger.info("[BCDC Entity Provider] >run");
|
|
496
58
|
}
|
|
497
59
|
}
|
|
498
60
|
|