@bcgov/plugin-catalog-backend-module-bc-data-catalogue 0.1.0 → 0.2.5-feature.aps-4105-connect-pipelines.cbf325cb

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,42 +1,43 @@
1
- # plugin-catalog-backend-module-bc-data-catalogue
1
+ # @bcgov/plugin-catalog-backend-module-bc-data-catalogue
2
2
 
3
- This plugin integrates the BC Data Catalogue with Backstage, allowing you to import datasets, resources, and their metadata as entities in Backstage's software catalogue.
3
+ This package is part of the BC Data Catalogue Backstage integration.
4
4
 
5
- ## Installation
5
+ 👉 **Refer to the main project README at the root of this repository for installation and configuration instructions.**
6
6
 
7
- This package is published to npmjs.com under the scope `@bcgov`.
7
+ ---
8
8
 
9
- ### Install the Package
9
+ ## Purpose
10
10
 
11
- #### Using Yarn
11
+ The `@bcgov/plugin-catalog-backend-module-bc-data-catalogue` package provides the backend integration with the BC Data Catalogue.
12
12
 
13
- yarn add @bcgov/plugin-catalog-backend-module-bc-data-catalogue
13
+ It is responsible for:
14
14
 
15
- ### Usage in Backstage
15
+ - fetching datasets from the BC Data Catalogue
16
+ - transforming them into Backstage entities
17
+ - registering entities such as:
18
+ - `Dataset`
19
+ - `API`
20
+ - `System`
21
+ - `Group`
22
+ - `User`
16
23
 
17
- Add the module to your Backstage backend in `packages/backend/src/index.ts`:
24
+ ---
18
25
 
19
- ```diff
20
- import { createBackend } from '@backstage/backend-defaults';
26
+ ## Usage
21
27
 
22
- const backend = createBackend();
23
- // ... other backend.add() calls ...
28
+ This package is intended to be registered as a backend module in Backstage:
24
29
 
25
- + backend.add(import('@bcgov/plugin-catalog-backend-module-bc-data-catalogue'));
26
-
27
- backend.start();
30
+ ```ts
31
+ backend.add(import('@bcgov/plugin-catalog-backend-module-bc-data-catalogue'));
28
32
  ```
29
33
 
30
- ### Configure backend reading allowlist
31
-
32
- Add the following configuration to your `app-config.yaml` to allow the backend to access the BC Data Catalogue API:
34
+ ---
33
35
 
34
- ```
35
- backend:
36
- reading:
37
- allow:
38
- - host: catalogue.data.gov.bc.ca
39
- scheme: https
40
- ```
36
+ ## Notes
41
37
 
42
- **Note:** Additional hosts may be required depending what resources are being loaded. See https://github.com/bcgov/csit-backstage-poc/blob/main/app-config.yaml for example.
38
+ - This package is **not intended to be used in isolation**
39
+ - It depends on:
40
+ - `@bcgov/plugin-catalog-common-bc-data-catalogue`
41
+ - It is designed to be used together with:
42
+ - `@bcgov/catalog-dataset` (frontend UI)
43
+ - All setup, configuration, and environment details are documented in the root project README
@@ -1,6 +1,7 @@
1
1
  'use strict';
2
2
 
3
- var BcDataCatalogueModel = require('./BcDataCatalogueModel.cjs.js');
3
+ var BcDataCatalogueClient = require('./BcDataCatalogueClient.cjs.js');
4
+ var BcDataCatalogueEntityFactory = require('./BcDataCatalogueEntityFactory.cjs.js');
4
5
 
5
6
  class BcDataCatalogueApisProvider {
6
7
  logger;
@@ -9,7 +10,6 @@ class BcDataCatalogueApisProvider {
9
10
  connection;
10
11
  taskRunner;
11
12
  allowedHosts;
12
- /** [1] */
13
13
  constructor(env, reader, taskRunner, allowedHosts, logger) {
14
14
  this.env = env;
15
15
  this.reader = reader;
@@ -17,13 +17,11 @@ class BcDataCatalogueApisProvider {
17
17
  this.allowedHosts = allowedHosts;
18
18
  this.logger = logger;
19
19
  }
20
- /** [2] */
21
20
  getProviderName() {
22
21
  return `bc-data-catalogue-${this.env}`;
23
22
  }
24
- /** [3] */
25
23
  async connect(connection) {
26
- this.logger.info("<connect");
24
+ this.logger.info("[BCDC Entity Provider] <connect");
27
25
  this.connection = connection;
28
26
  await this.taskRunner.run({
29
27
  id: this.getProviderName(),
@@ -31,468 +29,32 @@ class BcDataCatalogueApisProvider {
31
29
  await this.run();
32
30
  }
33
31
  });
34
- this.logger.info(">connect");
32
+ this.logger.info("[BCDC Entity Provider] >connect");
35
33
  }
36
- /** [4] */
37
34
  async run() {
38
- this.logger.info("<run");
35
+ this.logger.info("[BCDC Entity Provider] <run");
39
36
  if (!this.connection) {
40
37
  throw new Error("Not initialized");
41
38
  }
42
- let page = 0;
43
- let retries = 0;
44
- let allEntities = [];
45
- let allPackages = [];
46
- do {
47
- const start = 1e3 * page;
48
- const url = `https://catalogue.data.gov.bc.ca/api/3/action/package_search?start=${start}&rows=1000`;
49
- const response = await this.reader.readUrl(url);
50
- const data = JSON.parse((await response.buffer()).toString());
51
- if (!data?.success || !Array.isArray(data.result?.results)) {
52
- this.logger.warn("Invalid or unsuccessful response", { success: data?.success });
53
- ++retries;
54
- continue;
55
- }
56
- retries = 0;
57
- const results = data.result.results;
58
- if (results.length > 0) {
59
- const packages = this.getBcDataCataloguePackages(results);
60
- allPackages.push(...packages);
61
- ++page;
62
- } else {
63
- page = -1;
64
- }
65
- } while (page > 0 && retries < 3);
66
- this.logger.info(`Total packages ${allPackages.length}`);
67
- const allGroups = /* @__PURE__ */ new Map();
68
- const bcGovGroupId = this.getGroupId("gov.bc.ca");
69
- allGroups.set(bcGovGroupId, this.createGroupEntity("gov.bc.ca", "Governent of British Columbia", void 0));
70
- const allOrganizations = /* @__PURE__ */ new Map();
71
- allPackages.forEach((pkg) => {
72
- const organization = pkg.organization;
73
- allOrganizations.set(organization.id, organization);
74
- });
75
- const allSystems = /* @__PURE__ */ new Map();
76
- this.logger.info(`Organizations ${allOrganizations.size}`);
77
- allOrganizations.forEach((organization) => {
78
- const systemEntity = {
79
- apiVersion: "backstage.io/v1alpha1",
80
- kind: "System",
81
- spec: {
82
- owner: bcGovGroupId,
83
- type: "government"
84
- },
85
- metadata: {
86
- name: this.toSafeName(organization.name),
87
- title: organization.title,
88
- description: organization.description,
89
- annotations: {
90
- "backstage.io/managed-by-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search",
91
- "backstage.io/managed-by-origin-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search",
92
- "bcdata.gov.bc.ca/organization-id": organization.id,
93
- "bcdata.gov.bc.ca/organization-type": organization.type,
94
- "bcdata.gov.bc.ca/organization-created": organization.created,
95
- "bcdata.gov.bc.ca/organization-approval-status": organization.approval_status,
96
- "bcdata.gov.bc.ca/organization-state": organization.state
97
- }
98
- }
99
- };
100
- allSystems.set(this.getSystemId(organization.name), systemEntity);
39
+ const client = new BcDataCatalogueClient.BcDataCatalogueClient({
40
+ reader: this.reader,
41
+ logger: this.logger
101
42
  });
102
- const allUsers = /* @__PURE__ */ new Map();
103
- allPackages.forEach((pkg) => {
104
- pkg.contacts?.forEach((contact) => {
105
- const email = contact.email.toLowerCase();
106
- let user = allUsers.get(email);
107
- if (user == void 0) {
108
- user = {
109
- apiVersion: "backstage.io/v1alpha1",
110
- kind: "User",
111
- spec: {
112
- profile: {
113
- email
114
- },
115
- memberOf: []
116
- },
117
- metadata: {
118
- name: this.toSafeName(email),
119
- annotations: {
120
- "backstage.io/managed-by-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search",
121
- "backstage.io/managed-by-origin-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search"
122
- }
123
- }
124
- };
125
- allUsers.set(this.getUserId(email), user);
126
- }
127
- if (user?.spec.profile?.displayName == void 0) {
128
- user.spec.profile.displayName = contact.name;
129
- }
130
- const hostName = this.getEmailHostname(email);
131
- if (hostName == void 0) {
132
- this.logger.warn(`Failed to extract hostname from email address ${email}`);
133
- } else {
134
- const groupId = this.getGroupId(hostName);
135
- let group = allGroups.get(groupId);
136
- if (group == void 0) {
137
- allGroups.set(groupId, this.createGroupEntity(hostName, hostName, bcGovGroupId));
138
- }
139
- if (!user.spec.memberOf?.includes(groupId)) {
140
- user.spec.memberOf?.push(groupId);
141
- }
142
- }
143
- });
43
+ const factory = new BcDataCatalogueEntityFactory.BcDataCatalogueEntityFactory({
44
+ reader: this.reader,
45
+ logger: this.logger,
46
+ allowedHosts: this.allowedHosts
144
47
  });
145
- const allComponents = /* @__PURE__ */ new Map();
146
- const allApis = /* @__PURE__ */ new Map();
147
- for (const pkg of allPackages) {
148
- const safeName = this.toSafeName(pkg.name);
149
- const systemId = this.getSystemId(pkg.organization.name);
150
- const entityLinks = [];
151
- pkg.contacts.forEach((contact) => {
152
- const email = contact.email.toLowerCase();
153
- const entityLink = {
154
- url: `mailto:${email}`,
155
- title: `Contact: ${contact.name}`,
156
- icon: "email",
157
- type: "contact"
158
- };
159
- entityLinks.push(entityLink);
160
- });
161
- pkg.more_info?.forEach((more_info) => {
162
- if (more_info.url.length > 0) {
163
- const entityLink = {
164
- url: more_info.url,
165
- title: more_info.description || more_info.url,
166
- icon: "externalLink",
167
- type: "more_info"
168
- };
169
- entityLinks.push(entityLink);
170
- }
171
- });
172
- const apiResources = [];
173
- pkg.resources?.forEach((resource) => {
174
- if (resource.bcdc_type == "webservice" && resource.format != "kml" || resource.format == "arcgis_rest" || resource.format == "openapi-json") {
175
- apiResources.push(resource);
176
- } else if (resource.bcdc_type == "geographic") ; else if (resource.url.length > 0) {
177
- const entityLink = {
178
- url: resource.url,
179
- title: resource.name,
180
- icon: "catalog",
181
- type: resource.bcdc_type
182
- };
183
- entityLinks.push(entityLink);
184
- } else {
185
- this.logger.info(`Missing URL ${resource.bcdc_type} ${resource.format} ${pkg.name}`);
186
- }
187
- });
188
- pkg.dates.forEach((date) => {
189
- ({ type: date.type, date: date.type });
190
- });
191
- const tags = [];
192
- pkg.tags?.forEach((tag) => {
193
- tags.push(this.toSafeName(tag.display_name));
194
- });
195
- const componentEntity = {
196
- apiVersion: "backstage.io/v1alpha1",
197
- kind: "Component",
198
- spec: {
199
- type: pkg.type,
200
- lifecycle: "production",
201
- owner: bcGovGroupId,
202
- subcomponentOf: void 0,
203
- providesApis: [],
204
- consumesApis: void 0,
205
- dependsOn: void 0,
206
- dependencyOf: void 0,
207
- system: systemId
208
- },
209
- metadata: {
210
- name: safeName,
211
- description: pkg.notes || "No description available",
212
- annotations: {
213
- "backstage.io/managed-by-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search",
214
- "backstage.io/managed-by-origin-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search",
215
- "bcdata.gov.bc.ca/package-author": pkg.author || "Unknown",
216
- "bcdata.gov.bc.ca/package-author_email": pkg.author_email || "Unknown",
217
- "bcdata.gov.bc.ca/package-creator_user_id": pkg.creator_user_id,
218
- "bcdata.gov.bc.ca/package-download_audience": pkg.download_audience,
219
- "bcdata.gov.bc.ca/package-id": pkg.id,
220
- "bcdata.gov.bc.ca/package-isopen": `${pkg.isopen}`,
221
- "bcdata.gov.bc.ca/package-license_id": pkg.license_id,
222
- "bcdata.gov.bc.ca/package-license_title": pkg.license_title || "Unknown",
223
- "bcdata.gov.bc.ca/package-license_url": pkg.license_url,
224
- "bcdata.gov.bc.ca/package-maintainer": pkg.maintainer || "Unknown",
225
- "bcdata.gov.bc.ca/package-maintainer_email": pkg.maintainer_email || "Unknown",
226
- "bcdata.gov.bc.ca/package-metadata_created": pkg.metadata_created,
227
- "bcdata.gov.bc.ca/package-metadata_modified": pkg.metadata_modified,
228
- "bcdata.gov.bc.ca/package-metadata_visibility": pkg.metadata_visibility,
229
- "bcdata.gov.bc.ca/package-name": pkg.name,
230
- "bcdata.gov.bc.ca/package-notes": pkg.notes || "Unknown",
231
- "bcdata.gov.bc.ca/package-owner_org": pkg.owner_org,
232
- "bcdata.gov.bc.ca/package-private": `${pkg.private}`,
233
- "bcdata.gov.bc.ca/package-publish_state": pkg.publish_state,
234
- "bcdata.gov.bc.ca/package-record_create_date": pkg.record_create_date || "Unknown",
235
- "bcdata.gov.bc.ca/package-record_last_modified": pkg.record_last_modified,
236
- "bcdata.gov.bc.ca/package-record_publish_date": pkg.record_publish_date,
237
- "bcdata.gov.bc.ca/package-resource_status": pkg.resource_status,
238
- "bcdata.gov.bc.ca/package-security_class": pkg.security_class,
239
- "bcdata.gov.bc.ca/package-state": pkg.state,
240
- "bcdata.gov.bc.ca/package-title": pkg.title || "Unknown",
241
- "bcdata.gov.bc.ca/package-type": pkg.type,
242
- "bcdata.gov.bc.ca/package-url": pkg.url || "Unknown",
243
- "bcdata.gov.bc.ca/package-version": pkg.version || "Unknown",
244
- "bcdata.gov.bc.ca/package-view_audience": pkg.view_audience
245
- },
246
- links: entityLinks,
247
- tags
248
- }
249
- };
250
- allComponents.set(this.getComponentId(safeName), componentEntity);
251
- for (const apiResource of apiResources) {
252
- const name = apiResource.name;
253
- let resourceSafeName = this.toSafeName(name);
254
- let apiSafeName = resourceSafeName;
255
- if (apiResource.bcdc_type === "webservice" || apiResource.format === "arcgis_rest") {
256
- const prefix = apiResource.format === "openapi-json" ? "api" : apiResource.format;
257
- const formatSafeName = this.toSafeName(prefix);
258
- let baseName = this.toSafeName(`${formatSafeName}-${safeName}`);
259
- let candidateName = baseName;
260
- let candidateApiId = this.getApiId(candidateName);
261
- if (allApis.has(candidateApiId)) {
262
- const distinguishingSuffix = this.extractDistinguishingSuffix(name);
263
- const suffixWithSeparator = `-${distinguishingSuffix}`;
264
- const maxBaseLength = 63 - suffixWithSeparator.length;
265
- let truncatedBase = baseName;
266
- if (baseName.length > maxBaseLength) {
267
- truncatedBase = baseName.slice(0, maxBaseLength).replace(/[-_.]+$/, "");
268
- }
269
- candidateName = this.toSafeName(`${truncatedBase}${suffixWithSeparator}`);
270
- candidateApiId = this.getApiId(candidateName);
271
- let counter = 1;
272
- while (allApis.has(candidateApiId)) {
273
- const counterSuffix = `-${counter}`;
274
- const maxBaseWithCounter = 63 - suffixWithSeparator.length - counterSuffix.length;
275
- let truncatedBaseForCounter = baseName;
276
- if (baseName.length > maxBaseWithCounter) {
277
- truncatedBaseForCounter = baseName.slice(0, maxBaseWithCounter).replace(/[-_.]+$/, "");
278
- }
279
- candidateName = this.toSafeName(`${truncatedBaseForCounter}${suffixWithSeparator}${counterSuffix}`);
280
- candidateApiId = this.getApiId(candidateName);
281
- counter++;
282
- }
283
- }
284
- apiSafeName = candidateName;
285
- }
286
- const definition = apiResource.url;
287
- const url = new URL(definition);
288
- const host = url.host.toLowerCase();
289
- if (!this.allowedHosts.includes(host)) {
290
- this.logger.warn(`API definition host is NOT allowed: "${host}"`);
291
- }
292
- const apiEntityLinks = [];
293
- if (apiResource.url && apiResource.url.length > 0) {
294
- const apiEntityLink = {
295
- url: apiResource.url,
296
- title: apiResource.name,
297
- icon: "api",
298
- type: apiResource.bcdc_type
299
- };
300
- apiEntityLinks.push(apiEntityLink);
301
- }
302
- let definitionContent = apiResource.url;
303
- if (apiResource.format === "openapi-json") {
304
- try {
305
- const response = await this.reader.readUrl(apiResource.url);
306
- const content = (await response.buffer()).toString();
307
- definitionContent = content;
308
- } catch (error) {
309
- this.logger.warn(`Failed to fetch OpenAPI definition from ${apiResource.url}: ${error}`);
310
- definitionContent = apiResource.url;
311
- }
312
- }
313
- const apiEntity = {
314
- apiVersion: "backstage.io/v1alpha1",
315
- kind: "API",
316
- spec: {
317
- type: apiResource.format === "openapi-json" ? "openapi" : apiResource.bcdc_type,
318
- lifecycle: "production",
319
- owner: bcGovGroupId,
320
- definition: definitionContent,
321
- system: systemId
322
- },
323
- metadata: {
324
- name: apiSafeName,
325
- description: apiResource.description || "No description available",
326
- links: apiEntityLinks,
327
- // tags: [this.toSafeName(apiResource.format)],
328
- tags: [this.toSafeName(apiResource.format)],
329
- annotations: {
330
- "backstage.io/managed-by-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search",
331
- "backstage.io/managed-by-origin-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search",
332
- "bcdata.gov.bc.ca/resource-bcdc_type": apiResource.bcdc_type,
333
- "bcdata.gov.bc.ca/resource-cache_last_updated": apiResource.cache_last_updated || "Undefined",
334
- "bcdata.gov.bc.ca/resource-cache_url": apiResource.cache_url || "Undefined",
335
- "bcdata.gov.bc.ca/resource-created": apiResource.created,
336
- "bcdata.gov.bc.ca/resource-datastore_active": `${apiResource.datastore_active}`,
337
- "bcdata.gov.bc.ca/resource-description": apiResource.description || "Undefined",
338
- // 'bcdata.gov.bc.ca/resource-details': apiResource.details,
339
- "bcdata.gov.bc.ca/resource-format": apiResource.format,
340
- // 'bcdata.gov.bc.ca/resource-geographic_extent': apiResource.geographic_extent,
341
- "bcdata.gov.bc.ca/resource-hash": apiResource.hash,
342
- "bcdata.gov.bc.ca/resource-id": apiResource.id,
343
- // 'bcdata.gov.bc.ca/resource-iso_topic_category': apiResource.iso_topic_category || 'Undefined',
344
- "bcdata.gov.bc.ca/resource-metadata_modified": apiResource.metadata_modified,
345
- "bcdata.gov.bc.ca/resource-mimetype": apiResource.mimetype || "Undefined",
346
- "bcdata.gov.bc.ca/resource-name": apiResource.name,
347
- "bcdata.gov.bc.ca/resource-package_id": apiResource.package_id,
348
- "bcdata.gov.bc.ca/resource-position": `${apiResource.position}`,
349
- // 'bcdata.gov.bc.ca/resource-preview_info': apiResource.preview_info,
350
- "bcdata.gov.bc.ca/resource-projection_name": apiResource.projection_name || "Undefined",
351
- "bcdata.gov.bc.ca/resource-resource_access_method": apiResource.resource_access_method,
352
- "bcdata.gov.bc.ca/resource-resource_storage_location": apiResource.resource_storage_location,
353
- "bcdata.gov.bc.ca/resource-resource_type": apiResource.resource_type,
354
- "bcdata.gov.bc.ca/resource-resource_update_cycle": apiResource.resource_update_cycle,
355
- "bcdata.gov.bc.ca/resource-size": `${apiResource.size}`,
356
- "bcdata.gov.bc.ca/resource-spatial_datatype": apiResource.spatial_datatype || "Undefined",
357
- "bcdata.gov.bc.ca/resource-state": apiResource.state,
358
- "bcdata.gov.bc.ca/resource-url": apiResource.url,
359
- "bcdata.gov.bc.ca/resource-url_type": apiResource.url_type || "Undefined"
360
- }
361
- }
362
- };
363
- const apiId = this.getApiId(apiSafeName);
364
- if (allApis.has(apiId)) {
365
- const existingApi = allApis.get(apiId);
366
- this.logger.warn(
367
- `Duplicate API name detected: "${apiSafeName}" (ID: ${apiId}). Existing API: name="${existingApi.metadata.name}", Format: "${existingApi.metadata.annotations?.["bcdata.gov.bc.ca/resource-format"]}", resource-id="${existingApi.metadata.annotations?.["bcdata.gov.bc.ca/resource-id"]}", package-id="${existingApi.metadata.annotations?.["bcdata.gov.bc.ca/resource-package_id"]}", url="${existingApi.metadata.annotations?.["bcdata.gov.bc.ca/resource-url"]}". New API: name="${apiEntity.metadata.name}", resource-id="${apiResource.id}", package-id="${apiResource.package_id}", url="${apiResource.url}". The new API will overwrite the existing one.`
368
- );
369
- }
370
- allApis.set(apiId, apiEntity);
371
- componentEntity.spec.providesApis?.push(apiId);
372
- }
373
- }
374
- const userEntities = Array.from(allUsers.values());
375
- const groupEntities = Array.from(allGroups.values());
376
- const systemEntities = Array.from(allSystems.values());
377
- const componentEntities = Array.from(allComponents.values());
378
- const apiEntities = Array.from(allApis.values());
379
- allEntities.push(...userEntities);
380
- allEntities.push(...groupEntities);
381
- allEntities.push(...systemEntities);
382
- allEntities.push(...componentEntities);
383
- allEntities.push(...apiEntities);
48
+ const packages = await client.getAllPackages();
49
+ const entities = await factory.createEntities(packages);
384
50
  await this.connection.applyMutation({
385
51
  type: "full",
386
- entities: allEntities.map((entity) => ({
52
+ entities: entities.map((entity) => ({
387
53
  entity,
388
54
  locationKey: `bc-data-catalogue:${this.env}`
389
55
  }))
390
56
  });
391
- this.logger.info(">run");
392
- }
393
- getEmailHostname(email) {
394
- const match = email.trim().toLowerCase().match(/@([\w.-]+)/);
395
- return match ? match[1] : void 0;
396
- }
397
- getSystemId(name) {
398
- return `system:default/${this.toSafeName(name)}`;
399
- }
400
- getGroupId(hostName) {
401
- return `group:default/${this.toSafeName(hostName)}`;
402
- }
403
- getUserId(email) {
404
- return `user:default/${email.toLowerCase()}`;
405
- }
406
- getComponentId(name) {
407
- return `component:default/${name.toLowerCase()}`;
408
- }
409
- getApiId(name) {
410
- return `api:default/${name.toLowerCase()}`;
411
- }
412
- createGroupEntity(hostName, displayName, parentGroup) {
413
- const groupEntity = {
414
- apiVersion: "backstage.io/v1alpha1",
415
- kind: "Group",
416
- metadata: {
417
- name: this.toSafeName(hostName),
418
- annotations: {
419
- "backstage.io/managed-by-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search",
420
- "backstage.io/managed-by-origin-location": "url:https://catalogue.data.gov.bc.ca/api/3/action/package_search"
421
- }
422
- },
423
- spec: {
424
- type: "government",
425
- profile: {
426
- displayName
427
- },
428
- parent: parentGroup,
429
- children: [],
430
- members: []
431
- }
432
- };
433
- return groupEntity;
434
- }
435
- /**
436
- * Converts BC Data Catalogue (CKAN package_search) results into Backstage Entities
437
- */
438
- getBcDataCataloguePackages(results) {
439
- this.logger.info("<getBcDataCataloguePackages", {
440
- resultCount: Array.isArray(results) ? results.length : "unknown"
441
- });
442
- const rawPackages = results;
443
- this.logger.info(`rawPackageCount ${rawPackages.length}`);
444
- const validPackages = [];
445
- for (const item of rawPackages) {
446
- const parseResult = BcDataCatalogueModel.BcDataCataloguePackageSchema.safeParse(item);
447
- if (parseResult.success) {
448
- const pkg = parseResult.data;
449
- if (pkg.type === "bcdc_dataset" && pkg.state === "active") {
450
- validPackages.push(pkg);
451
- }
452
- } else {
453
- this.logger.warn(JSON.stringify(item));
454
- this.logger.warn("Invalid package skipped", {
455
- error: parseResult.error.message
456
- });
457
- throw new Error(
458
- `BC Data Catalogue package validation failedSee logs for details. First issue: ${parseResult.error.issues[0]?.message}`
459
- );
460
- }
461
- }
462
- this.logger.info(`>getBcDataCataloguePackages ${validPackages.length}`);
463
- return validPackages;
464
- }
465
- toSafeName(name) {
466
- return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/(?:^[-_.]+|[-_.]+$)/g, "").slice(0, 63).replace(/[-_.]+$/, "");
467
- }
468
- /**
469
- * Extracts distinguishing information from a resource name to help make API names unique.
470
- * Tries to extract meaningful parts like years, version numbers, or other identifiers.
471
- */
472
- extractDistinguishingSuffix(resourceName) {
473
- const safeName = this.toSafeName(resourceName);
474
- const yearMatch = resourceName.match(/\b(19|20)\d{2}\b/);
475
- if (yearMatch) {
476
- return yearMatch[0];
477
- }
478
- const versionMatch = resourceName.match(/\b(?:v|version)[\s_-]?(\d+)\b/i);
479
- if (versionMatch) {
480
- return `v${versionMatch[1]}`;
481
- }
482
- const trailingNumberMatch = resourceName.match(/\b(\d{2,})\b/);
483
- if (trailingNumberMatch) {
484
- return trailingNumberMatch[0];
485
- }
486
- const words = safeName.split("-").filter(
487
- (w) => w.length > 0 && !["service", "request", "getcapabilities", "wms", "kml", "arcgis", "rest", "online"].includes(w)
488
- );
489
- if (words.length >= 2) {
490
- return words.slice(-2).join("-");
491
- } else if (words.length === 1) {
492
- return words[0];
493
- } else {
494
- return safeName.split("-").slice(-1)[0].slice(0, 10);
495
- }
57
+ this.logger.info("[BCDC Entity Provider] >run");
496
58
  }
497
59
  }
498
60