@backstage/plugin-catalog-backend 3.0.2-next.1 → 3.1.1-next.0

package/CHANGELOG.md

@@ -1,5 +1,102 @@
 # @backstage/plugin-catalog-backend
 
+## 3.1.1-next.0
+
+### Patch Changes
+
+- 9890488: Internal refactor to remove remnants of the old backend system
+- 2aaf01a: Fix for duplicate search results in entity-facets API call
+- e489661: Moved catalog processor and provider disabling and priorities under own config objects.
+
+  This is due to issue with some existing providers, such as GitHub, using array syntax for the provider configuration.
+
+  The new config format is not backwards compatible, so users will need to update their config files. The new format
+  is as follows:
+
+  ```yaml
+  catalog:
+    providerOptions:
+      providerA:
+        disabled: false
+      providerB:
+        disabled: true
+    processorOptions:
+      processorA:
+        disabled: false
+        priority: 10
+      processorB:
+        disabled: true
+  ```
+
+- Updated dependencies
+  - @backstage/integration@1.18.1-next.0
+  - @backstage/plugin-permission-node@0.10.4
+  - @backstage/backend-openapi-utils@0.6.1
+  - @backstage/backend-plugin-api@1.4.3
+  - @backstage/catalog-client@1.12.0
+  - @backstage/catalog-model@1.7.5
+  - @backstage/config@1.3.3
+  - @backstage/errors@1.2.7
+  - @backstage/types@1.2.2
+  - @backstage/plugin-catalog-common@1.1.5
+  - @backstage/plugin-catalog-node@1.19.0
+  - @backstage/plugin-events-node@0.4.15
+  - @backstage/plugin-permission-common@0.9.1
+
+## 3.1.0
+
+### Minor Changes
+
+- 9b40a55: Add support for specifying an entity `spec.type` in `catalog.rules` and `catalog.locations.rules` within the catalog configuration.
+
+  For example, this enables allowing all `Template` entities with the type `website`:
+
+  ```diff
+    catalog:
+      rules:
+        - allow:
+            - Component
+            - API
+            - Resource
+            - System
+            - Domain
+            - Location
+  +     - allow:
+  +         - kind: Template
+  +           spec.type: website
+          locations:
+            - type: url
+              pattern: https://github.com/org/*\/blob/master/*.yaml
+  ```
+
+### Patch Changes
+
+- 37b4eaf: The 'get-catalog-entity' action now throws a ConflictError instead of generic Error if multiple entities are found, so MCP call doesn't fail with 500.
+- 2bbd24f: Order catalog processors by priority.
+
+  This change enables the ordering of catalog processors by their priority,
+  allowing for more control over the catalog processing sequence.
+  The default priority is set to 20, and processors can be assigned a custom
+  priority to influence their execution order. Lower number indicates higher priority.
+  The priority can be set by implementing the `getPriority` method in the processor class
+  or by adding a `catalog.processors.<processorName>.priority` configuration
+  in the `app-config.yaml` file. The configuration takes precedence over the method.
+
+- e934a27: Updating `catalog:get-catalog-entity` action to be `readOnly` and non destructive
+- 0efcc97: Updated generated schemas
+- 2204f5b: Prevent deadlock in catalog deferred stitching
+- 58874c4: Add support to disable catalog providers and processors via configuration
+- a4c82ad: Only run provider orphan cleanup if the engine is started in the first place
+- Updated dependencies
+  - @backstage/plugin-catalog-node@1.19.0
+  - @backstage/catalog-client@1.12.0
+  - @backstage/plugin-events-node@0.4.15
+  - @backstage/integration@1.18.0
+  - @backstage/types@1.2.2
+  - @backstage/backend-openapi-utils@0.6.1
+  - @backstage/backend-plugin-api@1.4.3
+  - @backstage/plugin-permission-node@0.10.4
+
 ## 3.0.2-next.1
 
 ### Patch Changes

package/config.d.ts

@@ -38,8 +38,11 @@ export interface Config {
        * Allow entities of these particular kinds.
        *
        * E.g. ["Component", "API", "Template", "Location"]
+       *
+       * You can also specify the type of the entity by using an object with `kind` and optional `spec.type` properties.
+       * E.g. [{ kind: "Component", 'spec.type': "service" }]
        */
-      allow: Array<string>;
+      allow: Array<string | { kind: string; 'spec.type'?: string }>;
       /**
        * Limit this rule to a specific location
        *
@@ -218,5 +221,41 @@ export interface Config {
      * housing catalog-info files.
      */
     processingInterval?: HumanDuration | false;
+
+    /**
+     * Provider-specific additional configuration options.
+     */
+    providerOptions?: {
+      /**
+       * Key is the provider name, value is an object with additional configuration
+       */
+      [name: string]: {
+        /**
+         * Determines whether this provider is disabled or not. If not specified,
+         * defaults to false.
+         */
+        disabled?: boolean;
+      };
+    };
+
+    /**
+     * Processor-specific additional configuration options.
+     */
+    processorOptions?: {
+      /**
+       * Key is the processor name, value is an object with additional configuration
+       */
+      [name: string]: {
+        /**
+         * Determines whether this processor is disabled or not. If not specified,
+         * defaults to false.
+         */
+        disabled?: boolean;
+        /**
+         * The default priority is 20, and lower value means that the processor runs earlier.
+         */
+        priority?: number;
+      };
+    };
   };
 }

package/dist/actions/createGetCatalogEntityAction.cjs.js

@@ -10,6 +10,11 @@ const createGetCatalogEntityAction = ({
   actionsRegistry.register({
     name: "get-catalog-entity",
     title: "Get Catalog Entity",
+    attributes: {
+      destructive: false,
+      readOnly: true,
+      idempotent: true
+    },
     description: `
 This allows you to get a single entity from the software catalog.
 Each entity in the software catalog has a unique name, kind, and namespace. The default namespace is "default".

package/dist/actions/createGetCatalogEntityAction.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"createGetCatalogEntityAction.cjs.js","sources":["../../src/actions/createGetCatalogEntityAction.ts"],"sourcesContent":["/*\n * Copyright 2025 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nimport { ActionsRegistryService } from '@backstage/backend-plugin-api/alpha';\nimport { stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError, InputError } from '@backstage/errors';\nimport { CatalogService } from '@backstage/plugin-catalog-node';\n\nexport const createGetCatalogEntityAction = ({\n  catalog,\n  actionsRegistry,\n}: {\n  catalog: CatalogService;\n  actionsRegistry: ActionsRegistryService;\n}) => {\n  actionsRegistry.register({\n    name: 'get-catalog-entity',\n    title: 'Get Catalog Entity',\n    description: `\nThis allows you to get a single entity from the software catalog.\nEach entity in the software catalog has a unique name, kind, and namespace. The default namespace is \"default\".\nEach entity is identified by a unique entity reference, which is a string of the form \"kind:namespace/name\".\n    `,\n    schema: {\n      input: z =>\n        z.object({\n          kind: z\n            .string()\n            .describe(\n              'The kind of the entity to query. If the kind is unknown it can be omitted.',\n            )\n            .optional(),\n          namespace: z\n            .string()\n            .describe(\n              'The namespace of the entity to query. If the namespace is unknown it can be omitted.',\n            )\n            .optional(),\n          name: z.string().describe('The name of the entity to query'),\n        }),\n      // TODO: is there a better way to do this?\n      output: z => z.object({}).passthrough(),\n    },\n    action: async ({ input, credentials }) => {\n      const filter: Record<string, string> = { 'metadata.name': input.name };\n\n      if (input.kind) {\n        filter.kind = input.kind;\n      }\n\n      if (input.namespace) {\n        filter['metadata.namespace'] = input.namespace;\n      }\n\n      const { items } = await catalog.queryEntities(\n        { filter },\n        {\n          credentials,\n        },\n      );\n\n      if (items.length === 0) {\n        throw new InputError(`No entity found with name \"${input.name}\"`);\n      }\n\n      if (items.length > 1) {\n        throw new ConflictError(\n          `Multiple entities found with name \"${\n            input.name\n          }\", please provide more specific filters. Entities found: ${items\n            .map(item => `\"${stringifyEntityRef(item)}\"`)\n            .join(', ')}`,\n        );\n      }\n\n      const [entity] = items;\n\n      return {\n        output: entity,\n      };\n    },\n  });\n};\n"],"names":["InputError","ConflictError","stringifyEntityRef"],"mappings":";;;;;AAoBO,MAAM,+BAA+B,CAAC;AAAA,EAC3C,OAAA;AAAA,EACA;AACF,CAAA,KAGM;AACJ,EAAA,eAAA,CAAgB,QAAA,CAAS;AAAA,IACvB,IAAA,EAAM,oBAAA;AAAA,IACN,KAAA,EAAO,oBAAA;AAAA,IACP,WAAA,EAAa;AAAA;AAAA;AAAA;AAAA,IAAA,CAAA;AAAA,IAKb,MAAA,EAAQ;AAAA,MACN,KAAA,EAAO,CAAA,CAAA,KACL,CAAA,CAAE,MAAA,CAAO;AAAA,QACP,IAAA,EAAM,CAAA,CACH,MAAA,EAAO,CACP,QAAA;AAAA,UACC;AAAA,UAED,QAAA,EAAS;AAAA,QACZ,SAAA,EAAW,CAAA,CACR,MAAA,EAAO,CACP,QAAA;AAAA,UACC;AAAA,UAED,QAAA,EAAS;AAAA,QACZ,IAAA,EAAM,CAAA,CAAE,MAAA,EAAO,CAAE,SAAS,iCAAiC;AAAA,OAC5D,CAAA;AAAA;AAAA,MAEH,QAAQ,CAAA,CAAA,KAAK,CAAA,CAAE,OAAO,EAAE,EAAE,WAAA;AAAY,KACxC;AAAA,IACA,MAAA,EAAQ,OAAO,EAAE,KAAA,EAAO,aAAY,KAAM;AACxC,MAAA,MAAM,MAAA,GAAiC,EAAE,eAAA,EAAiB,KAAA,CAAM,IAAA,EAAK;AAErE,MAAA,IAAI,MAAM,IAAA,EAAM;AACd,QAAA,MAAA,CAAO,OAAO,KAAA,CAAM,IAAA;AAAA,MACtB;AAEA,MAAA,IAAI,MAAM,SAAA,EAAW;AACnB,QAAA,MAAA,CAAO,oBAAoB,IAAI,KAAA,CAAM,SAAA;AAAA,MACvC;AAEA,MAAA,MAAM,EAAE,KAAA,EAAM,GAAI,MAAM,OAAA,CAAQ,aAAA;AAAA,QAC9B,EAAE,MAAA,EAAO;AAAA,QACT;AAAA,UACE;AAAA;AACF,OACF;AAEA,MAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AACtB,QAAA,MAAM,IAAIA,iBAAA,CAAW,CAAA,2BAAA,EAA8B,KAAA,CAAM,IAAI,CAAA,CAAA,CAAG,CAAA;AAAA,MAClE;AAEA,MAAA,IAAI,KAAA,CAAM,SAAS,CAAA,EAAG;AACpB,QAAA,MAAM,IAAIC,oBAAA;AAAA,UACR,CAAA,mCAAA,EACE,KAAA,CAAM,IACR,CAAA,yDAAA,EAA4D,MACzD,GAAA,CAAI,CAAA,IAAA,KAAQ,CAAA,CAAA,EAAIC,+BAAA,CAAmB,IAAI,CAAC,CAAA,CAAA,CAAG,CAAA,CAC3C,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,SACf;AAAA,MACF;AAEA,MAAA,MAAM,CAAC,MAAM,CAAA,GAAI,KAAA;AAEjB,MAAA,OAAO;AAAA,QACL,MAAA,EAAQ;AAAA,OACV;AAAA,IACF;AAAA,GACD,CAAA;AACH;;;;"}
+{"version":3,"file":"createGetCatalogEntityAction.cjs.js","sources":["../../src/actions/createGetCatalogEntityAction.ts"],"sourcesContent":["/*\n * Copyright 2025 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nimport { ActionsRegistryService } from '@backstage/backend-plugin-api/alpha';\nimport { stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError, InputError } from '@backstage/errors';\nimport { CatalogService } from '@backstage/plugin-catalog-node';\n\nexport const createGetCatalogEntityAction = ({\n  catalog,\n  actionsRegistry,\n}: {\n  catalog: CatalogService;\n  actionsRegistry: ActionsRegistryService;\n}) => {\n  actionsRegistry.register({\n    name: 'get-catalog-entity',\n    title: 'Get Catalog Entity',\n    attributes: {\n      destructive: false,\n      readOnly: true,\n      idempotent: true,\n    },\n    description: `\nThis allows you to get a single entity from the software catalog.\nEach entity in the software catalog has a unique name, kind, and namespace. The default namespace is \"default\".\nEach entity is identified by a unique entity reference, which is a string of the form \"kind:namespace/name\".\n    `,\n    schema: {\n      input: z =>\n        z.object({\n          kind: z\n            .string()\n            .describe(\n              'The kind of the entity to query. If the kind is unknown it can be omitted.',\n            )\n            .optional(),\n          namespace: z\n            .string()\n            .describe(\n              'The namespace of the entity to query. If the namespace is unknown it can be omitted.',\n            )\n            .optional(),\n          name: z.string().describe('The name of the entity to query'),\n        }),\n      // TODO: is there a better way to do this?\n      output: z => z.object({}).passthrough(),\n    },\n    action: async ({ input, credentials }) => {\n      const filter: Record<string, string> = { 'metadata.name': input.name };\n\n      if (input.kind) {\n        filter.kind = input.kind;\n      }\n\n      if (input.namespace) {\n        filter['metadata.namespace'] = input.namespace;\n      }\n\n      const { items } = await catalog.queryEntities(\n        { filter },\n        {\n          credentials,\n        },\n      );\n\n      if (items.length === 0) {\n        throw new InputError(`No entity found with name \"${input.name}\"`);\n      }\n\n      if (items.length > 1) {\n        throw new ConflictError(\n          `Multiple entities found with name \"${\n            input.name\n          }\", please provide more specific filters. Entities found: ${items\n            .map(item => `\"${stringifyEntityRef(item)}\"`)\n            .join(', ')}`,\n        );\n      }\n\n      const [entity] = items;\n\n      return {\n        output: entity,\n      };\n    },\n  });\n};\n"],"names":["InputError","ConflictError","stringifyEntityRef"],"mappings":";;;;;AAoBO,MAAM,+BAA+B,CAAC;AAAA,EAC3C,OAAA;AAAA,EACA;AACF,CAAA,KAGM;AACJ,EAAA,eAAA,CAAgB,QAAA,CAAS;AAAA,IACvB,IAAA,EAAM,oBAAA;AAAA,IACN,KAAA,EAAO,oBAAA;AAAA,IACP,UAAA,EAAY;AAAA,MACV,WAAA,EAAa,KAAA;AAAA,MACb,QAAA,EAAU,IAAA;AAAA,MACV,UAAA,EAAY;AAAA,KACd;AAAA,IACA,WAAA,EAAa;AAAA;AAAA;AAAA;AAAA,IAAA,CAAA;AAAA,IAKb,MAAA,EAAQ;AAAA,MACN,KAAA,EAAO,CAAA,CAAA,KACL,CAAA,CAAE,MAAA,CAAO;AAAA,QACP,IAAA,EAAM,CAAA,CACH,MAAA,EAAO,CACP,QAAA;AAAA,UACC;AAAA,UAED,QAAA,EAAS;AAAA,QACZ,SAAA,EAAW,CAAA,CACR,MAAA,EAAO,CACP,QAAA;AAAA,UACC;AAAA,UAED,QAAA,EAAS;AAAA,QACZ,IAAA,EAAM,CAAA,CAAE,MAAA,EAAO,CAAE,SAAS,iCAAiC;AAAA,OAC5D,CAAA;AAAA;AAAA,MAEH,QAAQ,CAAA,CAAA,KAAK,CAAA,CAAE,OAAO,EAAE,EAAE,WAAA;AAAY,KACxC;AAAA,IACA,MAAA,EAAQ,OAAO,EAAE,KAAA,EAAO,aAAY,KAAM;AACxC,MAAA,MAAM,MAAA,GAAiC,EAAE,eAAA,EAAiB,KAAA,CAAM,IAAA,EAAK;AAErE,MAAA,IAAI,MAAM,IAAA,EAAM;AACd,QAAA,MAAA,CAAO,OAAO,KAAA,CAAM,IAAA;AAAA,MACtB;AAEA,MAAA,IAAI,MAAM,SAAA,EAAW;AACnB,QAAA,MAAA,CAAO,oBAAoB,IAAI,KAAA,CAAM,SAAA;AAAA,MACvC;AAEA,MAAA,MAAM,EAAE,KAAA,EAAM,GAAI,MAAM,OAAA,CAAQ,aAAA;AAAA,QAC9B,EAAE,MAAA,EAAO;AAAA,QACT;AAAA,UACE;AAAA;AACF,OACF;AAEA,MAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AACtB,QAAA,MAAM,IAAIA,iBAAA,CAAW,CAAA,2BAAA,EAA8B,KAAA,CAAM,IAAI,CAAA,CAAA,CAAG,CAAA;AAAA,MAClE;AAEA,MAAA,IAAI,KAAA,CAAM,SAAS,CAAA,EAAG;AACpB,QAAA,MAAM,IAAIC,oBAAA;AAAA,UACR,CAAA,mCAAA,EACE,KAAA,CAAM,IACR,CAAA,yDAAA,EAA4D,MACzD,GAAA,CAAI,CAAA,IAAA,KAAQ,CAAA,CAAA,EAAIC,+BAAA,CAAmB,IAAI,CAAC,CAAA,CAAA,CAAG,CAAA,CAC3C,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,SACf;AAAA,MACF;AAEA,MAAA,MAAM,CAAC,MAAM,CAAA,GAAI,KAAA;AAEjB,MAAA,OAAO;AAAA,QACL,MAAA,EAAQ;AAAA,OACV;AAAA,IACF;AAAA,GACD,CAAA;AACH;;;;"}

package/dist/database/DefaultProcessingDatabase.cjs.js

@@ -222,7 +222,7 @@ class DefaultProcessingDatabase {
         this.options.logger.warn(
           `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`
         );
-        if (this.options.eventBroker && locationKey) {
+        if (locationKey) {
           const eventParams = {
             topic: constants.CATALOG_CONFLICTS_TOPIC,
             eventPayload: {
@@ -233,7 +233,7 @@ class DefaultProcessingDatabase {
               lastConflictAt: luxon.DateTime.now().toISO()
             }
           };
-          await this.options.eventBroker?.publish(eventParams);
+          await this.options.events.publish(eventParams);
         }
       }
     }

package/dist/database/DefaultProcessingDatabase.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultProcessingDatabase.cjs.js","sources":["../../src/database/DefaultProcessingDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError } from '@backstage/errors';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { ProcessingIntervalFunction } from '../processing/refresh';\nimport { rethrowError, timestampToDateTime } from './conversion';\nimport { initDatabaseMetrics } from './metrics';\nimport {\n  DbRefreshKeysRow,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n} from './tables';\nimport {\n  GetProcessableEntitiesResult,\n  ListParentsOptions,\n  ListParentsResult,\n  ProcessingDatabase,\n  RefreshStateItem,\n  Transaction,\n  UpdateEntityCacheOptions,\n  UpdateProcessedEntityOptions,\n} from './types';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { generateStableHash, generateTargetKey } from './util';\nimport {\n  EventBroker,\n  EventParams,\n  EventsService,\n} from '@backstage/plugin-events-node';\nimport { DateTime } from 'luxon';\nimport { CATALOG_CONFLICTS_TOPIC } from '../constants';\nimport { CatalogConflictEventPayload } from '../catalog/types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProcessingDatabase implements ProcessingDatabase {\n  constructor(\n    private readonly options: {\n      database: Knex;\n      logger: LoggerService;\n      refreshInterval: ProcessingIntervalFunction;\n      eventBroker?: EventBroker | EventsService;\n    },\n  ) {\n    initDatabaseMetrics(options.database);\n  }\n\n  async updateProcessedEntity(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<{ previous: { relations: DbRelationsRow[] } }> {\n    const tx = txOpaque as Knex.Transaction;\n    const {\n      id,\n      processedEntity,\n      resultHash,\n      errors,\n      relations,\n      deferredEntities,\n      refreshKeys,\n      locationKey,\n    } = options;\n    const configClient = tx.client.config.client;\n    const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        processed_entity: JSON.stringify(processedEntity),\n        result_hash: resultHash,\n        errors,\n        location_key: locationKey,\n      })\n      .where('entity_id', id)\n      .andWhere(inner => {\n        if (!locationKey) {\n          return inner.whereNull('location_key');\n        }\n        return inner\n          .where('location_key', locationKey)\n          .orWhereNull('location_key');\n      });\n    if (refreshResult === 0) {\n      throw new ConflictError(\n        `Conflicting write of processing result for ${id} with location key '${locationKey}'`,\n      );\n    }\n    const sourceEntityRef = stringifyEntityRef(processedEntity);\n\n    // Schedule all deferred entities for future processing.\n    await this.addUnprocessedEntities(tx, {\n      entities: deferredEntities,\n      sourceEntityRef,\n    });\n\n    // Delete old relations\n    // NOTE(freben): knex implemented support for returning() on update queries for sqlite, but at the current time of writing (Sep 2022) not for delete() queries.\n    let previousRelationRows: DbRelationsRow[];\n    if (configClient.includes('sqlite3') || configClient.includes('mysql')) {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .select('*')\n        .where({ originating_entity_id: id });\n      await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete();\n    } else {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete()\n        .returning('*');\n    }\n\n    // Batch insert new relations\n    const relationRows: DbRelationsRow[] = relations.map(\n      ({ source, target, type }) => ({\n        originating_entity_id: id,\n        source_entity_ref: stringifyEntityRef(source),\n        target_entity_ref: stringifyEntityRef(target),\n        type,\n      }),\n    );\n\n    await tx.batchInsert(\n      'relations',\n      this.deduplicateRelations(relationRows),\n      BATCH_SIZE,\n    );\n\n    // Delete old refresh keys\n    await tx<DbRefreshKeysRow>('refresh_keys')\n      .where({ entity_id: id })\n      .delete();\n\n    // Insert the refresh keys for the processed entity\n    await tx.batchInsert(\n      'refresh_keys',\n      refreshKeys.map(k => ({\n        entity_id: id,\n        key: generateTargetKey(k.key),\n      })),\n      BATCH_SIZE,\n    );\n\n    return {\n      previous: {\n        relations: previousRelationRows,\n      },\n    };\n  }\n\n  async updateProcessedEntityErrors(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, errors, resultHash } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        errors,\n        result_hash: resultHash,\n      })\n      .where('entity_id', id);\n  }\n\n  async updateEntityCache(\n    txOpaque: Transaction,\n    options: UpdateEntityCacheOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, state } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({ cache: JSON.stringify(state ?? {}) })\n      .where('entity_id', id);\n  }\n\n  async getProcessableEntities(\n    maybeTx: Transaction | Knex,\n    request: { processBatchSize: number },\n  ): Promise<GetProcessableEntitiesResult> {\n    const knex = maybeTx as Knex.Transaction | Knex;\n\n    let itemsQuery = knex<DbRefreshStateRow>('refresh_state').select([\n      'entity_id',\n      'entity_ref',\n      'unprocessed_entity',\n      'result_hash',\n      'cache',\n      'errors',\n      'location_key',\n      'next_update_at',\n    ]);\n\n    // This avoids duplication of work because of race conditions and is\n    // also fast because locked rows are ignored rather than blocking.\n    // It's only available in MySQL and PostgreSQL\n    if (['mysql', 'mysql2', 'pg'].includes(knex.client.config.client)) {\n      itemsQuery = itemsQuery.forUpdate().skipLocked();\n    }\n\n    const items = await itemsQuery\n      .where('next_update_at', '<=', knex.fn.now())\n      .limit(request.processBatchSize)\n      .orderBy('next_update_at', 'asc');\n\n    const interval = this.options.refreshInterval();\n\n    const nextUpdateAt = (refreshInterval: number) => {\n      if (knex.client.config.client.includes('sqlite3')) {\n        return knex.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);\n      } else if (knex.client.config.client.includes('mysql')) {\n        return knex.raw(`now() + interval ${refreshInterval} second`);\n      }\n      return knex.raw(`now() + interval '${refreshInterval} seconds'`);\n    };\n\n    await knex<DbRefreshStateRow>('refresh_state')\n      .whereIn(\n        'entity_ref',\n        items.map(i => i.entity_ref),\n      )\n      .update({\n        next_update_at: nextUpdateAt(interval),\n      });\n\n    return {\n      items: items.map(\n        i =>\n          ({\n            id: i.entity_id,\n            entityRef: i.entity_ref,\n            unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity,\n            resultHash: i.result_hash || '',\n            nextUpdateAt: timestampToDateTime(i.next_update_at),\n            state: i.cache ? JSON.parse(i.cache) : undefined,\n            errors: i.errors,\n            locationKey: i.location_key,\n          } satisfies RefreshStateItem),\n      ),\n    };\n  }\n\n  async listParents(\n    txOpaque: Transaction,\n    options: ListParentsOptions,\n  ): Promise<ListParentsResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .whereIn('target_entity_ref', options.entityRefs)\n      .select();\n\n    const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);\n\n    return { entityRefs };\n  }\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] {\n    return lodash.uniqBy(\n      rows,\n      r => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`,\n    );\n  }\n\n  /**\n   * Add a set of deferred entities for processing.\n   * The entities will be added at the front of the processing queue.\n   */\n  private async addUnprocessedEntities(\n    txOpaque: Transaction,\n    options: {\n      sourceEntityRef: string;\n      entities: DeferredEntity[];\n    },\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n\n    // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards\n    const stateReferences = new Array<string>();\n\n    // Upsert all of the unprocessed entities into the refresh_state table, by\n    // their entity ref.\n    for (const { entity, locationKey } of options.entities) {\n      const entityRef = stringifyEntityRef(entity);\n      const hash = generateStableHash(entity);\n\n      const updated = await updateUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n      });\n      if (updated) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      const inserted = await insertUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n        logger: this.options.logger,\n      });\n      if (inserted) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      // If the row can't be inserted, we have a conflict, but it could be either\n      // because of a conflicting locationKey or a race with another instance, so check\n      // whether the conflicting entity has the same entityRef but a different locationKey\n      const conflictingKey = await checkLocationKeyConflict({\n        tx,\n        entityRef,\n        locationKey,\n      });\n      if (conflictingKey) {\n        this.options.logger.warn(\n          `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n        );\n        if (this.options.eventBroker && locationKey) {\n          const eventParams: EventParams<CatalogConflictEventPayload> = {\n            topic: CATALOG_CONFLICTS_TOPIC,\n            eventPayload: {\n              unprocessedEntity: entity,\n              entityRef,\n              newLocationKey: locationKey,\n              existingLocationKey: conflictingKey,\n              lastConflictAt: DateTime.now().toISO()!,\n            },\n          };\n          await this.options.eventBroker?.publish(eventParams);\n        }\n      }\n    }\n\n    // Lastly, replace refresh state references for the originating entity and any successfully added entities\n    await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n      // Remove all existing references from the originating entity\n      .where({ source_entity_ref: options.sourceEntityRef })\n      // And remove any existing references to entities that we're inserting new references for\n      .orWhereIn('target_entity_ref', stateReferences)\n      .delete();\n    await tx.batchInsert(\n      'refresh_state_references',\n      stateReferences.map(entityRef => ({\n        source_entity_ref: options.sourceEntityRef,\n        target_entity_ref: entityRef,\n      })),\n      BATCH_SIZE,\n    );\n  }\n}\n"],"names":["initDatabaseMetrics","errors","ConflictError","stringifyEntityRef","generateTargetKey","timestampToDateTime","rethrowError","lodash","generateStableHash","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","CATALOG_CONFLICTS_TOPIC","DateTime"],"mappings":";;;;;;;;;;;;;;;;;;AA0DA,MAAM,UAAA,GAAa,EAAA;AAEZ,MAAM,yBAAA,CAAwD;AAAA,EACnE,YACmB,OAAA,EAMjB;AANiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAOjB,IAAAA,2BAAA,CAAoB,QAAQ,QAAQ,CAAA;AAAA,EACtC;AAAA,EAEA,MAAM,qBAAA,CACJ,QAAA,EACA,OAAA,EACwD;AACxD,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM;AAAA,MACJ,EAAA;AAAA,MACA,eAAA;AAAA,MACA,UAAA;AAAA,cACAC,QAAA;AAAA,MACA,SAAA;AAAA,MACA,gBAAA;AAAA,MACA,WAAA;AAAA,MACA;AAAA,KACF,GAAI,OAAA;AACJ,IAAA,MAAM,YAAA,GAAe,EAAA,CAAG,MAAA,CAAO,MAAA,CAAO,MAAA;AACtC,IAAA,MAAM,aAAA,GAAgB,MAAM,EAAA,CAAsB,eAAe,EAC9D,MAAA,CAAO;AAAA,MACN,gBAAA,EAAkB,IAAA,CAAK,SAAA,CAAU,eAAe,CAAA;AAAA,MAChD,WAAA,EAAa,UAAA;AAAA,cACbA,QAAA;AAAA,MACA,YAAA,EAAc;AAAA,KACf,CAAA,CACA,KAAA,CAAM,aAAa,EAAE,CAAA,CACrB,SAAS,CAAA,KAAA,KAAS;AACjB,MAAA,IAAI,CAAC,WAAA,EAAa;AAChB,QAAA,OAAO,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA,MACvC;AACA,MAAA,OAAO,MACJ,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,IAC/B,CAAC,CAAA;AACH,IAAA,IAAI,kBAAkB,CAAA,EAAG;AACvB,MAAA,MAAM,IAAIC,oBAAA;AAAA,QACR,CAAA,2CAAA,EAA8C,EAAE,CAAA,oBAAA,EAAuB,WAAW,CAAA,CAAA;AAAA,OACpF;AAAA,IACF;AACA,IAAA,MAAM,eAAA,GAAkBC,gCAAmB,eAAe,CAAA;AAG1D,IAAA,MAAM,IAAA,CAAK,uBAAuB,EAAA,EAAI;AAAA,MACpC,QAAA,EAAU,gBAAA;AAAA,MACV;AAAA,KACD,CAAA;AAID,IAAA,IAAI,oBAAA;AACJ,IAAA,IAAI,aAAa,QAAA,CAAS,SAAS,KAAK,YAAA,CAAa,QAAA,CAAS,OAAO,CAAA,EAAG;AACtE,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,MAAA,CAAO,GAAG,CAAA,CACV,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA;AACtC,MAAA,MAAM,EAAA,CAAmB,WAAW,CAAA,CACjC,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO;AAAA,IACZ,CAAA,MAAO;AACL,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO,CACP,UAAU,GAAG,CAAA;AAAA,IAClB;AAGA,IAAA,MAAM,eAAiC,SAAA,CAAU,GAAA;AAAA,MAC/C,CAAC,EAAE,MAAA,EAAQ,MAAA,EAAQ,MAAK,MAAO;AAAA,QAC7B,qBAAA,EAAuB,EAAA;AAAA,QACvB,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C;AAAA,OACF;AAAA,KACF;AAEA,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,WAAA;AAAA,MACA,IAAA,CAAK,qBAAqB,YAAY,CAAA;AAAA,MACtC;AAAA,KACF;AAGA,IAAA,MAAM,EAAA,CAAqB,cAAc,CAAA,CACtC,KAAA,CAAM,EAAE,SAAA,EAAW,EAAA,EAAI,CAAA,CACvB,MAAA,EAAO;AAGV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,cAAA;AAAA,MACA,WAAA,CAAY,IAAI,CAAA,CAAA,MAAM;AAAA,QACpB,SAAA,EAAW,EAAA;AAAA,QACX,GAAA,EAAKC,sBAAA,CAAkB,CAAA,CAAE,GAAG;AAAA,OAC9B,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAEA,IAAA,OAAO;AAAA,MACL,QAAA,EAAU;AAAA,QACR,SAAA,EAAW;AAAA;AACb,KACF;AAAA,EACF;AAAA,EAEA,MAAM,2BAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,MAAA,EAAQ,UAAA,EAAW,GAAI,OAAA;AAEnC,IAAA,MAAM,EAAA,CAAsB,eAAe,CAAA,CACxC,MAAA,CAAO;AAAA,MACN,MAAA;AAAA,MACA,WAAA,EAAa;AAAA,KACd,CAAA,CACA,KAAA,CAAM,WAAA,EAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,iBAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,KAAA,EAAM,GAAI,OAAA;AAEtB,IAAA,MAAM,GAAsB,eAAe,CAAA,CACxC,MAAA,CAAO,EAAE,OAAO,IAAA,CAAK,SAAA,CAAU,KAAA,IAAS,EAAE,CAAA,EAAG,CAAA,CAC7C,KAAA,CAAM,aAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,sBAAA,CACJ,OAAA,EACA,OAAA,EACuC;AACvC,IAAA,MAAM,IAAA,GAAO,OAAA;AAEb,IAAA,IAAI,UAAA,GAAa,IAAA,CAAwB,eAAe,CAAA,CAAE,MAAA,CAAO;AAAA,MAC/D,WAAA;AAAA,MACA,YAAA;AAAA,MACA,oBAAA;AAAA,MACA,aAAA;AAAA,MACA,OAAA;AAAA,MACA,QAAA;AAAA,MACA,cAAA;AAAA,MACA;AAAA,KACD,CAAA;AAKD,IAAA,IAAI,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,SAAS,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,MAAM,CAAA,EAAG;AACjE,MAAA,UAAA,GAAa,UAAA,CAAW,SAAA,EAAU,CAAE,UAAA,EAAW;AAAA,IACjD;AAEA,IAAA,MAAM,QAAQ,MAAM,UAAA,CACjB,KAAA,CAAM,gBAAA,EAAkB,MAAM,IAAA,CAAK,EAAA,CAAG,GAAA,EAAK,EAC3C,KAAA,CAAM,OAAA,CAAQ,gBAAgB,CAAA,CAC9B,OAAA,CAAQ,kBAAkB,KAAK,CAAA;AAElC,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,OAAA,CAAQ,eAAA,EAAgB;AAE9C,IAAA,MAAM,YAAA,GAAe,CAAC,eAAA,KAA4B;AAChD,MAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AACjD,QAAA,OAAO,KAAK,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,eAAe,UAAU,CAAC,CAAA;AAAA,MACtE,WAAW,IAAA,CAAK,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACtD,QAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,iBAAA,EAAoB,eAAe,CAAA,OAAA,CAAS,CAAA;AAAA,MAC9D;AACA,MAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,kBAAA,EAAqB,eAAe,CAAA,SAAA,CAAW,CAAA;AAAA,IACjE,CAAA;AAEA,IAAA,MAAM,IAAA,CAAwB,eAAe,CAAA,CAC1C,OAAA;AAAA,MACC,YAAA;AAAA,MACA,KAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,MAE5B,MAAA,CAAO;AAAA,MACN,cAAA,EAAgB,aAAa,QAAQ;AAAA,KACtC,CAAA;AAEH,IAAA,OAAO;AAAA,MACL,OAAO,KAAA,CAAM,GAAA;AAAA,QACX,CAAA,CAAA,MACG;AAAA,UACC,IAAI,CAAA,CAAE,SAAA;AAAA,UACN,WAAW,CAAA,CAAE,UAAA;AAAA,UACb,iBAAA,EAAmB,IAAA,CAAK,KAAA,CAAM,CAAA,CAAE,kBAAkB,CAAA;AAAA,UAClD,UAAA,EAAY,EAAE,WAAA,IAAe,EAAA;AAAA,UAC7B,YAAA,EAAcC,8BAAA,CAAoB,CAAA,CAAE,cAAc,CAAA;AAAA,UAClD,OAAO,CAAA,CAAE,KAAA,GAAQ,KAAK,KAAA,CAAM,CAAA,CAAE,KAAK,CAAA,GAAI,MAAA;AAAA,UACvC,QAAQ,CAAA,CAAE,MAAA;AAAA,UACV,aAAa,CAAA,CAAE;AAAA,SACjB;AAAA;AACJ,KACF;AAAA,EACF;AAAA,EAEA,MAAM,WAAA,CACJ,QAAA,EACA,OAAA,EAC4B;AAC5B,IAAA,MAAM,EAAA,GAAK,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,MAEC,OAAA,CAAQ,mBAAA,EAAqB,OAAA,CAAQ,UAAU,EAC/C,MAAA,EAAO;AAEV,IAAA,MAAM,UAAA,GAAa,KAAK,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,iBAAkB,CAAA,CAAE,OAAO,OAAO,CAAA;AAErE,IAAA,OAAO,EAAE,UAAA,EAAW;AAAA,EACtB;AAAA,EAEA,MAAM,YAAe,EAAA,EAAiD;AACpE,IAAA,IAAI;AACF,MAAA,IAAI,MAAA,GAAwB,KAAA,CAAA;AAE5B,MAAA,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,WAAA;AAAA,QAC1B,OAAM,EAAA,KAAM;AAGV,UAAA,MAAA,GAAS,MAAM,GAAG,EAAE,CAAA;AAAA,QACtB,CAAA;AAAA,QACA;AAAA;AAAA,UAEE,qBAAA,EAAuB;AAAA;AACzB,OACF;AAEA,MAAA,OAAO,MAAA;AAAA,IACT,SAAS,CAAA,EAAG;AACV,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,0BAAA,EAA6B,CAAC,CAAA,CAAE,CAAA;AAC1D,MAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA,IACtB;AAAA,EACF;AAAA,EAEQ,qBAAqB,IAAA,EAA0C;AACrE,IAAA,OAAOC,uBAAA,CAAO,MAAA;AAAA,MACZ,IAAA;AAAA,MACA,CAAA,CAAA,KAAK,GAAG,CAAA,CAAE,iBAAiB,IAAI,CAAA,CAAE,iBAAiB,CAAA,CAAA,EAAI,CAAA,CAAE,IAAI,CAAA;AAAA,KAC9D;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,sBAAA,CACZ,QAAA,EACA,OAAA,EAIe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AAGX,IAAA,MAAM,eAAA,GAAkB,IAAI,KAAA,EAAc;AAI1C,IAAA,KAAA,MAAW,EAAE,MAAA,EAAQ,WAAA,EAAY,IAAK,QAAQ,QAAA,EAAU;AACtD,MAAA,MAAM,SAAA,GAAYJ,gCAAmB,MAAM,CAAA;AAC3C,MAAA,MAAM,IAAA,GAAOK,wBAAmB,MAAM,CAAA;AAEtC,MAAA,MAAM,OAAA,GAAU,MAAMC,+CAAA,CAAwB;AAAA,QAC5C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,OAAA,EAAS;AACX,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,QAAA,GAAW,MAAMC,+CAAA,CAAwB;AAAA,QAC7C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA,WAAA;AAAA,QACA,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAKA,MAAA,MAAM,cAAA,GAAiB,MAAMC,iDAAA,CAAyB;AAAA,QACpD,EAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,cAAA,EAAgB;AAClB,QAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,UAClB,CAAA,+BAAA,EAAkC,SAAS,CAAA,uBAAA,EAA0B,cAAc,iBAAiB,WAAW,CAAA;AAAA,SACjH;AACA,QAAA,IAAI,IAAA,CAAK,OAAA,CAAQ,WAAA,IAAe,WAAA,EAAa;AAC3C,UAAA,MAAM,WAAA,GAAwD;AAAA,YAC5D,KAAA,EAAOC,iCAAA;AAAA,YACP,YAAA,EAAc;AAAA,cACZ,iBAAA,EAAmB,MAAA;AAAA,cACnB,SAAA;AAAA,cACA,cAAA,EAAgB,WAAA;AAAA,cAChB,mBAAA,EAAqB,cAAA;AAAA,cACrB,cAAA,EAAgBC,cAAA,CAAS,GAAA,EAAI,CAAE,KAAA;AAAM;AACvC,WACF;AACA,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,WAAA,EAAa,OAAA,CAAQ,WAAW,CAAA;AAAA,QACrD;AAAA,MACF;AAAA,IACF;AAGA,IAAA,MAAM,EAAA,CAAgC,0BAA0B,CAAA,CAE7D,KAAA,CAAM,EAAE,iBAAA,EAAmB,OAAA,CAAQ,eAAA,EAAiB,CAAA,CAEpD,SAAA,CAAU,mBAAA,EAAqB,eAAe,EAC9C,MAAA,EAAO;AACV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,0BAAA;AAAA,MACA,eAAA,CAAgB,IAAI,CAAA,SAAA,MAAc;AAAA,QAChC,mBAAmB,OAAA,CAAQ,eAAA;AAAA,QAC3B,iBAAA,EAAmB;AAAA,OACrB,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAAA,EACF;AACF;;;;"}
+{"version":3,"file":"DefaultProcessingDatabase.cjs.js","sources":["../../src/database/DefaultProcessingDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError } from '@backstage/errors';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { ProcessingIntervalFunction } from '../processing/refresh';\nimport { rethrowError, timestampToDateTime } from './conversion';\nimport { initDatabaseMetrics } from './metrics';\nimport {\n  DbRefreshKeysRow,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n} from './tables';\nimport {\n  GetProcessableEntitiesResult,\n  ListParentsOptions,\n  ListParentsResult,\n  ProcessingDatabase,\n  RefreshStateItem,\n  Transaction,\n  UpdateEntityCacheOptions,\n  UpdateProcessedEntityOptions,\n} from './types';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { generateStableHash, generateTargetKey } from './util';\nimport { EventParams, EventsService } from '@backstage/plugin-events-node';\nimport { DateTime } from 'luxon';\nimport { CATALOG_CONFLICTS_TOPIC } from '../constants';\nimport { CatalogConflictEventPayload } from '../catalog/types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProcessingDatabase implements ProcessingDatabase {\n  constructor(\n    private readonly options: {\n      database: Knex;\n      logger: LoggerService;\n      refreshInterval: ProcessingIntervalFunction;\n      events: EventsService;\n    },\n  ) {\n    initDatabaseMetrics(options.database);\n  }\n\n  async updateProcessedEntity(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<{ previous: { relations: DbRelationsRow[] } }> {\n    const tx = txOpaque as Knex.Transaction;\n    const {\n      id,\n      processedEntity,\n      resultHash,\n      errors,\n      relations,\n      deferredEntities,\n      refreshKeys,\n      locationKey,\n    } = options;\n    const configClient = tx.client.config.client;\n    const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        processed_entity: JSON.stringify(processedEntity),\n        result_hash: resultHash,\n        errors,\n        location_key: locationKey,\n      })\n      .where('entity_id', id)\n      .andWhere(inner => {\n        if (!locationKey) {\n          return inner.whereNull('location_key');\n        }\n        return inner\n          .where('location_key', locationKey)\n          .orWhereNull('location_key');\n      });\n    if (refreshResult === 0) {\n      throw new ConflictError(\n        `Conflicting write of processing result for ${id} with location key '${locationKey}'`,\n      );\n    }\n    const sourceEntityRef = stringifyEntityRef(processedEntity);\n\n    // Schedule all deferred entities for future processing.\n    await this.addUnprocessedEntities(tx, {\n      entities: deferredEntities,\n      sourceEntityRef,\n    });\n\n    // Delete old relations\n    // NOTE(freben): knex implemented support for returning() on update queries for sqlite, but at the current time of writing (Sep 2022) not for delete() queries.\n    let previousRelationRows: DbRelationsRow[];\n    if (configClient.includes('sqlite3') || configClient.includes('mysql')) {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .select('*')\n        .where({ originating_entity_id: id });\n      await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete();\n    } else {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete()\n        .returning('*');\n    }\n\n    // Batch insert new relations\n    const relationRows: DbRelationsRow[] = relations.map(\n      ({ source, target, type }) => ({\n        originating_entity_id: id,\n        source_entity_ref: stringifyEntityRef(source),\n        target_entity_ref: stringifyEntityRef(target),\n        type,\n      }),\n    );\n\n    await tx.batchInsert(\n      'relations',\n      this.deduplicateRelations(relationRows),\n      BATCH_SIZE,\n    );\n\n    // Delete old refresh keys\n    await tx<DbRefreshKeysRow>('refresh_keys')\n      .where({ entity_id: id })\n      .delete();\n\n    // Insert the refresh keys for the processed entity\n    await tx.batchInsert(\n      'refresh_keys',\n      refreshKeys.map(k => ({\n        entity_id: id,\n        key: generateTargetKey(k.key),\n      })),\n      BATCH_SIZE,\n    );\n\n    return {\n      previous: {\n        relations: previousRelationRows,\n      },\n    };\n  }\n\n  async updateProcessedEntityErrors(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, errors, resultHash } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        errors,\n        result_hash: resultHash,\n      })\n      .where('entity_id', id);\n  }\n\n  async updateEntityCache(\n    txOpaque: Transaction,\n    options: UpdateEntityCacheOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, state } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({ cache: JSON.stringify(state ?? {}) })\n      .where('entity_id', id);\n  }\n\n  async getProcessableEntities(\n    maybeTx: Transaction | Knex,\n    request: { processBatchSize: number },\n  ): Promise<GetProcessableEntitiesResult> {\n    const knex = maybeTx as Knex.Transaction | Knex;\n\n    let itemsQuery = knex<DbRefreshStateRow>('refresh_state').select([\n      'entity_id',\n      'entity_ref',\n      'unprocessed_entity',\n      'result_hash',\n      'cache',\n      'errors',\n      'location_key',\n      'next_update_at',\n    ]);\n\n    // This avoids duplication of work because of race conditions and is\n    // also fast because locked rows are ignored rather than blocking.\n    // It's only available in MySQL and PostgreSQL\n    if (['mysql', 'mysql2', 'pg'].includes(knex.client.config.client)) {\n      itemsQuery = itemsQuery.forUpdate().skipLocked();\n    }\n\n    const items = await itemsQuery\n      .where('next_update_at', '<=', knex.fn.now())\n      .limit(request.processBatchSize)\n      .orderBy('next_update_at', 'asc');\n\n    const interval = this.options.refreshInterval();\n\n    const nextUpdateAt = (refreshInterval: number) => {\n      if (knex.client.config.client.includes('sqlite3')) {\n        return knex.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);\n      } else if (knex.client.config.client.includes('mysql')) {\n        return knex.raw(`now() + interval ${refreshInterval} second`);\n      }\n      return knex.raw(`now() + interval '${refreshInterval} seconds'`);\n    };\n\n    await knex<DbRefreshStateRow>('refresh_state')\n      .whereIn(\n        'entity_ref',\n        items.map(i => i.entity_ref),\n      )\n      .update({\n        next_update_at: nextUpdateAt(interval),\n      });\n\n    return {\n      items: items.map(\n        i =>\n          ({\n            id: i.entity_id,\n            entityRef: i.entity_ref,\n            unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity,\n            resultHash: i.result_hash || '',\n            nextUpdateAt: timestampToDateTime(i.next_update_at),\n            state: i.cache ? JSON.parse(i.cache) : undefined,\n            errors: i.errors,\n            locationKey: i.location_key,\n          } satisfies RefreshStateItem),\n      ),\n    };\n  }\n\n  async listParents(\n    txOpaque: Transaction,\n    options: ListParentsOptions,\n  ): Promise<ListParentsResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .whereIn('target_entity_ref', options.entityRefs)\n      .select();\n\n    const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);\n\n    return { entityRefs };\n  }\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] {\n    return lodash.uniqBy(\n      rows,\n      r => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`,\n    );\n  }\n\n  /**\n   * Add a set of deferred entities for processing.\n   * The entities will be added at the front of the processing queue.\n   */\n  private async addUnprocessedEntities(\n    txOpaque: Transaction,\n    options: {\n      sourceEntityRef: string;\n      entities: DeferredEntity[];\n    },\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n\n    // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards\n    const stateReferences = new Array<string>();\n\n    // Upsert all of the unprocessed entities into the refresh_state table, by\n    // their entity ref.\n    for (const { entity, locationKey } of options.entities) {\n      const entityRef = stringifyEntityRef(entity);\n      const hash = generateStableHash(entity);\n\n      const updated = await updateUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n      });\n      if (updated) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      const inserted = await insertUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n        logger: this.options.logger,\n      });\n      if (inserted) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      // If the row can't be inserted, we have a conflict, but it could be either\n      // because of a conflicting locationKey or a race with another instance, so check\n      // whether the conflicting entity has the same entityRef but a different locationKey\n      const conflictingKey = await checkLocationKeyConflict({\n        tx,\n        entityRef,\n        locationKey,\n      });\n      if (conflictingKey) {\n        this.options.logger.warn(\n          `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n        );\n        if (locationKey) {\n          const eventParams: EventParams<CatalogConflictEventPayload> = {\n            topic: CATALOG_CONFLICTS_TOPIC,\n            eventPayload: {\n              unprocessedEntity: entity,\n              entityRef,\n              newLocationKey: locationKey,\n              existingLocationKey: conflictingKey,\n              lastConflictAt: DateTime.now().toISO()!,\n            },\n          };\n          await this.options.events.publish(eventParams);\n        }\n      }\n    }\n\n    // Lastly, replace refresh state references for the originating entity and any successfully added entities\n    await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n      // Remove all existing references from the originating entity\n      .where({ source_entity_ref: options.sourceEntityRef })\n      // And remove any existing references to entities that we're inserting new references for\n      .orWhereIn('target_entity_ref', stateReferences)\n      .delete();\n    await tx.batchInsert(\n      'refresh_state_references',\n      stateReferences.map(entityRef => ({\n        source_entity_ref: options.sourceEntityRef,\n        target_entity_ref: entityRef,\n      })),\n      BATCH_SIZE,\n    );\n  }\n}\n"],"names":["initDatabaseMetrics","errors","ConflictError","stringifyEntityRef","generateTargetKey","timestampToDateTime","rethrowError","lodash","generateStableHash","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","CATALOG_CONFLICTS_TOPIC","DateTime"],"mappings":";;;;;;;;;;;;;;;;;;AAsDA,MAAM,UAAA,GAAa,EAAA;AAEZ,MAAM,yBAAA,CAAwD;AAAA,EACnE,YACmB,OAAA,EAMjB;AANiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAOjB,IAAAA,2BAAA,CAAoB,QAAQ,QAAQ,CAAA;AAAA,EACtC;AAAA,EAEA,MAAM,qBAAA,CACJ,QAAA,EACA,OAAA,EACwD;AACxD,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM;AAAA,MACJ,EAAA;AAAA,MACA,eAAA;AAAA,MACA,UAAA;AAAA,cACAC,QAAA;AAAA,MACA,SAAA;AAAA,MACA,gBAAA;AAAA,MACA,WAAA;AAAA,MACA;AAAA,KACF,GAAI,OAAA;AACJ,IAAA,MAAM,YAAA,GAAe,EAAA,CAAG,MAAA,CAAO,MAAA,CAAO,MAAA;AACtC,IAAA,MAAM,aAAA,GAAgB,MAAM,EAAA,CAAsB,eAAe,EAC9D,MAAA,CAAO;AAAA,MACN,gBAAA,EAAkB,IAAA,CAAK,SAAA,CAAU,eAAe,CAAA;AAAA,MAChD,WAAA,EAAa,UAAA;AAAA,cACbA,QAAA;AAAA,MACA,YAAA,EAAc;AAAA,KACf,CAAA,CACA,KAAA,CAAM,aAAa,EAAE,CAAA,CACrB,SAAS,CAAA,KAAA,KAAS;AACjB,MAAA,IAAI,CAAC,WAAA,EAAa;AAChB,QAAA,OAAO,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA,MACvC;AACA,MAAA,OAAO,MACJ,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,IAC/B,CAAC,CAAA;AACH,IAAA,IAAI,kBAAkB,CAAA,EAAG;AACvB,MAAA,MAAM,IAAIC,oBAAA;AAAA,QACR,CAAA,2CAAA,EAA8C,EAAE,CAAA,oBAAA,EAAuB,WAAW,CAAA,CAAA;AAAA,OACpF;AAAA,IACF;AACA,IAAA,MAAM,eAAA,GAAkBC,gCAAmB,eAAe,CAAA;AAG1D,IAAA,MAAM,IAAA,CAAK,uBAAuB,EAAA,EAAI;AAAA,MACpC,QAAA,EAAU,gBAAA;AAAA,MACV;AAAA,KACD,CAAA;AAID,IAAA,IAAI,oBAAA;AACJ,IAAA,IAAI,aAAa,QAAA,CAAS,SAAS,KAAK,YAAA,CAAa,QAAA,CAAS,OAAO,CAAA,EAAG;AACtE,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,MAAA,CAAO,GAAG,CAAA,CACV,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA;AACtC,MAAA,MAAM,EAAA,CAAmB,WAAW,CAAA,CACjC,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO;AAAA,IACZ,CAAA,MAAO;AACL,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO,CACP,UAAU,GAAG,CAAA;AAAA,IAClB;AAGA,IAAA,MAAM,eAAiC,SAAA,CAAU,GAAA;AAAA,MAC/C,CAAC,EAAE,MAAA,EAAQ,MAAA,EAAQ,MAAK,MAAO;AAAA,QAC7B,qBAAA,EAAuB,EAAA;AAAA,QACvB,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C;AAAA,OACF;AAAA,KACF;AAEA,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,WAAA;AAAA,MACA,IAAA,CAAK,qBAAqB,YAAY,CAAA;AAAA,MACtC;AAAA,KACF;AAGA,IAAA,MAAM,EAAA,CAAqB,cAAc,CAAA,CACtC,KAAA,CAAM,EAAE,SAAA,EAAW,EAAA,EAAI,CAAA,CACvB,MAAA,EAAO;AAGV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,cAAA;AAAA,MACA,WAAA,CAAY,IAAI,CAAA,CAAA,MAAM;AAAA,QACpB,SAAA,EAAW,EAAA;AAAA,QACX,GAAA,EAAKC,sBAAA,CAAkB,CAAA,CAAE,GAAG;AAAA,OAC9B,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAEA,IAAA,OAAO;AAAA,MACL,QAAA,EAAU;AAAA,QACR,SAAA,EAAW;AAAA;AACb,KACF;AAAA,EACF;AAAA,EAEA,MAAM,2BAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,MAAA,EAAQ,UAAA,EAAW,GAAI,OAAA;AAEnC,IAAA,MAAM,EAAA,CAAsB,eAAe,CAAA,CACxC,MAAA,CAAO;AAAA,MACN,MAAA;AAAA,MACA,WAAA,EAAa;AAAA,KACd,CAAA,CACA,KAAA,CAAM,WAAA,EAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,iBAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,KAAA,EAAM,GAAI,OAAA;AAEtB,IAAA,MAAM,GAAsB,eAAe,CAAA,CACxC,MAAA,CAAO,EAAE,OAAO,IAAA,CAAK,SAAA,CAAU,KAAA,IAAS,EAAE,CAAA,EAAG,CAAA,CAC7C,KAAA,CAAM,aAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,sBAAA,CACJ,OAAA,EACA,OAAA,EACuC;AACvC,IAAA,MAAM,IAAA,GAAO,OAAA;AAEb,IAAA,IAAI,UAAA,GAAa,IAAA,CAAwB,eAAe,CAAA,CAAE,MAAA,CAAO;AAAA,MAC/D,WAAA;AAAA,MACA,YAAA;AAAA,MACA,oBAAA;AAAA,MACA,aAAA;AAAA,MACA,OAAA;AAAA,MACA,QAAA;AAAA,MACA,cAAA;AAAA,MACA;AAAA,KACD,CAAA;AAKD,IAAA,IAAI,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,SAAS,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,MAAM,CAAA,EAAG;AACjE,MAAA,UAAA,GAAa,UAAA,CAAW,SAAA,EAAU,CAAE,UAAA,EAAW;AAAA,IACjD;AAEA,IAAA,MAAM,QAAQ,MAAM,UAAA,CACjB,KAAA,CAAM,gBAAA,EAAkB,MAAM,IAAA,CAAK,EAAA,CAAG,GAAA,EAAK,EAC3C,KAAA,CAAM,OAAA,CAAQ,gBAAgB,CAAA,CAC9B,OAAA,CAAQ,kBAAkB,KAAK,CAAA;AAElC,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,OAAA,CAAQ,eAAA,EAAgB;AAE9C,IAAA,MAAM,YAAA,GAAe,CAAC,eAAA,KAA4B;AAChD,MAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AACjD,QAAA,OAAO,KAAK,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,eAAe,UAAU,CAAC,CAAA;AAAA,MACtE,WAAW,IAAA,CAAK,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACtD,QAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,iBAAA,EAAoB,eAAe,CAAA,OAAA,CAAS,CAAA;AAAA,MAC9D;AACA,MAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,kBAAA,EAAqB,eAAe,CAAA,SAAA,CAAW,CAAA;AAAA,IACjE,CAAA;AAEA,IAAA,MAAM,IAAA,CAAwB,eAAe,CAAA,CAC1C,OAAA;AAAA,MACC,YAAA;AAAA,MACA,KAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,MAE5B,MAAA,CAAO;AAAA,MACN,cAAA,EAAgB,aAAa,QAAQ;AAAA,KACtC,CAAA;AAEH,IAAA,OAAO;AAAA,MACL,OAAO,KAAA,CAAM,GAAA;AAAA,QACX,CAAA,CAAA,MACG;AAAA,UACC,IAAI,CAAA,CAAE,SAAA;AAAA,UACN,WAAW,CAAA,CAAE,UAAA;AAAA,UACb,iBAAA,EAAmB,IAAA,CAAK,KAAA,CAAM,CAAA,CAAE,kBAAkB,CAAA;AAAA,UAClD,UAAA,EAAY,EAAE,WAAA,IAAe,EAAA;AAAA,UAC7B,YAAA,EAAcC,8BAAA,CAAoB,CAAA,CAAE,cAAc,CAAA;AAAA,UAClD,OAAO,CAAA,CAAE,KAAA,GAAQ,KAAK,KAAA,CAAM,CAAA,CAAE,KAAK,CAAA,GAAI,MAAA;AAAA,UACvC,QAAQ,CAAA,CAAE,MAAA;AAAA,UACV,aAAa,CAAA,CAAE;AAAA,SACjB;AAAA;AACJ,KACF;AAAA,EACF;AAAA,EAEA,MAAM,WAAA,CACJ,QAAA,EACA,OAAA,EAC4B;AAC5B,IAAA,MAAM,EAAA,GAAK,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,MAEC,OAAA,CAAQ,mBAAA,EAAqB,OAAA,CAAQ,UAAU,EAC/C,MAAA,EAAO;AAEV,IAAA,MAAM,UAAA,GAAa,KAAK,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,iBAAkB,CAAA,CAAE,OAAO,OAAO,CAAA;AAErE,IAAA,OAAO,EAAE,UAAA,EAAW;AAAA,EACtB;AAAA,EAEA,MAAM,YAAe,EAAA,EAAiD;AACpE,IAAA,IAAI;AACF,MAAA,IAAI,MAAA,GAAwB,KAAA,CAAA;AAE5B,MAAA,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,WAAA;AAAA,QAC1B,OAAM,EAAA,KAAM;AAGV,UAAA,MAAA,GAAS,MAAM,GAAG,EAAE,CAAA;AAAA,QACtB,CAAA;AAAA,QACA;AAAA;AAAA,UAEE,qBAAA,EAAuB;AAAA;AACzB,OACF;AAEA,MAAA,OAAO,MAAA;AAAA,IACT,SAAS,CAAA,EAAG;AACV,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,0BAAA,EAA6B,CAAC,CAAA,CAAE,CAAA;AAC1D,MAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA,IACtB;AAAA,EACF;AAAA,EAEQ,qBAAqB,IAAA,EAA0C;AACrE,IAAA,OAAOC,uBAAA,CAAO,MAAA;AAAA,MACZ,IAAA;AAAA,MACA,CAAA,CAAA,KAAK,GAAG,CAAA,CAAE,iBAAiB,IAAI,CAAA,CAAE,iBAAiB,CAAA,CAAA,EAAI,CAAA,CAAE,IAAI,CAAA;AAAA,KAC9D;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,sBAAA,CACZ,QAAA,EACA,OAAA,EAIe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AAGX,IAAA,MAAM,eAAA,GAAkB,IAAI,KAAA,EAAc;AAI1C,IAAA,KAAA,MAAW,EAAE,MAAA,EAAQ,WAAA,EAAY,IAAK,QAAQ,QAAA,EAAU;AACtD,MAAA,MAAM,SAAA,GAAYJ,gCAAmB,MAAM,CAAA;AAC3C,MAAA,MAAM,IAAA,GAAOK,wBAAmB,MAAM,CAAA;AAEtC,MAAA,MAAM,OAAA,GAAU,MAAMC,+CAAA,CAAwB;AAAA,QAC5C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,OAAA,EAAS;AACX,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,QAAA,GAAW,MAAMC,+CAAA,CAAwB;AAAA,QAC7C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA,WAAA;AAAA,QACA,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAKA,MAAA,MAAM,cAAA,GAAiB,MAAMC,iDAAA,CAAyB;AAAA,QACpD,EAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,cAAA,EAAgB;AAClB,QAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,UAClB,CAAA,+BAAA,EAAkC,SAAS,CAAA,uBAAA,EAA0B,cAAc,iBAAiB,WAAW,CAAA;AAAA,SACjH;AACA,QAAA,IAAI,WAAA,EAAa;AACf,UAAA,MAAM,WAAA,GAAwD;AAAA,YAC5D,KAAA,EAAOC,iCAAA;AAAA,YACP,YAAA,EAAc;AAAA,cACZ,iBAAA,EAAmB,MAAA;AAAA,cACnB,SAAA;AAAA,cACA,cAAA,EAAgB,WAAA;AAAA,cAChB,mBAAA,EAAqB,cAAA;AAAA,cACrB,cAAA,EAAgBC,cAAA,CAAS,GAAA,EAAI,CAAE,KAAA;AAAM;AACvC,WACF;AACA,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,OAAA,CAAQ,WAAW,CAAA;AAAA,QAC/C;AAAA,MACF;AAAA,IACF;AAGA,IAAA,MAAM,EAAA,CAAgC,0BAA0B,CAAA,CAE7D,KAAA,CAAM,EAAE,iBAAA,EAAmB,OAAA,CAAQ,eAAA,EAAiB,CAAA,CAEpD,SAAA,CAAU,mBAAA,EAAqB,eAAe,EAC9C,MAAA,EAAO;AACV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,0BAAA;AAAA,MACA,eAAA,CAAgB,IAAI,CAAA,SAAA,MAAc;AAAA,QAChC,mBAAmB,OAAA,CAAQ,eAAA;AAAA,QAC3B,iBAAA,EAAmB;AAAA,OACrB,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAAA,EACF;AACF;;;;"}

package/dist/ingestion/CatalogRules.cjs.js

@@ -2,11 +2,18 @@
 
 var path = require('path');
 var minimatch = require('minimatch');
+var zod = require('zod');
 
 function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e : { default: e }; }
 
 var path__default = /*#__PURE__*/_interopDefaultCompat(path);
 
+const allowRuleParser = zod.z.array(
+  zod.z.object({
+    kind: zod.z.string(),
+    "spec.type": zod.z.string().optional()
+  }).or(zod.z.string()).transform((val) => typeof val === "string" ? { kind: val } : val)
+);
 class DefaultCatalogRulesEnforcer {
   constructor(rules) {
     this.rules = rules;
@@ -39,6 +46,9 @@ class DefaultCatalogRulesEnforcer {
    * catalog:
    *   rules:
    *   - allow: [Component, API]
+   *   - allow:
+   *     - kind: Resource
+   *       'spec.type': database
    *   - allow: [Template]
    *     locations:
    *       - type: url
@@ -63,7 +73,7 @@ class DefaultCatalogRulesEnforcer {
     const rules = new Array();
     if (config.has("catalog.rules")) {
       const globalRules = config.getConfigArray("catalog.rules").map((ruleConf) => ({
-        allow: ruleConf.getStringArray("allow").map((kind) => ({ kind })),
+        allow: allowRuleParser.parse(ruleConf.get("allow")),
         locations: ruleConf.getOptionalConfigArray("locations")?.map((locationConfig) => {
           const location = {
             pattern: locationConfig.getOptionalString("pattern"),
@@ -139,9 +149,17 @@ class DefaultCatalogRulesEnforcer {
       return true;
     }
     for (const matcher of matchers) {
-      if (entity?.kind?.toLowerCase() !== matcher.kind.toLowerCase()) {
+      if (entity.kind?.toLocaleLowerCase("en-US") !== matcher.kind.toLocaleLowerCase("en-US")) {
         continue;
       }
+      if (matcher["spec.type"]) {
+        if (typeof entity.spec?.type !== "string") {
+          continue;
+        }
+        if (matcher["spec.type"].toLocaleLowerCase("en-US") !== entity.spec.type.toLocaleLowerCase("en-US")) {
+          continue;
+        }
+      }
       return true;
     }
     return false;

package/dist/ingestion/CatalogRules.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"CatalogRules.cjs.js","sources":["../../src/ingestion/CatalogRules.ts"],"sourcesContent":["/*\n * Copyright 2020 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Config } from '@backstage/config';\nimport { Entity } from '@backstage/catalog-model';\nimport path from 'path';\nimport { LocationSpec } from '@backstage/plugin-catalog-common';\nimport { minimatch } from 'minimatch';\n\n/**\n * Rules to apply to catalog entities.\n *\n * An undefined list of matchers means match all, an empty list of matchers means match none.\n */\nexport type CatalogRule = {\n  allow: Array<{\n    kind: string;\n  }>;\n  locations?: Array<{\n    exact?: string;\n    type: string;\n    pattern?: string;\n  }>;\n};\n\n/**\n * Decides whether an entity from a given location is allowed to enter the\n * catalog, according to some rule set.\n */\nexport type CatalogRulesEnforcer = {\n  isAllowed(entity: Entity, location: LocationSpec): boolean;\n};\n\n/**\n * Implements the default catalog rule set, consuming the config keys\n * `catalog.rules` and `catalog.locations.[].rules`.\n */\nexport class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer {\n  /**\n   * Default rules used by the catalog.\n   *\n   * Denies any location from specifying user or group entities.\n   */\n  static readonly defaultRules: CatalogRule[] = [\n    {\n      allow: ['Component', 'API', 'Location'].map(kind => ({ kind })),\n    },\n  ];\n\n  /**\n   * Loads catalog rules from config.\n   *\n   * This reads `catalog.rules` and defaults to the default rules if no value is present.\n   * The value of the config should be a list of config objects, each with a single `allow`\n   * field which in turn is a list of entity kinds to allow.\n   *\n   * If there is no matching rule to allow an ingested entity, it will be rejected by the catalog.\n   *\n   * It also reads in rules from `catalog.locations`, where each location can have a list\n   * of rules for that specific location, specified in a `rules` field.\n   *\n   * For example:\n   *\n   * ```yaml\n   * catalog:\n   *   rules:\n   *   - allow: [Component, API]\n   *   - allow: [Template]\n   *     locations:\n   *       - type: url\n   *         pattern: https://github.com/org/*\\/blob/master/template.yaml\n   *   - allow: [Location]\n   *     locations:\n   *       - type: url\n   *         pattern: https://github.com/org/repo/blob/master/location.yaml\n   *\n   *   locations:\n   *   - type: url\n   *     target: https://github.com/org/repo/blob/master/users.yaml\n   *     rules:\n   *       - allow: [User, Group]\n   *   - type: url\n   *     target: https://github.com/org/repo/blob/master/systems.yaml\n   *     rules:\n   *       - allow: [System]\n   * ```\n   */\n  static fromConfig(config: Config) {\n    const rules = new Array<CatalogRule>();\n\n    if (config.has('catalog.rules')) {\n      const globalRules = config\n        .getConfigArray('catalog.rules')\n        .map(ruleConf => ({\n          allow: ruleConf.getStringArray('allow').map(kind => ({ kind })),\n          locations: ruleConf\n            .getOptionalConfigArray('locations')\n            ?.map(locationConfig => {\n              const location = {\n                pattern: locationConfig.getOptionalString('pattern'),\n                type: locationConfig.getString('type'),\n                exact: locationConfig.getOptionalString('exact'),\n              };\n              if (location.pattern && location.exact) {\n                throw new Error(\n                  'A catalog rule location cannot have both exact and pattern values',\n                );\n              }\n              return location;\n            }),\n        }));\n      rules.push(...globalRules);\n    } else {\n      rules.push(...DefaultCatalogRulesEnforcer.defaultRules);\n    }\n\n    if (config.has('catalog.locations')) {\n      const locationRules = config\n        .getConfigArray('catalog.locations')\n        .flatMap(locConf => {\n          if (!locConf.has('rules')) {\n            return [];\n          }\n          const type = locConf.getString('type');\n          const exact = resolveTarget(type, locConf.getString('target'));\n\n          return locConf.getConfigArray('rules').map(ruleConf => ({\n            allow: ruleConf.getStringArray('allow').map(kind => ({ kind })),\n            locations: [{ type, exact }],\n          }));\n        });\n\n      rules.push(...locationRules);\n    }\n\n    return new DefaultCatalogRulesEnforcer(rules);\n  }\n\n  constructor(private readonly rules: CatalogRule[]) {}\n\n  /**\n   * Checks whether a specific entity/location combination is allowed\n   * according to the configured rules.\n   */\n  isAllowed(entity: Entity, location: LocationSpec) {\n    for (const rule of this.rules) {\n      if (!this.matchLocation(location, rule.locations)) {\n        continue;\n      }\n\n      if (this.matchEntity(entity, rule.allow)) {\n        return true;\n      }\n    }\n\n    return false;\n  }\n\n  private matchLocation(\n    location: LocationSpec,\n    matchers?: { exact?: string; type: string; pattern?: string }[],\n  ): boolean {\n    if (!matchers) {\n      return true;\n    }\n\n    for (const matcher of matchers) {\n      if (matcher.type !== location?.type) {\n        continue;\n      }\n      if (matcher.exact && matcher.exact !== location?.target) {\n        continue;\n      }\n      if (\n        matcher.pattern &&\n        !minimatch(location?.target, matcher.pattern, {\n          nocase: true,\n          dot: true,\n        })\n      ) {\n        continue;\n      }\n      return true;\n    }\n\n    return false;\n  }\n\n  private matchEntity(entity: Entity, matchers?: { kind: string }[]): boolean {\n    if (!matchers) {\n      return true;\n    }\n\n    for (const matcher of matchers) {\n      if (entity?.kind?.toLowerCase() !== matcher.kind.toLowerCase()) {\n        continue;\n      }\n\n      return true;\n    }\n\n    return false;\n  }\n}\n\nfunction resolveTarget(type: string, target: string): string {\n  if (type !== 'file') {\n    return target;\n  }\n\n  return path.resolve(target);\n}\n"],"names":["minimatch","path"],"mappings":";;;;;;;;;AAkDO,MAAM,2BAAA,CAA4D;AAAA,EAqGvE,YAA6B,KAAA,EAAsB;AAAtB,IAAA,IAAA,CAAA,KAAA,GAAA,KAAA;AAAA,EAAuB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EA/FpD,OAAgB,YAAA,GAA8B;AAAA,IAC5C;AAAA,MACE,KAAA,EAAO,CAAC,WAAA,EAAa,KAAA,EAAO,UAAU,EAAE,GAAA,CAAI,CAAA,IAAA,MAAS,EAAE,IAAA,EAAK,CAAE;AAAA;AAChE,GACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAwCA,OAAO,WAAW,MAAA,EAAgB;AAChC,IAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAmB;AAErC,IAAA,IAAI,MAAA,CAAO,GAAA,CAAI,eAAe,CAAA,EAAG;AAC/B,MAAA,MAAM,cAAc,MAAA,CACjB,cAAA,CAAe,eAAe,CAAA,CAC9B,IAAI,CAAA,QAAA,MAAa;AAAA,QAChB,KAAA,EAAO,SAAS,cAAA,CAAe,OAAO,EAAE,GAAA,CAAI,CAAA,IAAA,MAAS,EAAE,IAAA,EAAK,CAAE,CAAA;AAAA,QAC9D,WAAW,QAAA,CACR,sBAAA,CAAuB,WAAW,CAAA,EACjC,IAAI,CAAA,cAAA,KAAkB;AACtB,UAAA,MAAM,QAAA,GAAW;AAAA,YACf,OAAA,EAAS,cAAA,CAAe,iBAAA,CAAkB,SAAS,CAAA;AAAA,YACnD,IAAA,EAAM,cAAA,CAAe,SAAA,CAAU,MAAM,CAAA;AAAA,YACrC,KAAA,EAAO,cAAA,CAAe,iBAAA,CAAkB,OAAO;AAAA,WACjD;AACA,UAAA,IAAI,QAAA,CAAS,OAAA,IAAW,QAAA,CAAS,KAAA,EAAO;AACtC,YAAA,MAAM,IAAI,KAAA;AAAA,cACR;AAAA,aACF;AAAA,UACF;AACA,UAAA,OAAO,QAAA;AAAA,QACT,CAAC;AAAA,OACL,CAAE,CAAA;AACJ,MAAA,KAAA,CAAM,IAAA,CAAK,GAAG,WAAW,CAAA;AAAA,IAC3B,CAAA,MAAO;AACL,MAAA,KAAA,CAAM,IAAA,CAAK,GAAG,2BAAA,CAA4B,YAAY,CAAA;AAAA,IACxD;AAEA,IAAA,IAAI,MAAA,CAAO,GAAA,CAAI,mBAAmB,CAAA,EAAG;AACnC,MAAA,MAAM,gBAAgB,MAAA,CACnB,cAAA,CAAe,mBAAmB,CAAA,CAClC,QAAQ,CAAA,OAAA,KAAW;AAClB,QAAA,IAAI,CAAC,OAAA,CAAQ,GAAA,CAAI,OAAO,CAAA,EAAG;AACzB,UAAA,OAAO,EAAC;AAAA,QACV;AACA,QAAA,MAAM,IAAA,GAAO,OAAA,CAAQ,SAAA,CAAU,MAAM,CAAA;AACrC,QAAA,MAAM,QAAQ,aAAA,CAAc,IAAA,EAAM,OAAA,CAAQ,SAAA,CAAU,QAAQ,CAAC,CAAA;AAE7D,QAAA,OAAO,OAAA,CAAQ,cAAA,CAAe,OAAO,CAAA,CAAE,IAAI,CAAA,QAAA,MAAa;AAAA,UACtD,KAAA,EAAO,SAAS,cAAA,CAAe,OAAO,EAAE,GAAA,CAAI,CAAA,IAAA,MAAS,EAAE,IAAA,EAAK,CAAE,CAAA;AAAA,UAC9D,SAAA,EAAW,CAAC,EAAE,IAAA,EAAM,OAAO;AAAA,SAC7B,CAAE,CAAA;AAAA,MACJ,CAAC,CAAA;AAEH,MAAA,KAAA,CAAM,IAAA,CAAK,GAAG,aAAa,CAAA;AAAA,IAC7B;AAEA,IAAA,OAAO,IAAI,4BAA4B,KAAK,CAAA;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,SAAA,CAAU,QAAgB,QAAA,EAAwB;AAChD,IAAA,KAAA,MAAW,IAAA,IAAQ,KAAK,KAAA,EAAO;AAC7B,MAAA,IAAI,CAAC,IAAA,CAAK,aAAA,CAAc,QAAA,EAAU,IAAA,CAAK,SAAS,CAAA,EAAG;AACjD,QAAA;AAAA,MACF;AAEA,MAAA,IAAI,IAAA,CAAK,WAAA,CAAY,MAAA,EAAQ,IAAA,CAAK,KAAK,CAAA,EAAG;AACxC,QAAA,OAAO,IAAA;AAAA,MACT;AAAA,IACF;AAEA,IAAA,OAAO,KAAA;AAAA,EACT;AAAA,EAEQ,aAAA,CACN,UACA,QAAA,EACS;AACT,IAAA,IAAI,CAAC,QAAA,EAAU;AACb,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,KAAA,MAAW,WAAW,QAAA,EAAU;AAC9B,MAAA,IAAI,OAAA,CAAQ,IAAA,KAAS,QAAA,EAAU,IAAA,EAAM;AACnC,QAAA;AAAA,MACF;AACA,MAAA,IAAI,OAAA,CAAQ,KAAA,IAAS,OAAA,CAAQ,KAAA,KAAU,UAAU,MAAA,EAAQ;AACvD,QAAA;AAAA,MACF;AACA,MAAA,IACE,QAAQ,OAAA,IACR,CAACA,oBAAU,QAAA,EAAU,MAAA,EAAQ,QAAQ,OAAA,EAAS;AAAA,QAC5C,MAAA,EAAQ,IAAA;AAAA,QACR,GAAA,EAAK;AAAA,OACN,CAAA,EACD;AACA,QAAA;AAAA,MACF;AACA,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,OAAO,KAAA;AAAA,EACT;AAAA,EAEQ,WAAA,CAAY,QAAgB,QAAA,EAAwC;AAC1E,IAAA,IAAI,CAAC,QAAA,EAAU;AACb,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,KAAA,MAAW,WAAW,QAAA,EAAU;AAC9B,MAAA,IAAI,QAAQ,IAAA,EAAM,WAAA,OAAkB,OAAA,CAAQ,IAAA,CAAK,aAAY,EAAG;AAC9D,QAAA;AAAA,MACF;AAEA,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,OAAO,KAAA;AAAA,EACT;AACF;AAEA,SAAS,aAAA,CAAc,MAAc,MAAA,EAAwB;AAC3D,EAAA,IAAI,SAAS,MAAA,EAAQ;AACnB,IAAA,OAAO,MAAA;AAAA,EACT;AAEA,EAAA,OAAOC,qBAAA,CAAK,QAAQ,MAAM,CAAA;AAC5B;;;;"}
+{"version":3,"file":"CatalogRules.cjs.js","sources":["../../src/ingestion/CatalogRules.ts"],"sourcesContent":["/*\n * Copyright 2020 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Config } from '@backstage/config';\nimport { Entity } from '@backstage/catalog-model';\nimport path from 'path';\nimport { LocationSpec } from '@backstage/plugin-catalog-common';\nimport { minimatch } from 'minimatch';\nimport { z } from 'zod';\n\n/**\n * Rules to apply to catalog entities.\n *\n * An undefined list of matchers means match all, an empty list of matchers means match none.\n */\nexport type CatalogRule = {\n  allow: CatalogRuleAllow[];\n  locations?: Array<{\n    exact?: string;\n    type: string;\n    pattern?: string;\n  }>;\n};\n\ntype CatalogRuleAllow = {\n  kind: string;\n  'spec.type'?: string;\n};\n\n/**\n * Decides whether an entity from a given location is allowed to enter the\n * catalog, according to some rule set.\n */\nexport type CatalogRulesEnforcer = {\n  isAllowed(entity: Entity, location: LocationSpec): boolean;\n};\n\nconst allowRuleParser = z.array(\n  z\n    .object({\n      kind: z.string(),\n      'spec.type': z.string().optional(),\n    })\n    .or(z.string())\n    .transform(val => (typeof val === 'string' ? { kind: val } : val)),\n);\n\n/**\n * Implements the default catalog rule set, consuming the config keys\n * `catalog.rules` and `catalog.locations.[].rules`.\n */\nexport class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer {\n  /**\n   * Default rules used by the catalog.\n   *\n   * Denies any location from specifying user or group entities.\n   */\n  static readonly defaultRules: CatalogRule[] = [\n    {\n      allow: ['Component', 'API', 'Location'].map(kind => ({ kind })),\n    },\n  ];\n\n  /**\n   * Loads catalog rules from config.\n   *\n   * This reads `catalog.rules` and defaults to the default rules if no value is present.\n   * The value of the config should be a list of config objects, each with a single `allow`\n   * field which in turn is a list of entity kinds to allow.\n   *\n   * If there is no matching rule to allow an ingested entity, it will be rejected by the catalog.\n   *\n   * It also reads in rules from `catalog.locations`, where each location can have a list\n   * of rules for that specific location, specified in a `rules` field.\n   *\n   * For example:\n   *\n   * ```yaml\n   * catalog:\n   *   rules:\n   *   - allow: [Component, API]\n   *   - allow:\n   *     - kind: Resource\n   *       'spec.type': database\n   *   - allow: [Template]\n   *     locations:\n   *       - type: url\n   *         pattern: https://github.com/org/*\\/blob/master/template.yaml\n   *   - allow: [Location]\n   *     locations:\n   *       - type: url\n   *         pattern: https://github.com/org/repo/blob/master/location.yaml\n   *\n   *   locations:\n   *   - type: url\n   *     target: https://github.com/org/repo/blob/master/users.yaml\n   *     rules:\n   *       - allow: [User, Group]\n   *   - type: url\n   *     target: https://github.com/org/repo/blob/master/systems.yaml\n   *     rules:\n   *       - allow: [System]\n   * ```\n   */\n  static fromConfig(config: Config) {\n    const rules = new Array<CatalogRule>();\n\n    if (config.has('catalog.rules')) {\n      const globalRules = config\n        .getConfigArray('catalog.rules')\n        .map(ruleConf => ({\n          allow: allowRuleParser.parse(ruleConf.get('allow')),\n          locations: ruleConf\n            .getOptionalConfigArray('locations')\n            ?.map(locationConfig => {\n              const location = {\n                pattern: locationConfig.getOptionalString('pattern'),\n                type: locationConfig.getString('type'),\n                exact: locationConfig.getOptionalString('exact'),\n              };\n              if (location.pattern && location.exact) {\n                throw new Error(\n                  'A catalog rule location cannot have both exact and pattern values',\n                );\n              }\n              return location;\n            }),\n        }));\n      rules.push(...globalRules);\n    } else {\n      rules.push(...DefaultCatalogRulesEnforcer.defaultRules);\n    }\n\n    if (config.has('catalog.locations')) {\n      const locationRules = config\n        .getConfigArray('catalog.locations')\n        .flatMap(locConf => {\n          if (!locConf.has('rules')) {\n            return [];\n          }\n          const type = locConf.getString('type');\n          const exact = resolveTarget(type, locConf.getString('target'));\n\n          return locConf.getConfigArray('rules').map(ruleConf => ({\n            allow: ruleConf.getStringArray('allow').map(kind => ({ kind })),\n            locations: [{ type, exact }],\n          }));\n        });\n\n      rules.push(...locationRules);\n    }\n\n    return new DefaultCatalogRulesEnforcer(rules);\n  }\n\n  constructor(private readonly rules: CatalogRule[]) {}\n\n  /**\n   * Checks whether a specific entity/location combination is allowed\n   * according to the configured rules.\n   */\n  isAllowed(entity: Entity, location: LocationSpec) {\n    for (const rule of this.rules) {\n      if (!this.matchLocation(location, rule.locations)) {\n        continue;\n      }\n\n      if (this.matchEntity(entity, rule.allow)) {\n        return true;\n      }\n    }\n\n    return false;\n  }\n\n  private matchLocation(\n    location: LocationSpec,\n    matchers?: { exact?: string; type: string; pattern?: string }[],\n  ): boolean {\n    if (!matchers) {\n      return true;\n    }\n\n    for (const matcher of matchers) {\n      if (matcher.type !== location?.type) {\n        continue;\n      }\n      if (matcher.exact && matcher.exact !== location?.target) {\n        continue;\n      }\n      if (\n        matcher.pattern &&\n        !minimatch(location?.target, matcher.pattern, {\n          nocase: true,\n          dot: true,\n        })\n      ) {\n        continue;\n      }\n      return true;\n    }\n\n    return false;\n  }\n\n  private matchEntity(entity: Entity, matchers?: CatalogRuleAllow[]): boolean {\n    if (!matchers) {\n      return true;\n    }\n\n    for (const matcher of matchers) {\n      if (\n        entity.kind?.toLocaleLowerCase('en-US') !==\n        matcher.kind.toLocaleLowerCase('en-US')\n      ) {\n        continue;\n      }\n\n      if (matcher['spec.type']) {\n        if (typeof entity.spec?.type !== 'string') {\n          continue;\n        }\n        if (\n          matcher['spec.type'].toLocaleLowerCase('en-US') !==\n          entity.spec.type.toLocaleLowerCase('en-US')\n        ) {\n          continue;\n        }\n      }\n\n      return true;\n    }\n\n    return false;\n  }\n}\n\nfunction resolveTarget(type: string, target: string): string {\n  if (type !== 'file') {\n    return target;\n  }\n\n  return path.resolve(target);\n}\n"],"names":["z","minimatch","path"],"mappings":";;;;;;;;;;AAkDA,MAAM,kBAAkBA,KAAA,CAAE,KAAA;AAAA,EACxBA,MACG,MAAA,CAAO;AAAA,IACN,IAAA,EAAMA,MAAE,MAAA,EAAO;AAAA,IACf,WAAA,EAAaA,KAAA,CAAE,MAAA,EAAO,CAAE,QAAA;AAAS,GAClC,CAAA,CACA,EAAA,CAAGA,KAAA,CAAE,MAAA,EAAQ,CAAA,CACb,SAAA,CAAU,CAAA,GAAA,KAAQ,OAAO,QAAQ,QAAA,GAAW,EAAE,IAAA,EAAM,GAAA,KAAQ,GAAI;AACrE,CAAA;AAMO,MAAM,2BAAA,CAA4D;AAAA,EAwGvE,YAA6B,KAAA,EAAsB;AAAtB,IAAA,IAAA,CAAA,KAAA,GAAA,KAAA;AAAA,EAAuB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAlGpD,OAAgB,YAAA,GAA8B;AAAA,IAC5C;AAAA,MACE,KAAA,EAAO,CAAC,WAAA,EAAa,KAAA,EAAO,UAAU,EAAE,GAAA,CAAI,CAAA,IAAA,MAAS,EAAE,IAAA,EAAK,CAAE;AAAA;AAChE,GACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EA2CA,OAAO,WAAW,MAAA,EAAgB;AAChC,IAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAmB;AAErC,IAAA,IAAI,MAAA,CAAO,GAAA,CAAI,eAAe,CAAA,EAAG;AAC/B,MAAA,MAAM,cAAc,MAAA,CACjB,cAAA,CAAe,eAAe,CAAA,CAC9B,IAAI,CAAA,QAAA,MAAa;AAAA,QAChB,OAAO,eAAA,CAAgB,KAAA,CAAM,QAAA,CAAS,GAAA,CAAI,OAAO,CAAC,CAAA;AAAA,QAClD,WAAW,QAAA,CACR,sBAAA,CAAuB,WAAW,CAAA,EACjC,IAAI,CAAA,cAAA,KAAkB;AACtB,UAAA,MAAM,QAAA,GAAW;AAAA,YACf,OAAA,EAAS,cAAA,CAAe,iBAAA,CAAkB,SAAS,CAAA;AAAA,YACnD,IAAA,EAAM,cAAA,CAAe,SAAA,CAAU,MAAM,CAAA;AAAA,YACrC,KAAA,EAAO,cAAA,CAAe,iBAAA,CAAkB,OAAO;AAAA,WACjD;AACA,UAAA,IAAI,QAAA,CAAS,OAAA,IAAW,QAAA,CAAS,KAAA,EAAO;AACtC,YAAA,MAAM,IAAI,KAAA;AAAA,cACR;AAAA,aACF;AAAA,UACF;AACA,UAAA,OAAO,QAAA;AAAA,QACT,CAAC;AAAA,OACL,CAAE,CAAA;AACJ,MAAA,KAAA,CAAM,IAAA,CAAK,GAAG,WAAW,CAAA;AAAA,IAC3B,CAAA,MAAO;AACL,MAAA,KAAA,CAAM,IAAA,CAAK,GAAG,2BAAA,CAA4B,YAAY,CAAA;AAAA,IACxD;AAEA,IAAA,IAAI,MAAA,CAAO,GAAA,CAAI,mBAAmB,CAAA,EAAG;AACnC,MAAA,MAAM,gBAAgB,MAAA,CACnB,cAAA,CAAe,mBAAmB,CAAA,CAClC,QAAQ,CAAA,OAAA,KAAW;AAClB,QAAA,IAAI,CAAC,OAAA,CAAQ,GAAA,CAAI,OAAO,CAAA,EAAG;AACzB,UAAA,OAAO,EAAC;AAAA,QACV;AACA,QAAA,MAAM,IAAA,GAAO,OAAA,CAAQ,SAAA,CAAU,MAAM,CAAA;AACrC,QAAA,MAAM,QAAQ,aAAA,CAAc,IAAA,EAAM,OAAA,CAAQ,SAAA,CAAU,QAAQ,CAAC,CAAA;AAE7D,QAAA,OAAO,OAAA,CAAQ,cAAA,CAAe,OAAO,CAAA,CAAE,IAAI,CAAA,QAAA,MAAa;AAAA,UACtD,KAAA,EAAO,SAAS,cAAA,CAAe,OAAO,EAAE,GAAA,CAAI,CAAA,IAAA,MAAS,EAAE,IAAA,EAAK,CAAE,CAAA;AAAA,UAC9D,SAAA,EAAW,CAAC,EAAE,IAAA,EAAM,OAAO;AAAA,SAC7B,CAAE,CAAA;AAAA,MACJ,CAAC,CAAA;AAEH,MAAA,KAAA,CAAM,IAAA,CAAK,GAAG,aAAa,CAAA;AAAA,IAC7B;AAEA,IAAA,OAAO,IAAI,4BAA4B,KAAK,CAAA;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,SAAA,CAAU,QAAgB,QAAA,EAAwB;AAChD,IAAA,KAAA,MAAW,IAAA,IAAQ,KAAK,KAAA,EAAO;AAC7B,MAAA,IAAI,CAAC,IAAA,CAAK,aAAA,CAAc,QAAA,EAAU,IAAA,CAAK,SAAS,CAAA,EAAG;AACjD,QAAA;AAAA,MACF;AAEA,MAAA,IAAI,IAAA,CAAK,WAAA,CAAY,MAAA,EAAQ,IAAA,CAAK,KAAK,CAAA,EAAG;AACxC,QAAA,OAAO,IAAA;AAAA,MACT;AAAA,IACF;AAEA,IAAA,OAAO,KAAA;AAAA,EACT;AAAA,EAEQ,aAAA,CACN,UACA,QAAA,EACS;AACT,IAAA,IAAI,CAAC,QAAA,EAAU;AACb,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,KAAA,MAAW,WAAW,QAAA,EAAU;AAC9B,MAAA,IAAI,OAAA,CAAQ,IAAA,KAAS,QAAA,EAAU,IAAA,EAAM;AACnC,QAAA;AAAA,MACF;AACA,MAAA,IAAI,OAAA,CAAQ,KAAA,IAAS,OAAA,CAAQ,KAAA,KAAU,UAAU,MAAA,EAAQ;AACvD,QAAA;AAAA,MACF;AACA,MAAA,IACE,QAAQ,OAAA,IACR,CAACC,oBAAU,QAAA,EAAU,MAAA,EAAQ,QAAQ,OAAA,EAAS;AAAA,QAC5C,MAAA,EAAQ,IAAA;AAAA,QACR,GAAA,EAAK;AAAA,OACN,CAAA,EACD;AACA,QAAA;AAAA,MACF;AACA,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,OAAO,KAAA;AAAA,EACT;AAAA,EAEQ,WAAA,CAAY,QAAgB,QAAA,EAAwC;AAC1E,IAAA,IAAI,CAAC,QAAA,EAAU;AACb,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,KAAA,MAAW,WAAW,QAAA,EAAU;AAC9B,MAAA,IACE,MAAA,CAAO,MAAM,iBAAA,CAAkB,OAAO,MACtC,OAAA,CAAQ,IAAA,CAAK,iBAAA,CAAkB,OAAO,CAAA,EACtC;AACA,QAAA;AAAA,MACF;AAEA,MAAA,IAAI,OAAA,CAAQ,WAAW,CAAA,EAAG;AACxB,QAAA,IAAI,OAAO,MAAA,CAAO,IAAA,EAAM,IAAA,KAAS,QAAA,EAAU;AACzC,UAAA;AAAA,QACF;AACA,QAAA,IACE,OAAA,CAAQ,WAAW,CAAA,CAAE,iBAAA,CAAkB,OAAO,CAAA,KAC9C,MAAA,CAAO,IAAA,CAAK,IAAA,CAAK,iBAAA,CAAkB,OAAO,CAAA,EAC1C;AACA,UAAA;AAAA,QACF;AAAA,MACF;AAEA,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,OAAO,KAAA;AAAA,EACT;AACF;AAEA,SAAS,aAAA,CAAc,MAAc,MAAA,EAAwB;AAC3D,EAAA,IAAI,SAAS,MAAA,EAAQ;AACnB,IAAA,OAAO,MAAA;AAAA,EACT;AAEA,EAAA,OAAOC,qBAAA,CAAK,QAAQ,MAAM,CAAA;AAC5B;;;;"}

package/dist/processing/DefaultCatalogProcessingEngine.cjs.js

@@ -34,7 +34,7 @@ class DefaultCatalogProcessingEngine {
   orphanCleanupIntervalMs;
   onProcessingError;
   tracker;
-  eventBroker;
+  events;
   stopFunc;
   constructor(options) {
     this.config = options.config;
@@ -49,7 +49,7 @@ class DefaultCatalogProcessingEngine {
     this.orphanCleanupIntervalMs = options.orphanCleanupIntervalMs ?? 3e4;
     this.onProcessingError = options.onProcessingError;
     this.tracker = options.tracker ?? progressTracker();
-    this.eventBroker = options.eventBroker;
+    this.events = options.events;
     this.stopFunc = void 0;
   }
   async start() {
@@ -128,7 +128,7 @@ class DefaultCatalogProcessingEngine {
             }
             const location = unprocessedEntity?.metadata?.annotations?.[catalogModel.ANNOTATION_LOCATION];
             if (result.errors.length) {
-              this.eventBroker?.publish({
+              this.events.publish({
                 topic: constants.CATALOG_ERRORS_TOPIC,
                 eventPayload: {
                   entity: entityRef,
@@ -260,22 +260,16 @@ class DefaultCatalogProcessingEngine {
         this.logger.warn(`Failed to delete orphaned entities`, error);
       }
     };
-    if (this.scheduler) {
-      const abortController = new AbortController();
-      this.scheduler.scheduleTask({
-        id: "catalog_orphan_cleanup",
-        frequency: { milliseconds: this.orphanCleanupIntervalMs },
-        timeout: { milliseconds: this.orphanCleanupIntervalMs * 0.8 },
-        fn: runOnce,
-        signal: abortController.signal
-      });
-      return () => {
-        abortController.abort();
-      };
-    }
-    const intervalKey = setInterval(runOnce, this.orphanCleanupIntervalMs);
+    const abortController = new AbortController();
+    this.scheduler.scheduleTask({
+      id: "catalog_orphan_cleanup",
+      frequency: { milliseconds: this.orphanCleanupIntervalMs },
+      timeout: { milliseconds: this.orphanCleanupIntervalMs * 0.8 },
+      fn: runOnce,
+      signal: abortController.signal
+    });
     return () => {
-      clearInterval(intervalKey);
+      abortController.abort();
     };
   }
 }