@backstage/plugin-catalog-backend 3.1.1 → 3.1.2

package/CHANGELOG.md

@@ -1,10 +1,13 @@
 # @backstage/plugin-catalog-backend
 
-## 3.1.1
+## 3.1.2
 
 ### Patch Changes
 
-- 27b6849: Moved catalog processor and provider disabling and priorities under own config objects.
+- 9890488: Internal refactor to remove remnants of the old backend system
+- 6493c98: Log before provider-orphaning eviction happens
+- 2aaf01a: Fix for duplicate search results in entity-facets API call
+- e489661: Moved catalog processor and provider disabling and priorities under own config objects.
 
   This is due to issue with some existing providers, such as GitHub, using array syntax for the provider configuration.
 
@@ -26,8 +29,92 @@
         disabled: true
   ```
 
+- 77516c5: Added new `catalog:validate-entity` action to actions registry.
+
+  This action can be used to validate entities against the software catalog.
+  This is useful for validating `catalog-info.yaml` file changes locally using the
+  Backstage MCP server.
+
 - Updated dependencies
-  - @backstage/config@1.3.4
+  - @backstage/integration@1.18.1
+  - @backstage/config@1.3.5
+  - @backstage/backend-openapi-utils@0.6.2
+  - @backstage/backend-plugin-api@1.4.4
+  - @backstage/plugin-catalog-common@1.1.6
+  - @backstage/plugin-catalog-node@1.19.1
+  - @backstage/plugin-events-node@0.4.16
+  - @backstage/plugin-permission-common@0.9.2
+  - @backstage/plugin-permission-node@0.10.5
+
+## 3.1.2-next.2
+
+### Patch Changes
+
+- 6493c98: Log before provider-orphaning eviction happens
+- 77516c5: Added new `catalog:validate-entity` action to actions registry.
+
+  This action can be used to validate entities against the software catalog.
+  This is useful for validating `catalog-info.yaml` file changes locally using the
+  Backstage MCP server.
+
+## 3.1.1-next.1
+
+### Patch Changes
+
+- Updated dependencies
+  - @backstage/config@1.3.4-next.0
+  - @backstage/integration@1.18.1-next.1
+  - @backstage/backend-plugin-api@1.4.4-next.0
+  - @backstage/plugin-permission-common@0.9.2-next.0
+  - @backstage/plugin-permission-node@0.10.5-next.0
+  - @backstage/backend-openapi-utils@0.6.2-next.0
+  - @backstage/plugin-catalog-node@1.19.1-next.0
+  - @backstage/plugin-events-node@0.4.16-next.0
+  - @backstage/plugin-catalog-common@1.1.6-next.0
+  - @backstage/catalog-client@1.12.0
+
+## 3.1.1-next.0
+
+### Patch Changes
+
+- 9890488: Internal refactor to remove remnants of the old backend system
+- 2aaf01a: Fix for duplicate search results in entity-facets API call
+- e489661: Moved catalog processor and provider disabling and priorities under own config objects.
+
+  This is due to issue with some existing providers, such as GitHub, using array syntax for the provider configuration.
+
+  The new config format is not backwards compatible, so users will need to update their config files. The new format
+  is as follows:
+
+  ```yaml
+  catalog:
+    providerOptions:
+      providerA:
+        disabled: false
+      providerB:
+        disabled: true
+    processorOptions:
+      processorA:
+        disabled: false
+        priority: 10
+      processorB:
+        disabled: true
+  ```
+
+- Updated dependencies
+  - @backstage/integration@1.18.1-next.0
+  - @backstage/plugin-permission-node@0.10.4
+  - @backstage/backend-openapi-utils@0.6.1
+  - @backstage/backend-plugin-api@1.4.3
+  - @backstage/catalog-client@1.12.0
+  - @backstage/catalog-model@1.7.5
+  - @backstage/config@1.3.3
+  - @backstage/errors@1.2.7
+  - @backstage/types@1.2.2
+  - @backstage/plugin-catalog-common@1.1.5
+  - @backstage/plugin-catalog-node@1.19.0
+  - @backstage/plugin-events-node@0.4.15
+  - @backstage/plugin-permission-common@0.9.1
 
 ## 3.1.0
 

package/dist/actions/createValidateEntityAction.cjs.js

@@ -0,0 +1,77 @@
+'use strict';
+
+var yaml = require('yaml');
+
+function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e : { default: e }; }
+
+var yaml__default = /*#__PURE__*/_interopDefaultCompat(yaml);
+
+const createValidateEntityAction = (options) => {
+  const { actionsRegistry, catalog } = options;
+  actionsRegistry.register({
+    name: "validate-entity",
+    title: "Validate Catalog Entity",
+    description: `
+      This action can be used to validate catalog-info.yaml file contents meant to be used with the software catalog.
+      It checks both that the YAML syntax is correct, and that the entity content is valid according to the catalog's rules.
+      `,
+    attributes: {
+      readOnly: true,
+      idempotent: true,
+      destructive: false
+    },
+    schema: {
+      input: (z) => z.object({
+        entity: z.string().describe("Entity YAML content to validate"),
+        location: z.string().url().optional().describe("Location to validate")
+      }),
+      output: (z) => z.object({
+        isValid: z.boolean().describe("Whether the entity is valid"),
+        isValidYaml: z.boolean().describe("Whether the YAML syntax is valid"),
+        errors: z.array(z.string()).describe("Array of validation errors"),
+        entity: z.record(z.any()).optional().describe("Parsed entity object if valid")
+      })
+    },
+    action: async ({ input, credentials }) => {
+      const { entity: content, location } = input;
+      try {
+        let entity;
+        try {
+          entity = yaml__default.default.parse(content);
+        } catch (yamlError) {
+          return {
+            output: {
+              isValid: false,
+              isValidYaml: false,
+              errors: [`YAML parsing error: ${yamlError.message}`]
+            }
+          };
+        }
+        const resp = await catalog.validateEntity(
+          entity,
+          location ?? "url:https://localhost/entity-validator",
+          { credentials }
+        );
+        return {
+          output: {
+            isValid: resp.valid,
+            isValidYaml: true,
+            errors: resp.valid ? [] : resp.errors.map((e) => e.message),
+            entity: resp.valid ? entity : void 0
+          }
+        };
+      } catch (error) {
+        return {
+          output: {
+            isValid: false,
+            isValidYaml: false,
+            errors: [`Validation error: ${error.message}`]
+          }
+        };
+      }
+    }
+  });
+};
+
+exports.createValidateEntityAction = createValidateEntityAction;
+//# sourceMappingURL=createValidateEntityAction.cjs.js.map

package/dist/actions/createValidateEntityAction.cjs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createValidateEntityAction.cjs.js","sources":["../../src/actions/createValidateEntityAction.ts"],"sourcesContent":["/*\n * Copyright 2025 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nimport { ActionsRegistryService } from '@backstage/backend-plugin-api/alpha';\nimport yaml from 'yaml';\nimport { CatalogService } from '@backstage/plugin-catalog-node';\n\nexport const createValidateEntityAction = (options: {\n  actionsRegistry: ActionsRegistryService;\n  catalog: CatalogService;\n}) => {\n  const { actionsRegistry, catalog } = options;\n  actionsRegistry.register({\n    name: 'validate-entity',\n    title: 'Validate Catalog Entity',\n    description: `\n      This action can be used to validate catalog-info.yaml file contents meant to be used with the software catalog.\n      It checks both that the YAML syntax is correct, and that the entity content is valid according to the catalog's rules.\n      `,\n    attributes: {\n      readOnly: true,\n      idempotent: true,\n      destructive: false,\n    },\n    schema: {\n      input: z =>\n        z.object({\n          entity: z.string().describe('Entity YAML content to validate'),\n          location: z\n            .string()\n            .url()\n            .optional()\n            .describe('Location to validate'),\n        }),\n      output: z =>\n        z.object({\n          isValid: z.boolean().describe('Whether the entity is valid'),\n          isValidYaml: z.boolean().describe('Whether the YAML syntax is valid'),\n          errors: z.array(z.string()).describe('Array of validation errors'),\n          entity: z\n            .record(z.any())\n            .optional()\n            .describe('Parsed entity object if valid'),\n        }),\n    },\n    action: async ({ input, credentials }) => {\n      const { entity: content, location } = input;\n      try {\n        let entity;\n        try {\n          entity = yaml.parse(content);\n        } catch (yamlError) {\n          return {\n            output: {\n              isValid: false,\n              isValidYaml: false,\n              errors: [`YAML parsing error: ${yamlError.message}`],\n            },\n          };\n        }\n\n        const resp = await catalog.validateEntity(\n          entity,\n          location ?? 'url:https://localhost/entity-validator',\n          { credentials },\n        );\n\n        return {\n          output: {\n            isValid: resp.valid,\n            isValidYaml: true,\n            errors: resp.valid ? [] : resp.errors.map(e => e.message),\n            entity: resp.valid ? entity : undefined,\n          },\n        };\n      } catch (error) {\n        return {\n          output: {\n            isValid: false,\n            isValidYaml: false,\n            errors: [`Validation error: ${error.message}`],\n          },\n        };\n      }\n    },\n  });\n};\n"],"names":["yaml"],"mappings":";;;;;;;;AAmBO,MAAM,0BAAA,GAA6B,CAAC,OAAA,KAGrC;AACJ,EAAA,MAAM,EAAE,eAAA,EAAiB,OAAA,EAAQ,GAAI,OAAA;AACrC,EAAA,eAAA,CAAgB,QAAA,CAAS;AAAA,IACvB,IAAA,EAAM,iBAAA;AAAA,IACN,KAAA,EAAO,yBAAA;AAAA,IACP,WAAA,EAAa;AAAA;AAAA;AAAA,MAAA,CAAA;AAAA,IAIb,UAAA,EAAY;AAAA,MACV,QAAA,EAAU,IAAA;AAAA,MACV,UAAA,EAAY,IAAA;AAAA,MACZ,WAAA,EAAa;AAAA,KACf;AAAA,IACA,MAAA,EAAQ;AAAA,MACN,KAAA,EAAO,CAAA,CAAA,KACL,CAAA,CAAE,MAAA,CAAO;AAAA,QACP,MAAA,EAAQ,CAAA,CAAE,MAAA,EAAO,CAAE,SAAS,iCAAiC,CAAA;AAAA,QAC7D,QAAA,EAAU,EACP,MAAA,EAAO,CACP,KAAI,CACJ,QAAA,EAAS,CACT,QAAA,CAAS,sBAAsB;AAAA,OACnC,CAAA;AAAA,MACH,MAAA,EAAQ,CAAA,CAAA,KACN,CAAA,CAAE,MAAA,CAAO;AAAA,QACP,OAAA,EAAS,CAAA,CAAE,OAAA,EAAQ,CAAE,SAAS,6BAA6B,CAAA;AAAA,QAC3D,WAAA,EAAa,CAAA,CAAE,OAAA,EAAQ,CAAE,SAAS,kCAAkC,CAAA;AAAA,QACpE,MAAA,EAAQ,EAAE,KAAA,CAAM,CAAA,CAAE,QAAQ,CAAA,CAAE,SAAS,4BAA4B,CAAA;AAAA,QACjE,MAAA,EAAQ,CAAA,CACL,MAAA,CAAO,CAAA,CAAE,GAAA,EAAK,CAAA,CACd,QAAA,EAAS,CACT,QAAA,CAAS,+BAA+B;AAAA,OAC5C;AAAA,KACL;AAAA,IACA,MAAA,EAAQ,OAAO,EAAE,KAAA,EAAO,aAAY,KAAM;AACxC,MAAA,MAAM,EAAE,MAAA,EAAQ,OAAA,EAAS,QAAA,EAAS,GAAI,KAAA;AACtC,MAAA,IAAI;AACF,QAAA,IAAI,MAAA;AACJ,QAAA,IAAI;AACF,UAAA,MAAA,GAASA,qBAAA,CAAK,MAAM,OAAO,CAAA;AAAA,QAC7B,SAAS,SAAA,EAAW;AAClB,UAAA,OAAO;AAAA,YACL,MAAA,EAAQ;AAAA,cACN,OAAA,EAAS,KAAA;AAAA,cACT,WAAA,EAAa,KAAA;AAAA,cACb,MAAA,EAAQ,CAAC,CAAA,oBAAA,EAAuB,SAAA,CAAU,OAAO,CAAA,CAAE;AAAA;AACrD,WACF;AAAA,QACF;AAEA,QAAA,MAAM,IAAA,GAAO,MAAM,OAAA,CAAQ,cAAA;AAAA,UACzB,MAAA;AAAA,UACA,QAAA,IAAY,wCAAA;AAAA,UACZ,EAAE,WAAA;AAAY,SAChB;AAEA,QAAA,OAAO;AAAA,UACL,MAAA,EAAQ;AAAA,YACN,SAAS,IAAA,CAAK,KAAA;AAAA,YACd,WAAA,EAAa,IAAA;AAAA,YACb,MAAA,EAAQ,IAAA,CAAK,KAAA,GAAQ,EAAC,GAAI,KAAK,MAAA,CAAO,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,OAAO,CAAA;AAAA,YACxD,MAAA,EAAQ,IAAA,CAAK,KAAA,GAAQ,MAAA,GAAS,KAAA;AAAA;AAChC,SACF;AAAA,MACF,SAAS,KAAA,EAAO;AACd,QAAA,OAAO;AAAA,UACL,MAAA,EAAQ;AAAA,YACN,OAAA,EAAS,KAAA;AAAA,YACT,WAAA,EAAa,KAAA;AAAA,YACb,MAAA,EAAQ,CAAC,CAAA,kBAAA,EAAqB,KAAA,CAAM,OAAO,CAAA,CAAE;AAAA;AAC/C,SACF;AAAA,MACF;AAAA,IACF;AAAA,GACD,CAAA;AACH;;;;"}

package/dist/actions/index.cjs.js

@@ -0,0 +1,12 @@
+'use strict';
+
+var createGetCatalogEntityAction = require('./createGetCatalogEntityAction.cjs.js');
+var createValidateEntityAction = require('./createValidateEntityAction.cjs.js');
+
+const createCatalogActions = (options) => {
+  createGetCatalogEntityAction.createGetCatalogEntityAction(options);
+  createValidateEntityAction.createValidateEntityAction(options);
+};
+
+exports.createCatalogActions = createCatalogActions;
+//# sourceMappingURL=index.cjs.js.map

package/dist/actions/index.cjs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.cjs.js","sources":["../../src/actions/index.ts"],"sourcesContent":["/*\n * Copyright 2025 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nimport { ActionsRegistryService } from '@backstage/backend-plugin-api/alpha';\nimport { CatalogService } from '@backstage/plugin-catalog-node';\nimport { createGetCatalogEntityAction } from './createGetCatalogEntityAction.ts';\nimport { createValidateEntityAction } from './createValidateEntityAction.ts';\n\nexport const createCatalogActions = (options: {\n  actionsRegistry: ActionsRegistryService;\n  catalog: CatalogService;\n}) => {\n  createGetCatalogEntityAction(options);\n  createValidateEntityAction(options);\n};\n"],"names":["createGetCatalogEntityAction","createValidateEntityAction"],"mappings":";;;;;AAoBO,MAAM,oBAAA,GAAuB,CAAC,OAAA,KAG/B;AACJ,EAAAA,yDAAA,CAA6B,OAAO,CAAA;AACpC,EAAAC,qDAAA,CAA2B,OAAO,CAAA;AACpC;;;;"}

package/dist/database/DefaultProcessingDatabase.cjs.js

@@ -222,7 +222,7 @@ class DefaultProcessingDatabase {
         this.options.logger.warn(
           `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`
         );
-        if (this.options.eventBroker && locationKey) {
+        if (locationKey) {
           const eventParams = {
             topic: constants.CATALOG_CONFLICTS_TOPIC,
             eventPayload: {
@@ -233,7 +233,7 @@ class DefaultProcessingDatabase {
               lastConflictAt: luxon.DateTime.now().toISO()
             }
           };
-          await this.options.eventBroker?.publish(eventParams);
+          await this.options.events.publish(eventParams);
         }
       }
     }

package/dist/database/DefaultProcessingDatabase.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultProcessingDatabase.cjs.js","sources":["../../src/database/DefaultProcessingDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError } from '@backstage/errors';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { ProcessingIntervalFunction } from '../processing/refresh';\nimport { rethrowError, timestampToDateTime } from './conversion';\nimport { initDatabaseMetrics } from './metrics';\nimport {\n  DbRefreshKeysRow,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n} from './tables';\nimport {\n  GetProcessableEntitiesResult,\n  ListParentsOptions,\n  ListParentsResult,\n  ProcessingDatabase,\n  RefreshStateItem,\n  Transaction,\n  UpdateEntityCacheOptions,\n  UpdateProcessedEntityOptions,\n} from './types';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { generateStableHash, generateTargetKey } from './util';\nimport {\n  EventBroker,\n  EventParams,\n  EventsService,\n} from '@backstage/plugin-events-node';\nimport { DateTime } from 'luxon';\nimport { CATALOG_CONFLICTS_TOPIC } from '../constants';\nimport { CatalogConflictEventPayload } from '../catalog/types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProcessingDatabase implements ProcessingDatabase {\n  constructor(\n    private readonly options: {\n      database: Knex;\n      logger: LoggerService;\n      refreshInterval: ProcessingIntervalFunction;\n      eventBroker?: EventBroker | EventsService;\n    },\n  ) {\n    initDatabaseMetrics(options.database);\n  }\n\n  async updateProcessedEntity(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<{ previous: { relations: DbRelationsRow[] } }> {\n    const tx = txOpaque as Knex.Transaction;\n    const {\n      id,\n      processedEntity,\n      resultHash,\n      errors,\n      relations,\n      deferredEntities,\n      refreshKeys,\n      locationKey,\n    } = options;\n    const configClient = tx.client.config.client;\n    const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        processed_entity: JSON.stringify(processedEntity),\n        result_hash: resultHash,\n        errors,\n        location_key: locationKey,\n      })\n      .where('entity_id', id)\n      .andWhere(inner => {\n        if (!locationKey) {\n          return inner.whereNull('location_key');\n        }\n        return inner\n          .where('location_key', locationKey)\n          .orWhereNull('location_key');\n      });\n    if (refreshResult === 0) {\n      throw new ConflictError(\n        `Conflicting write of processing result for ${id} with location key '${locationKey}'`,\n      );\n    }\n    const sourceEntityRef = stringifyEntityRef(processedEntity);\n\n    // Schedule all deferred entities for future processing.\n    await this.addUnprocessedEntities(tx, {\n      entities: deferredEntities,\n      sourceEntityRef,\n    });\n\n    // Delete old relations\n    // NOTE(freben): knex implemented support for returning() on update queries for sqlite, but at the current time of writing (Sep 2022) not for delete() queries.\n    let previousRelationRows: DbRelationsRow[];\n    if (configClient.includes('sqlite3') || configClient.includes('mysql')) {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .select('*')\n        .where({ originating_entity_id: id });\n      await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete();\n    } else {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete()\n        .returning('*');\n    }\n\n    // Batch insert new relations\n    const relationRows: DbRelationsRow[] = relations.map(\n      ({ source, target, type }) => ({\n        originating_entity_id: id,\n        source_entity_ref: stringifyEntityRef(source),\n        target_entity_ref: stringifyEntityRef(target),\n        type,\n      }),\n    );\n\n    await tx.batchInsert(\n      'relations',\n      this.deduplicateRelations(relationRows),\n      BATCH_SIZE,\n    );\n\n    // Delete old refresh keys\n    await tx<DbRefreshKeysRow>('refresh_keys')\n      .where({ entity_id: id })\n      .delete();\n\n    // Insert the refresh keys for the processed entity\n    await tx.batchInsert(\n      'refresh_keys',\n      refreshKeys.map(k => ({\n        entity_id: id,\n        key: generateTargetKey(k.key),\n      })),\n      BATCH_SIZE,\n    );\n\n    return {\n      previous: {\n        relations: previousRelationRows,\n      },\n    };\n  }\n\n  async updateProcessedEntityErrors(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, errors, resultHash } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        errors,\n        result_hash: resultHash,\n      })\n      .where('entity_id', id);\n  }\n\n  async updateEntityCache(\n    txOpaque: Transaction,\n    options: UpdateEntityCacheOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, state } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({ cache: JSON.stringify(state ?? {}) })\n      .where('entity_id', id);\n  }\n\n  async getProcessableEntities(\n    maybeTx: Transaction | Knex,\n    request: { processBatchSize: number },\n  ): Promise<GetProcessableEntitiesResult> {\n    const knex = maybeTx as Knex.Transaction | Knex;\n\n    let itemsQuery = knex<DbRefreshStateRow>('refresh_state').select([\n      'entity_id',\n      'entity_ref',\n      'unprocessed_entity',\n      'result_hash',\n      'cache',\n      'errors',\n      'location_key',\n      'next_update_at',\n    ]);\n\n    // This avoids duplication of work because of race conditions and is\n    // also fast because locked rows are ignored rather than blocking.\n    // It's only available in MySQL and PostgreSQL\n    if (['mysql', 'mysql2', 'pg'].includes(knex.client.config.client)) {\n      itemsQuery = itemsQuery.forUpdate().skipLocked();\n    }\n\n    const items = await itemsQuery\n      .where('next_update_at', '<=', knex.fn.now())\n      .limit(request.processBatchSize)\n      .orderBy('next_update_at', 'asc');\n\n    const interval = this.options.refreshInterval();\n\n    const nextUpdateAt = (refreshInterval: number) => {\n      if (knex.client.config.client.includes('sqlite3')) {\n        return knex.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);\n      } else if (knex.client.config.client.includes('mysql')) {\n        return knex.raw(`now() + interval ${refreshInterval} second`);\n      }\n      return knex.raw(`now() + interval '${refreshInterval} seconds'`);\n    };\n\n    await knex<DbRefreshStateRow>('refresh_state')\n      .whereIn(\n        'entity_ref',\n        items.map(i => i.entity_ref),\n      )\n      .update({\n        next_update_at: nextUpdateAt(interval),\n      });\n\n    return {\n      items: items.map(\n        i =>\n          ({\n            id: i.entity_id,\n            entityRef: i.entity_ref,\n            unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity,\n            resultHash: i.result_hash || '',\n            nextUpdateAt: timestampToDateTime(i.next_update_at),\n            state: i.cache ? JSON.parse(i.cache) : undefined,\n            errors: i.errors,\n            locationKey: i.location_key,\n          } satisfies RefreshStateItem),\n      ),\n    };\n  }\n\n  async listParents(\n    txOpaque: Transaction,\n    options: ListParentsOptions,\n  ): Promise<ListParentsResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .whereIn('target_entity_ref', options.entityRefs)\n      .select();\n\n    const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);\n\n    return { entityRefs };\n  }\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] {\n    return lodash.uniqBy(\n      rows,\n      r => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`,\n    );\n  }\n\n  /**\n   * Add a set of deferred entities for processing.\n   * The entities will be added at the front of the processing queue.\n   */\n  private async addUnprocessedEntities(\n    txOpaque: Transaction,\n    options: {\n      sourceEntityRef: string;\n      entities: DeferredEntity[];\n    },\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n\n    // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards\n    const stateReferences = new Array<string>();\n\n    // Upsert all of the unprocessed entities into the refresh_state table, by\n    // their entity ref.\n    for (const { entity, locationKey } of options.entities) {\n      const entityRef = stringifyEntityRef(entity);\n      const hash = generateStableHash(entity);\n\n      const updated = await updateUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n      });\n      if (updated) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      const inserted = await insertUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n        logger: this.options.logger,\n      });\n      if (inserted) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      // If the row can't be inserted, we have a conflict, but it could be either\n      // because of a conflicting locationKey or a race with another instance, so check\n      // whether the conflicting entity has the same entityRef but a different locationKey\n      const conflictingKey = await checkLocationKeyConflict({\n        tx,\n        entityRef,\n        locationKey,\n      });\n      if (conflictingKey) {\n        this.options.logger.warn(\n          `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n        );\n        if (this.options.eventBroker && locationKey) {\n          const eventParams: EventParams<CatalogConflictEventPayload> = {\n            topic: CATALOG_CONFLICTS_TOPIC,\n            eventPayload: {\n              unprocessedEntity: entity,\n              entityRef,\n              newLocationKey: locationKey,\n              existingLocationKey: conflictingKey,\n              lastConflictAt: DateTime.now().toISO()!,\n            },\n          };\n          await this.options.eventBroker?.publish(eventParams);\n        }\n      }\n    }\n\n    // Lastly, replace refresh state references for the originating entity and any successfully added entities\n    await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n      // Remove all existing references from the originating entity\n      .where({ source_entity_ref: options.sourceEntityRef })\n      // And remove any existing references to entities that we're inserting new references for\n      .orWhereIn('target_entity_ref', stateReferences)\n      .delete();\n    await tx.batchInsert(\n      'refresh_state_references',\n      stateReferences.map(entityRef => ({\n        source_entity_ref: options.sourceEntityRef,\n        target_entity_ref: entityRef,\n      })),\n      BATCH_SIZE,\n    );\n  }\n}\n"],"names":["initDatabaseMetrics","errors","ConflictError","stringifyEntityRef","generateTargetKey","timestampToDateTime","rethrowError","lodash","generateStableHash","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","CATALOG_CONFLICTS_TOPIC","DateTime"],"mappings":";;;;;;;;;;;;;;;;;;AA0DA,MAAM,UAAA,GAAa,EAAA;AAEZ,MAAM,yBAAA,CAAwD;AAAA,EACnE,YACmB,OAAA,EAMjB;AANiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAOjB,IAAAA,2BAAA,CAAoB,QAAQ,QAAQ,CAAA;AAAA,EACtC;AAAA,EAEA,MAAM,qBAAA,CACJ,QAAA,EACA,OAAA,EACwD;AACxD,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM;AAAA,MACJ,EAAA;AAAA,MACA,eAAA;AAAA,MACA,UAAA;AAAA,cACAC,QAAA;AAAA,MACA,SAAA;AAAA,MACA,gBAAA;AAAA,MACA,WAAA;AAAA,MACA;AAAA,KACF,GAAI,OAAA;AACJ,IAAA,MAAM,YAAA,GAAe,EAAA,CAAG,MAAA,CAAO,MAAA,CAAO,MAAA;AACtC,IAAA,MAAM,aAAA,GAAgB,MAAM,EAAA,CAAsB,eAAe,EAC9D,MAAA,CAAO;AAAA,MACN,gBAAA,EAAkB,IAAA,CAAK,SAAA,CAAU,eAAe,CAAA;AAAA,MAChD,WAAA,EAAa,UAAA;AAAA,cACbA,QAAA;AAAA,MACA,YAAA,EAAc;AAAA,KACf,CAAA,CACA,KAAA,CAAM,aAAa,EAAE,CAAA,CACrB,SAAS,CAAA,KAAA,KAAS;AACjB,MAAA,IAAI,CAAC,WAAA,EAAa;AAChB,QAAA,OAAO,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA,MACvC;AACA,MAAA,OAAO,MACJ,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,IAC/B,CAAC,CAAA;AACH,IAAA,IAAI,kBAAkB,CAAA,EAAG;AACvB,MAAA,MAAM,IAAIC,oBAAA;AAAA,QACR,CAAA,2CAAA,EAA8C,EAAE,CAAA,oBAAA,EAAuB,WAAW,CAAA,CAAA;AAAA,OACpF;AAAA,IACF;AACA,IAAA,MAAM,eAAA,GAAkBC,gCAAmB,eAAe,CAAA;AAG1D,IAAA,MAAM,IAAA,CAAK,uBAAuB,EAAA,EAAI;AAAA,MACpC,QAAA,EAAU,gBAAA;AAAA,MACV;AAAA,KACD,CAAA;AAID,IAAA,IAAI,oBAAA;AACJ,IAAA,IAAI,aAAa,QAAA,CAAS,SAAS,KAAK,YAAA,CAAa,QAAA,CAAS,OAAO,CAAA,EAAG;AACtE,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,MAAA,CAAO,GAAG,CAAA,CACV,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA;AACtC,MAAA,MAAM,EAAA,CAAmB,WAAW,CAAA,CACjC,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO;AAAA,IACZ,CAAA,MAAO;AACL,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO,CACP,UAAU,GAAG,CAAA;AAAA,IAClB;AAGA,IAAA,MAAM,eAAiC,SAAA,CAAU,GAAA;AAAA,MAC/C,CAAC,EAAE,MAAA,EAAQ,MAAA,EAAQ,MAAK,MAAO;AAAA,QAC7B,qBAAA,EAAuB,EAAA;AAAA,QACvB,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C;AAAA,OACF;AAAA,KACF;AAEA,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,WAAA;AAAA,MACA,IAAA,CAAK,qBAAqB,YAAY,CAAA;AAAA,MACtC;AAAA,KACF;AAGA,IAAA,MAAM,EAAA,CAAqB,cAAc,CAAA,CACtC,KAAA,CAAM,EAAE,SAAA,EAAW,EAAA,EAAI,CAAA,CACvB,MAAA,EAAO;AAGV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,cAAA;AAAA,MACA,WAAA,CAAY,IAAI,CAAA,CAAA,MAAM;AAAA,QACpB,SAAA,EAAW,EAAA;AAAA,QACX,GAAA,EAAKC,sBAAA,CAAkB,CAAA,CAAE,GAAG;AAAA,OAC9B,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAEA,IAAA,OAAO;AAAA,MACL,QAAA,EAAU;AAAA,QACR,SAAA,EAAW;AAAA;AACb,KACF;AAAA,EACF;AAAA,EAEA,MAAM,2BAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,MAAA,EAAQ,UAAA,EAAW,GAAI,OAAA;AAEnC,IAAA,MAAM,EAAA,CAAsB,eAAe,CAAA,CACxC,MAAA,CAAO;AAAA,MACN,MAAA;AAAA,MACA,WAAA,EAAa;AAAA,KACd,CAAA,CACA,KAAA,CAAM,WAAA,EAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,iBAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,KAAA,EAAM,GAAI,OAAA;AAEtB,IAAA,MAAM,GAAsB,eAAe,CAAA,CACxC,MAAA,CAAO,EAAE,OAAO,IAAA,CAAK,SAAA,CAAU,KAAA,IAAS,EAAE,CAAA,EAAG,CAAA,CAC7C,KAAA,CAAM,aAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,sBAAA,CACJ,OAAA,EACA,OAAA,EACuC;AACvC,IAAA,MAAM,IAAA,GAAO,OAAA;AAEb,IAAA,IAAI,UAAA,GAAa,IAAA,CAAwB,eAAe,CAAA,CAAE,MAAA,CAAO;AAAA,MAC/D,WAAA;AAAA,MACA,YAAA;AAAA,MACA,oBAAA;AAAA,MACA,aAAA;AAAA,MACA,OAAA;AAAA,MACA,QAAA;AAAA,MACA,cAAA;AAAA,MACA;AAAA,KACD,CAAA;AAKD,IAAA,IAAI,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,SAAS,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,MAAM,CAAA,EAAG;AACjE,MAAA,UAAA,GAAa,UAAA,CAAW,SAAA,EAAU,CAAE,UAAA,EAAW;AAAA,IACjD;AAEA,IAAA,MAAM,QAAQ,MAAM,UAAA,CACjB,KAAA,CAAM,gBAAA,EAAkB,MAAM,IAAA,CAAK,EAAA,CAAG,GAAA,EAAK,EAC3C,KAAA,CAAM,OAAA,CAAQ,gBAAgB,CAAA,CAC9B,OAAA,CAAQ,kBAAkB,KAAK,CAAA;AAElC,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,OAAA,CAAQ,eAAA,EAAgB;AAE9C,IAAA,MAAM,YAAA,GAAe,CAAC,eAAA,KAA4B;AAChD,MAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AACjD,QAAA,OAAO,KAAK,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,eAAe,UAAU,CAAC,CAAA;AAAA,MACtE,WAAW,IAAA,CAAK,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACtD,QAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,iBAAA,EAAoB,eAAe,CAAA,OAAA,CAAS,CAAA;AAAA,MAC9D;AACA,MAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,kBAAA,EAAqB,eAAe,CAAA,SAAA,CAAW,CAAA;AAAA,IACjE,CAAA;AAEA,IAAA,MAAM,IAAA,CAAwB,eAAe,CAAA,CAC1C,OAAA;AAAA,MACC,YAAA;AAAA,MACA,KAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,MAE5B,MAAA,CAAO;AAAA,MACN,cAAA,EAAgB,aAAa,QAAQ;AAAA,KACtC,CAAA;AAEH,IAAA,OAAO;AAAA,MACL,OAAO,KAAA,CAAM,GAAA;AAAA,QACX,CAAA,CAAA,MACG;AAAA,UACC,IAAI,CAAA,CAAE,SAAA;AAAA,UACN,WAAW,CAAA,CAAE,UAAA;AAAA,UACb,iBAAA,EAAmB,IAAA,CAAK,KAAA,CAAM,CAAA,CAAE,kBAAkB,CAAA;AAAA,UAClD,UAAA,EAAY,EAAE,WAAA,IAAe,EAAA;AAAA,UAC7B,YAAA,EAAcC,8BAAA,CAAoB,CAAA,CAAE,cAAc,CAAA;AAAA,UAClD,OAAO,CAAA,CAAE,KAAA,GAAQ,KAAK,KAAA,CAAM,CAAA,CAAE,KAAK,CAAA,GAAI,MAAA;AAAA,UACvC,QAAQ,CAAA,CAAE,MAAA;AAAA,UACV,aAAa,CAAA,CAAE;AAAA,SACjB;AAAA;AACJ,KACF;AAAA,EACF;AAAA,EAEA,MAAM,WAAA,CACJ,QAAA,EACA,OAAA,EAC4B;AAC5B,IAAA,MAAM,EAAA,GAAK,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,MAEC,OAAA,CAAQ,mBAAA,EAAqB,OAAA,CAAQ,UAAU,EAC/C,MAAA,EAAO;AAEV,IAAA,MAAM,UAAA,GAAa,KAAK,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,iBAAkB,CAAA,CAAE,OAAO,OAAO,CAAA;AAErE,IAAA,OAAO,EAAE,UAAA,EAAW;AAAA,EACtB;AAAA,EAEA,MAAM,YAAe,EAAA,EAAiD;AACpE,IAAA,IAAI;AACF,MAAA,IAAI,MAAA,GAAwB,KAAA,CAAA;AAE5B,MAAA,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,WAAA;AAAA,QAC1B,OAAM,EAAA,KAAM;AAGV,UAAA,MAAA,GAAS,MAAM,GAAG,EAAE,CAAA;AAAA,QACtB,CAAA;AAAA,QACA;AAAA;AAAA,UAEE,qBAAA,EAAuB;AAAA;AACzB,OACF;AAEA,MAAA,OAAO,MAAA;AAAA,IACT,SAAS,CAAA,EAAG;AACV,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,0BAAA,EAA6B,CAAC,CAAA,CAAE,CAAA;AAC1D,MAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA,IACtB;AAAA,EACF;AAAA,EAEQ,qBAAqB,IAAA,EAA0C;AACrE,IAAA,OAAOC,uBAAA,CAAO,MAAA;AAAA,MACZ,IAAA;AAAA,MACA,CAAA,CAAA,KAAK,GAAG,CAAA,CAAE,iBAAiB,IAAI,CAAA,CAAE,iBAAiB,CAAA,CAAA,EAAI,CAAA,CAAE,IAAI,CAAA;AAAA,KAC9D;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,sBAAA,CACZ,QAAA,EACA,OAAA,EAIe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AAGX,IAAA,MAAM,eAAA,GAAkB,IAAI,KAAA,EAAc;AAI1C,IAAA,KAAA,MAAW,EAAE,MAAA,EAAQ,WAAA,EAAY,IAAK,QAAQ,QAAA,EAAU;AACtD,MAAA,MAAM,SAAA,GAAYJ,gCAAmB,MAAM,CAAA;AAC3C,MAAA,MAAM,IAAA,GAAOK,wBAAmB,MAAM,CAAA;AAEtC,MAAA,MAAM,OAAA,GAAU,MAAMC,+CAAA,CAAwB;AAAA,QAC5C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,OAAA,EAAS;AACX,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,QAAA,GAAW,MAAMC,+CAAA,CAAwB;AAAA,QAC7C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA,WAAA;AAAA,QACA,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAKA,MAAA,MAAM,cAAA,GAAiB,MAAMC,iDAAA,CAAyB;AAAA,QACpD,EAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,cAAA,EAAgB;AAClB,QAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,UAClB,CAAA,+BAAA,EAAkC,SAAS,CAAA,uBAAA,EAA0B,cAAc,iBAAiB,WAAW,CAAA;AAAA,SACjH;AACA,QAAA,IAAI,IAAA,CAAK,OAAA,CAAQ,WAAA,IAAe,WAAA,EAAa;AAC3C,UAAA,MAAM,WAAA,GAAwD;AAAA,YAC5D,KAAA,EAAOC,iCAAA;AAAA,YACP,YAAA,EAAc;AAAA,cACZ,iBAAA,EAAmB,MAAA;AAAA,cACnB,SAAA;AAAA,cACA,cAAA,EAAgB,WAAA;AAAA,cAChB,mBAAA,EAAqB,cAAA;AAAA,cACrB,cAAA,EAAgBC,cAAA,CAAS,GAAA,EAAI,CAAE,KAAA;AAAM;AACvC,WACF;AACA,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,WAAA,EAAa,OAAA,CAAQ,WAAW,CAAA;AAAA,QACrD;AAAA,MACF;AAAA,IACF;AAGA,IAAA,MAAM,EAAA,CAAgC,0BAA0B,CAAA,CAE7D,KAAA,CAAM,EAAE,iBAAA,EAAmB,OAAA,CAAQ,eAAA,EAAiB,CAAA,CAEpD,SAAA,CAAU,mBAAA,EAAqB,eAAe,EAC9C,MAAA,EAAO;AACV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,0BAAA;AAAA,MACA,eAAA,CAAgB,IAAI,CAAA,SAAA,MAAc;AAAA,QAChC,mBAAmB,OAAA,CAAQ,eAAA;AAAA,QAC3B,iBAAA,EAAmB;AAAA,OACrB,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAAA,EACF;AACF;;;;"}
+{"version":3,"file":"DefaultProcessingDatabase.cjs.js","sources":["../../src/database/DefaultProcessingDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError } from '@backstage/errors';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { ProcessingIntervalFunction } from '../processing/refresh';\nimport { rethrowError, timestampToDateTime } from './conversion';\nimport { initDatabaseMetrics } from './metrics';\nimport {\n  DbRefreshKeysRow,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n} from './tables';\nimport {\n  GetProcessableEntitiesResult,\n  ListParentsOptions,\n  ListParentsResult,\n  ProcessingDatabase,\n  RefreshStateItem,\n  Transaction,\n  UpdateEntityCacheOptions,\n  UpdateProcessedEntityOptions,\n} from './types';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { generateStableHash, generateTargetKey } from './util';\nimport { EventParams, EventsService } from '@backstage/plugin-events-node';\nimport { DateTime } from 'luxon';\nimport { CATALOG_CONFLICTS_TOPIC } from '../constants';\nimport { CatalogConflictEventPayload } from '../catalog/types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProcessingDatabase implements ProcessingDatabase {\n  constructor(\n    private readonly options: {\n      database: Knex;\n      logger: LoggerService;\n      refreshInterval: ProcessingIntervalFunction;\n      events: EventsService;\n    },\n  ) {\n    initDatabaseMetrics(options.database);\n  }\n\n  async updateProcessedEntity(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<{ previous: { relations: DbRelationsRow[] } }> {\n    const tx = txOpaque as Knex.Transaction;\n    const {\n      id,\n      processedEntity,\n      resultHash,\n      errors,\n      relations,\n      deferredEntities,\n      refreshKeys,\n      locationKey,\n    } = options;\n    const configClient = tx.client.config.client;\n    const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        processed_entity: JSON.stringify(processedEntity),\n        result_hash: resultHash,\n        errors,\n        location_key: locationKey,\n      })\n      .where('entity_id', id)\n      .andWhere(inner => {\n        if (!locationKey) {\n          return inner.whereNull('location_key');\n        }\n        return inner\n          .where('location_key', locationKey)\n          .orWhereNull('location_key');\n      });\n    if (refreshResult === 0) {\n      throw new ConflictError(\n        `Conflicting write of processing result for ${id} with location key '${locationKey}'`,\n      );\n    }\n    const sourceEntityRef = stringifyEntityRef(processedEntity);\n\n    // Schedule all deferred entities for future processing.\n    await this.addUnprocessedEntities(tx, {\n      entities: deferredEntities,\n      sourceEntityRef,\n    });\n\n    // Delete old relations\n    // NOTE(freben): knex implemented support for returning() on update queries for sqlite, but at the current time of writing (Sep 2022) not for delete() queries.\n    let previousRelationRows: DbRelationsRow[];\n    if (configClient.includes('sqlite3') || configClient.includes('mysql')) {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .select('*')\n        .where({ originating_entity_id: id });\n      await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete();\n    } else {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete()\n        .returning('*');\n    }\n\n    // Batch insert new relations\n    const relationRows: DbRelationsRow[] = relations.map(\n      ({ source, target, type }) => ({\n        originating_entity_id: id,\n        source_entity_ref: stringifyEntityRef(source),\n        target_entity_ref: stringifyEntityRef(target),\n        type,\n      }),\n    );\n\n    await tx.batchInsert(\n      'relations',\n      this.deduplicateRelations(relationRows),\n      BATCH_SIZE,\n    );\n\n    // Delete old refresh keys\n    await tx<DbRefreshKeysRow>('refresh_keys')\n      .where({ entity_id: id })\n      .delete();\n\n    // Insert the refresh keys for the processed entity\n    await tx.batchInsert(\n      'refresh_keys',\n      refreshKeys.map(k => ({\n        entity_id: id,\n        key: generateTargetKey(k.key),\n      })),\n      BATCH_SIZE,\n    );\n\n    return {\n      previous: {\n        relations: previousRelationRows,\n      },\n    };\n  }\n\n  async updateProcessedEntityErrors(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, errors, resultHash } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        errors,\n        result_hash: resultHash,\n      })\n      .where('entity_id', id);\n  }\n\n  async updateEntityCache(\n    txOpaque: Transaction,\n    options: UpdateEntityCacheOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, state } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({ cache: JSON.stringify(state ?? {}) })\n      .where('entity_id', id);\n  }\n\n  async getProcessableEntities(\n    maybeTx: Transaction | Knex,\n    request: { processBatchSize: number },\n  ): Promise<GetProcessableEntitiesResult> {\n    const knex = maybeTx as Knex.Transaction | Knex;\n\n    let itemsQuery = knex<DbRefreshStateRow>('refresh_state').select([\n      'entity_id',\n      'entity_ref',\n      'unprocessed_entity',\n      'result_hash',\n      'cache',\n      'errors',\n      'location_key',\n      'next_update_at',\n    ]);\n\n    // This avoids duplication of work because of race conditions and is\n    // also fast because locked rows are ignored rather than blocking.\n    // It's only available in MySQL and PostgreSQL\n    if (['mysql', 'mysql2', 'pg'].includes(knex.client.config.client)) {\n      itemsQuery = itemsQuery.forUpdate().skipLocked();\n    }\n\n    const items = await itemsQuery\n      .where('next_update_at', '<=', knex.fn.now())\n      .limit(request.processBatchSize)\n      .orderBy('next_update_at', 'asc');\n\n    const interval = this.options.refreshInterval();\n\n    const nextUpdateAt = (refreshInterval: number) => {\n      if (knex.client.config.client.includes('sqlite3')) {\n        return knex.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);\n      } else if (knex.client.config.client.includes('mysql')) {\n        return knex.raw(`now() + interval ${refreshInterval} second`);\n      }\n      return knex.raw(`now() + interval '${refreshInterval} seconds'`);\n    };\n\n    await knex<DbRefreshStateRow>('refresh_state')\n      .whereIn(\n        'entity_ref',\n        items.map(i => i.entity_ref),\n      )\n      .update({\n        next_update_at: nextUpdateAt(interval),\n      });\n\n    return {\n      items: items.map(\n        i =>\n          ({\n            id: i.entity_id,\n            entityRef: i.entity_ref,\n            unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity,\n            resultHash: i.result_hash || '',\n            nextUpdateAt: timestampToDateTime(i.next_update_at),\n            state: i.cache ? JSON.parse(i.cache) : undefined,\n            errors: i.errors,\n            locationKey: i.location_key,\n          } satisfies RefreshStateItem),\n      ),\n    };\n  }\n\n  async listParents(\n    txOpaque: Transaction,\n    options: ListParentsOptions,\n  ): Promise<ListParentsResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .whereIn('target_entity_ref', options.entityRefs)\n      .select();\n\n    const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);\n\n    return { entityRefs };\n  }\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] {\n    return lodash.uniqBy(\n      rows,\n      r => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`,\n    );\n  }\n\n  /**\n   * Add a set of deferred entities for processing.\n   * The entities will be added at the front of the processing queue.\n   */\n  private async addUnprocessedEntities(\n    txOpaque: Transaction,\n    options: {\n      sourceEntityRef: string;\n      entities: DeferredEntity[];\n    },\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n\n    // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards\n    const stateReferences = new Array<string>();\n\n    // Upsert all of the unprocessed entities into the refresh_state table, by\n    // their entity ref.\n    for (const { entity, locationKey } of options.entities) {\n      const entityRef = stringifyEntityRef(entity);\n      const hash = generateStableHash(entity);\n\n      const updated = await updateUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n      });\n      if (updated) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      const inserted = await insertUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n        logger: this.options.logger,\n      });\n      if (inserted) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      // If the row can't be inserted, we have a conflict, but it could be either\n      // because of a conflicting locationKey or a race with another instance, so check\n      // whether the conflicting entity has the same entityRef but a different locationKey\n      const conflictingKey = await checkLocationKeyConflict({\n        tx,\n        entityRef,\n        locationKey,\n      });\n      if (conflictingKey) {\n        this.options.logger.warn(\n          `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n        );\n        if (locationKey) {\n          const eventParams: EventParams<CatalogConflictEventPayload> = {\n            topic: CATALOG_CONFLICTS_TOPIC,\n            eventPayload: {\n              unprocessedEntity: entity,\n              entityRef,\n              newLocationKey: locationKey,\n              existingLocationKey: conflictingKey,\n              lastConflictAt: DateTime.now().toISO()!,\n            },\n          };\n          await this.options.events.publish(eventParams);\n        }\n      }\n    }\n\n    // Lastly, replace refresh state references for the originating entity and any successfully added entities\n    await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n      // Remove all existing references from the originating entity\n      .where({ source_entity_ref: options.sourceEntityRef })\n      // And remove any existing references to entities that we're inserting new references for\n      .orWhereIn('target_entity_ref', stateReferences)\n      .delete();\n    await tx.batchInsert(\n      'refresh_state_references',\n      stateReferences.map(entityRef => ({\n        source_entity_ref: options.sourceEntityRef,\n        target_entity_ref: entityRef,\n      })),\n      BATCH_SIZE,\n    );\n  }\n}\n"],"names":["initDatabaseMetrics","errors","ConflictError","stringifyEntityRef","generateTargetKey","timestampToDateTime","rethrowError","lodash","generateStableHash","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","CATALOG_CONFLICTS_TOPIC","DateTime"],"mappings":";;;;;;;;;;;;;;;;;;AAsDA,MAAM,UAAA,GAAa,EAAA;AAEZ,MAAM,yBAAA,CAAwD;AAAA,EACnE,YACmB,OAAA,EAMjB;AANiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAOjB,IAAAA,2BAAA,CAAoB,QAAQ,QAAQ,CAAA;AAAA,EACtC;AAAA,EAEA,MAAM,qBAAA,CACJ,QAAA,EACA,OAAA,EACwD;AACxD,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM;AAAA,MACJ,EAAA;AAAA,MACA,eAAA;AAAA,MACA,UAAA;AAAA,cACAC,QAAA;AAAA,MACA,SAAA;AAAA,MACA,gBAAA;AAAA,MACA,WAAA;AAAA,MACA;AAAA,KACF,GAAI,OAAA;AACJ,IAAA,MAAM,YAAA,GAAe,EAAA,CAAG,MAAA,CAAO,MAAA,CAAO,MAAA;AACtC,IAAA,MAAM,aAAA,GAAgB,MAAM,EAAA,CAAsB,eAAe,EAC9D,MAAA,CAAO;AAAA,MACN,gBAAA,EAAkB,IAAA,CAAK,SAAA,CAAU,eAAe,CAAA;AAAA,MAChD,WAAA,EAAa,UAAA;AAAA,cACbA,QAAA;AAAA,MACA,YAAA,EAAc;AAAA,KACf,CAAA,CACA,KAAA,CAAM,aAAa,EAAE,CAAA,CACrB,SAAS,CAAA,KAAA,KAAS;AACjB,MAAA,IAAI,CAAC,WAAA,EAAa;AAChB,QAAA,OAAO,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA,MACvC;AACA,MAAA,OAAO,MACJ,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,IAC/B,CAAC,CAAA;AACH,IAAA,IAAI,kBAAkB,CAAA,EAAG;AACvB,MAAA,MAAM,IAAIC,oBAAA;AAAA,QACR,CAAA,2CAAA,EAA8C,EAAE,CAAA,oBAAA,EAAuB,WAAW,CAAA,CAAA;AAAA,OACpF;AAAA,IACF;AACA,IAAA,MAAM,eAAA,GAAkBC,gCAAmB,eAAe,CAAA;AAG1D,IAAA,MAAM,IAAA,CAAK,uBAAuB,EAAA,EAAI;AAAA,MACpC,QAAA,EAAU,gBAAA;AAAA,MACV;AAAA,KACD,CAAA;AAID,IAAA,IAAI,oBAAA;AACJ,IAAA,IAAI,aAAa,QAAA,CAAS,SAAS,KAAK,YAAA,CAAa,QAAA,CAAS,OAAO,CAAA,EAAG;AACtE,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,MAAA,CAAO,GAAG,CAAA,CACV,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA;AACtC,MAAA,MAAM,EAAA,CAAmB,WAAW,CAAA,CACjC,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO;AAAA,IACZ,CAAA,MAAO;AACL,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO,CACP,UAAU,GAAG,CAAA;AAAA,IAClB;AAGA,IAAA,MAAM,eAAiC,SAAA,CAAU,GAAA;AAAA,MAC/C,CAAC,EAAE,MAAA,EAAQ,MAAA,EAAQ,MAAK,MAAO;AAAA,QAC7B,qBAAA,EAAuB,EAAA;AAAA,QACvB,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C;AAAA,OACF;AAAA,KACF;AAEA,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,WAAA;AAAA,MACA,IAAA,CAAK,qBAAqB,YAAY,CAAA;AAAA,MACtC;AAAA,KACF;AAGA,IAAA,MAAM,EAAA,CAAqB,cAAc,CAAA,CACtC,KAAA,CAAM,EAAE,SAAA,EAAW,EAAA,EAAI,CAAA,CACvB,MAAA,EAAO;AAGV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,cAAA;AAAA,MACA,WAAA,CAAY,IAAI,CAAA,CAAA,MAAM;AAAA,QACpB,SAAA,EAAW,EAAA;AAAA,QACX,GAAA,EAAKC,sBAAA,CAAkB,CAAA,CAAE,GAAG;AAAA,OAC9B,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAEA,IAAA,OAAO;AAAA,MACL,QAAA,EAAU;AAAA,QACR,SAAA,EAAW;AAAA;AACb,KACF;AAAA,EACF;AAAA,EAEA,MAAM,2BAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,MAAA,EAAQ,UAAA,EAAW,GAAI,OAAA;AAEnC,IAAA,MAAM,EAAA,CAAsB,eAAe,CAAA,CACxC,MAAA,CAAO;AAAA,MACN,MAAA;AAAA,MACA,WAAA,EAAa;AAAA,KACd,CAAA,CACA,KAAA,CAAM,WAAA,EAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,iBAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,KAAA,EAAM,GAAI,OAAA;AAEtB,IAAA,MAAM,GAAsB,eAAe,CAAA,CACxC,MAAA,CAAO,EAAE,OAAO,IAAA,CAAK,SAAA,CAAU,KAAA,IAAS,EAAE,CAAA,EAAG,CAAA,CAC7C,KAAA,CAAM,aAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,sBAAA,CACJ,OAAA,EACA,OAAA,EACuC;AACvC,IAAA,MAAM,IAAA,GAAO,OAAA;AAEb,IAAA,IAAI,UAAA,GAAa,IAAA,CAAwB,eAAe,CAAA,CAAE,MAAA,CAAO;AAAA,MAC/D,WAAA;AAAA,MACA,YAAA;AAAA,MACA,oBAAA;AAAA,MACA,aAAA;AAAA,MACA,OAAA;AAAA,MACA,QAAA;AAAA,MACA,cAAA;AAAA,MACA;AAAA,KACD,CAAA;AAKD,IAAA,IAAI,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,SAAS,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,MAAM,CAAA,EAAG;AACjE,MAAA,UAAA,GAAa,UAAA,CAAW,SAAA,EAAU,CAAE,UAAA,EAAW;AAAA,IACjD;AAEA,IAAA,MAAM,QAAQ,MAAM,UAAA,CACjB,KAAA,CAAM,gBAAA,EAAkB,MAAM,IAAA,CAAK,EAAA,CAAG,GAAA,EAAK,EAC3C,KAAA,CAAM,OAAA,CAAQ,gBAAgB,CAAA,CAC9B,OAAA,CAAQ,kBAAkB,KAAK,CAAA;AAElC,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,OAAA,CAAQ,eAAA,EAAgB;AAE9C,IAAA,MAAM,YAAA,GAAe,CAAC,eAAA,KAA4B;AAChD,MAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AACjD,QAAA,OAAO,KAAK,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,eAAe,UAAU,CAAC,CAAA;AAAA,MACtE,WAAW,IAAA,CAAK,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACtD,QAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,iBAAA,EAAoB,eAAe,CAAA,OAAA,CAAS,CAAA;AAAA,MAC9D;AACA,MAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,kBAAA,EAAqB,eAAe,CAAA,SAAA,CAAW,CAAA;AAAA,IACjE,CAAA;AAEA,IAAA,MAAM,IAAA,CAAwB,eAAe,CAAA,CAC1C,OAAA;AAAA,MACC,YAAA;AAAA,MACA,KAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,MAE5B,MAAA,CAAO;AAAA,MACN,cAAA,EAAgB,aAAa,QAAQ;AAAA,KACtC,CAAA;AAEH,IAAA,OAAO;AAAA,MACL,OAAO,KAAA,CAAM,GAAA;AAAA,QACX,CAAA,CAAA,MACG;AAAA,UACC,IAAI,CAAA,CAAE,SAAA;AAAA,UACN,WAAW,CAAA,CAAE,UAAA;AAAA,UACb,iBAAA,EAAmB,IAAA,CAAK,KAAA,CAAM,CAAA,CAAE,kBAAkB,CAAA;AAAA,UAClD,UAAA,EAAY,EAAE,WAAA,IAAe,EAAA;AAAA,UAC7B,YAAA,EAAcC,8BAAA,CAAoB,CAAA,CAAE,cAAc,CAAA;AAAA,UAClD,OAAO,CAAA,CAAE,KAAA,GAAQ,KAAK,KAAA,CAAM,CAAA,CAAE,KAAK,CAAA,GAAI,MAAA;AAAA,UACvC,QAAQ,CAAA,CAAE,MAAA;AAAA,UACV,aAAa,CAAA,CAAE;AAAA,SACjB;AAAA;AACJ,KACF;AAAA,EACF;AAAA,EAEA,MAAM,WAAA,CACJ,QAAA,EACA,OAAA,EAC4B;AAC5B,IAAA,MAAM,EAAA,GAAK,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,MAEC,OAAA,CAAQ,mBAAA,EAAqB,OAAA,CAAQ,UAAU,EAC/C,MAAA,EAAO;AAEV,IAAA,MAAM,UAAA,GAAa,KAAK,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,iBAAkB,CAAA,CAAE,OAAO,OAAO,CAAA;AAErE,IAAA,OAAO,EAAE,UAAA,EAAW;AAAA,EACtB;AAAA,EAEA,MAAM,YAAe,EAAA,EAAiD;AACpE,IAAA,IAAI;AACF,MAAA,IAAI,MAAA,GAAwB,KAAA,CAAA;AAE5B,MAAA,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,WAAA;AAAA,QAC1B,OAAM,EAAA,KAAM;AAGV,UAAA,MAAA,GAAS,MAAM,GAAG,EAAE,CAAA;AAAA,QACtB,CAAA;AAAA,QACA;AAAA;AAAA,UAEE,qBAAA,EAAuB;AAAA;AACzB,OACF;AAEA,MAAA,OAAO,MAAA;AAAA,IACT,SAAS,CAAA,EAAG;AACV,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,0BAAA,EAA6B,CAAC,CAAA,CAAE,CAAA;AAC1D,MAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA,IACtB;AAAA,EACF;AAAA,EAEQ,qBAAqB,IAAA,EAA0C;AACrE,IAAA,OAAOC,uBAAA,CAAO,MAAA;AAAA,MACZ,IAAA;AAAA,MACA,CAAA,CAAA,KAAK,GAAG,CAAA,CAAE,iBAAiB,IAAI,CAAA,CAAE,iBAAiB,CAAA,CAAA,EAAI,CAAA,CAAE,IAAI,CAAA;AAAA,KAC9D;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,sBAAA,CACZ,QAAA,EACA,OAAA,EAIe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AAGX,IAAA,MAAM,eAAA,GAAkB,IAAI,KAAA,EAAc;AAI1C,IAAA,KAAA,MAAW,EAAE,MAAA,EAAQ,WAAA,EAAY,IAAK,QAAQ,QAAA,EAAU;AACtD,MAAA,MAAM,SAAA,GAAYJ,gCAAmB,MAAM,CAAA;AAC3C,MAAA,MAAM,IAAA,GAAOK,wBAAmB,MAAM,CAAA;AAEtC,MAAA,MAAM,OAAA,GAAU,MAAMC,+CAAA,CAAwB;AAAA,QAC5C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,OAAA,EAAS;AACX,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,QAAA,GAAW,MAAMC,+CAAA,CAAwB;AAAA,QAC7C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA,WAAA;AAAA,QACA,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAKA,MAAA,MAAM,cAAA,GAAiB,MAAMC,iDAAA,CAAyB;AAAA,QACpD,EAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,cAAA,EAAgB;AAClB,QAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,UAClB,CAAA,+BAAA,EAAkC,SAAS,CAAA,uBAAA,EAA0B,cAAc,iBAAiB,WAAW,CAAA;AAAA,SACjH;AACA,QAAA,IAAI,WAAA,EAAa;AACf,UAAA,MAAM,WAAA,GAAwD;AAAA,YAC5D,KAAA,EAAOC,iCAAA;AAAA,YACP,YAAA,EAAc;AAAA,cACZ,iBAAA,EAAmB,MAAA;AAAA,cACnB,SAAA;AAAA,cACA,cAAA,EAAgB,WAAA;AAAA,cAChB,mBAAA,EAAqB,cAAA;AAAA,cACrB,cAAA,EAAgBC,cAAA,CAAS,GAAA,EAAI,CAAE,KAAA;AAAM;AACvC,WACF;AACA,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,OAAA,CAAQ,WAAW,CAAA;AAAA,QAC/C;AAAA,MACF;AAAA,IACF;AAGA,IAAA,MAAM,EAAA,CAAgC,0BAA0B,CAAA,CAE7D,KAAA,CAAM,EAAE,iBAAA,EAAmB,OAAA,CAAQ,eAAA,EAAiB,CAAA,CAEpD,SAAA,CAAU,mBAAA,EAAqB,eAAe,EAC9C,MAAA,EAAO;AACV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,0BAAA;AAAA,MACA,eAAA,CAAgB,IAAI,CAAA,SAAA,MAAc;AAAA,QAChC,mBAAmB,OAAA,CAAQ,eAAA;AAAA,QAC3B,iBAAA,EAAmB;AAAA,OACrB,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAAA,EACF;AACF;;;;"}

package/dist/processing/DefaultCatalogProcessingEngine.cjs.js

@@ -34,7 +34,7 @@ class DefaultCatalogProcessingEngine {
   orphanCleanupIntervalMs;
   onProcessingError;
   tracker;
-  eventBroker;
+  events;
   stopFunc;
   constructor(options) {
     this.config = options.config;
@@ -49,7 +49,7 @@ class DefaultCatalogProcessingEngine {
     this.orphanCleanupIntervalMs = options.orphanCleanupIntervalMs ?? 3e4;
     this.onProcessingError = options.onProcessingError;
     this.tracker = options.tracker ?? progressTracker();
-    this.eventBroker = options.eventBroker;
+    this.events = options.events;
     this.stopFunc = void 0;
   }
   async start() {
@@ -128,7 +128,7 @@ class DefaultCatalogProcessingEngine {
             }
             const location = unprocessedEntity?.metadata?.annotations?.[catalogModel.ANNOTATION_LOCATION];
             if (result.errors.length) {
-              this.eventBroker?.publish({
+              this.events.publish({
                 topic: constants.CATALOG_ERRORS_TOPIC,
                 eventPayload: {
                   entity: entityRef,
@@ -260,22 +260,16 @@ class DefaultCatalogProcessingEngine {
         this.logger.warn(`Failed to delete orphaned entities`, error);
       }
     };
-    if (this.scheduler) {
-      const abortController = new AbortController();
-      this.scheduler.scheduleTask({
-        id: "catalog_orphan_cleanup",
-        frequency: { milliseconds: this.orphanCleanupIntervalMs },
-        timeout: { milliseconds: this.orphanCleanupIntervalMs * 0.8 },
-        fn: runOnce,
-        signal: abortController.signal
-      });
-      return () => {
-        abortController.abort();
-      };
-    }
-    const intervalKey = setInterval(runOnce, this.orphanCleanupIntervalMs);
+    const abortController = new AbortController();
+    this.scheduler.scheduleTask({
+      id: "catalog_orphan_cleanup",
+      frequency: { milliseconds: this.orphanCleanupIntervalMs },
+      timeout: { milliseconds: this.orphanCleanupIntervalMs * 0.8 },
+      fn: runOnce,
+      signal: abortController.signal
+    });
     return () => {
-      clearInterval(intervalKey);
+      abortController.abort();
     };
   }
 }

package/dist/processing/DefaultCatalogProcessingEngine.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultCatalogProcessingEngine.cjs.js","sources":["../../src/processing/DefaultCatalogProcessingEngine.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  ANNOTATION_LOCATION,\n  Entity,\n  stringifyEntityRef,\n} from '@backstage/catalog-model';\nimport { assertError, serializeError, stringifyError } from '@backstage/errors';\nimport { Hash } from 'crypto';\nimport stableStringify from 'fast-json-stable-stringify';\nimport { Knex } from 'knex';\nimport { metrics, trace } from '@opentelemetry/api';\nimport { ProcessingDatabase, RefreshStateItem } from '../database/types';\nimport { createCounterMetric, createSummaryMetric } from '../util/metrics';\nimport { CatalogProcessingOrchestrator, EntityProcessingResult } from './types';\nimport { Stitcher, stitchingStrategyFromConfig } from '../stitching/types';\nimport { startTaskPipeline } from './TaskPipeline';\nimport { Config } from '@backstage/config';\nimport {\n  addEntityAttributes,\n  TRACER_ID,\n  withActiveSpan,\n} from '../util/opentelemetry';\nimport { deleteOrphanedEntities } from '../database/operations/util/deleteOrphanedEntities';\nimport { EventBroker, EventsService } from '@backstage/plugin-events-node';\nimport { CATALOG_ERRORS_TOPIC } from '../constants';\nimport { LoggerService, SchedulerService } from '@backstage/backend-plugin-api';\n\nconst CACHE_TTL = 5;\n\nconst tracer = trace.getTracer(TRACER_ID);\n\nexport type ProgressTracker = ReturnType<typeof progressTracker>;\n\nconst stableStringifyArray = (arr: any[]) => {\n  const sorted = arr.map(stableStringify).sort();\n  return `[${sorted.join(',')}]`;\n};\n\n// NOTE(freben): Perhaps surprisingly, this class does not implement the\n// CatalogProcessingEngine type. That type is externally visible and its name is\n// the way it is for historic reasons. This class has no particular reason to\n// implement that precise interface; nowadays there are several different\n// engines \"hiding\" behind the CatalogProcessingEngine interface, of which this\n// is just one.\nexport class DefaultCatalogProcessingEngine {\n  private readonly config: Config;\n  private readonly scheduler?: SchedulerService;\n  private readonly logger: LoggerService;\n  private readonly knex: Knex;\n  private readonly processingDatabase: ProcessingDatabase;\n  private readonly orchestrator: CatalogProcessingOrchestrator;\n  private readonly stitcher: Stitcher;\n  private readonly createHash: () => Hash;\n  private readonly pollingIntervalMs: number;\n  private readonly orphanCleanupIntervalMs: number;\n  private readonly onProcessingError?: (event: {\n    unprocessedEntity: Entity;\n    errors: Error[];\n  }) => Promise<void> | void;\n  private readonly tracker: ProgressTracker;\n  private readonly eventBroker?: EventBroker | EventsService;\n\n  private stopFunc?: () => void;\n\n  constructor(options: {\n    config: Config;\n    scheduler?: SchedulerService;\n    logger: LoggerService;\n    knex: Knex;\n    processingDatabase: ProcessingDatabase;\n    orchestrator: CatalogProcessingOrchestrator;\n    stitcher: Stitcher;\n    createHash: () => Hash;\n    pollingIntervalMs?: number;\n    orphanCleanupIntervalMs?: number;\n    onProcessingError?: (event: {\n      unprocessedEntity: Entity;\n      errors: Error[];\n    }) => Promise<void> | void;\n    tracker?: ProgressTracker;\n    eventBroker?: EventBroker | EventsService;\n  }) {\n    this.config = options.config;\n    this.scheduler = options.scheduler;\n    this.logger = options.logger;\n    this.knex = options.knex;\n    this.processingDatabase = options.processingDatabase;\n    this.orchestrator = options.orchestrator;\n    this.stitcher = options.stitcher;\n    this.createHash = options.createHash;\n    this.pollingIntervalMs = options.pollingIntervalMs ?? 1_000;\n    this.orphanCleanupIntervalMs = options.orphanCleanupIntervalMs ?? 30_000;\n    this.onProcessingError = options.onProcessingError;\n    this.tracker = options.tracker ?? progressTracker();\n    this.eventBroker = options.eventBroker;\n\n    this.stopFunc = undefined;\n  }\n\n  async start() {\n    if (this.stopFunc) {\n      throw new Error('Processing engine is already started');\n    }\n\n    const stopPipeline = this.startPipeline();\n    const stopCleanup = this.startOrphanCleanup();\n\n    this.stopFunc = () => {\n      stopPipeline();\n      stopCleanup();\n    };\n  }\n\n  async stop() {\n    if (this.stopFunc) {\n      this.stopFunc();\n      this.stopFunc = undefined;\n    }\n  }\n\n  private startPipeline(): () => void {\n    return startTaskPipeline<RefreshStateItem>({\n      lowWatermark: 5,\n      highWatermark: 10,\n      pollingIntervalMs: this.pollingIntervalMs,\n      loadTasks: async count => {\n        try {\n          const { items } =\n            await this.processingDatabase.getProcessableEntities(this.knex, {\n              processBatchSize: count,\n            });\n          return items;\n        } catch (error) {\n          this.logger.warn('Failed to load processing items', error);\n          return [];\n        }\n      },\n      processTask: async item => {\n        await withActiveSpan(tracer, 'ProcessingRun', async span => {\n          const track = this.tracker.processStart(item, this.logger);\n          addEntityAttributes(span, item.unprocessedEntity);\n\n          try {\n            const {\n              id,\n              state,\n              unprocessedEntity,\n              entityRef,\n              locationKey,\n              resultHash: previousResultHash,\n            } = item;\n            const result = await this.orchestrator.process({\n              entity: unprocessedEntity,\n              state,\n            });\n\n            track.markProcessorsCompleted(result);\n\n            if (result.ok) {\n              const { ttl: _, ...stateWithoutTtl } = state ?? {};\n              if (\n                stableStringify(stateWithoutTtl) !==\n                stableStringify(result.state)\n              ) {\n                await this.processingDatabase.transaction(async tx => {\n                  await this.processingDatabase.updateEntityCache(tx, {\n                    id,\n                    state: {\n                      ttl: CACHE_TTL,\n                      ...result.state,\n                    },\n                  });\n                });\n              }\n            } else {\n              const maybeTtl = state?.ttl;\n              const ttl = Number.isInteger(maybeTtl) ? (maybeTtl as number) : 0;\n              await this.processingDatabase.transaction(async tx => {\n                await this.processingDatabase.updateEntityCache(tx, {\n                  id,\n                  state: ttl > 0 ? { ...state, ttl: ttl - 1 } : {},\n                });\n              });\n            }\n\n            const location =\n              unprocessedEntity?.metadata?.annotations?.[ANNOTATION_LOCATION];\n            if (result.errors.length) {\n              this.eventBroker?.publish({\n                topic: CATALOG_ERRORS_TOPIC,\n                eventPayload: {\n                  entity: entityRef,\n                  location,\n                  errors: result.errors,\n                },\n              });\n            }\n            const errorsString = JSON.stringify(\n              result.errors.map(e => serializeError(e)),\n            );\n\n            let hashBuilder = this.createHash().update(errorsString);\n\n            if (result.ok) {\n              const { entityRefs: parents } =\n                await this.processingDatabase.transaction(tx =>\n                  this.processingDatabase.listParents(tx, {\n                    entityRefs: [\n                      entityRef,\n                      ...result.deferredEntities.map(e =>\n                        stringifyEntityRef(e.entity),\n                      ),\n                    ],\n                  }),\n                );\n\n              hashBuilder = hashBuilder\n                .update(stableStringify({ ...result.completedEntity }))\n                .update(stableStringifyArray([...result.deferredEntities]))\n                .update(stableStringifyArray([...result.relations]))\n                .update(stableStringifyArray([...result.refreshKeys]))\n                .update(stableStringifyArray([...parents]));\n            }\n\n            const resultHash = hashBuilder.digest('hex');\n            if (resultHash === previousResultHash) {\n              // If nothing changed in our produced outputs, we cannot have any\n              // significant effect on our surroundings; therefore, we just abort\n              // without any updates / stitching.\n              track.markSuccessfulWithNoChanges();\n              return;\n            }\n\n            // If the result was marked as not OK, it signals that some part of the\n            // processing pipeline threw an exception. This can happen both as part of\n            // non-catastrophic things such as due to validation errors, as well as if\n            // something fatal happens inside the processing for other reasons. In any\n            // case, this means we can't trust that anything in the output is okay. So\n            // just store the errors and trigger a stich so that they become visible to\n            // the outside.\n            if (!result.ok) {\n              // notify the error listener if the entity can not be processed.\n              Promise.resolve(undefined)\n                .then(() =>\n                  this.onProcessingError?.({\n                    unprocessedEntity,\n                    errors: result.errors,\n                  }),\n                )\n                .catch(error => {\n                  this.logger.debug(\n                    `Processing error listener threw an exception, ${stringifyError(\n                      error,\n                    )}`,\n                  );\n                });\n\n              await this.processingDatabase.transaction(async tx => {\n                await this.processingDatabase.updateProcessedEntityErrors(tx, {\n                  id,\n                  errors: errorsString,\n                  resultHash,\n                });\n              });\n\n              await this.stitcher.stitch({\n                entityRefs: [stringifyEntityRef(unprocessedEntity)],\n              });\n\n              track.markSuccessfulWithErrors();\n              return;\n            }\n\n            result.completedEntity.metadata.uid = id;\n            let oldRelationSources: Map<string, string>;\n            await this.processingDatabase.transaction(async tx => {\n              const { previous } =\n                await this.processingDatabase.updateProcessedEntity(tx, {\n                  id,\n                  processedEntity: result.completedEntity,\n                  resultHash,\n                  errors: errorsString,\n                  relations: result.relations,\n                  deferredEntities: result.deferredEntities,\n                  locationKey,\n                  refreshKeys: result.refreshKeys,\n                });\n              oldRelationSources = new Map(\n                previous.relations.map(r => [\n                  `${r.source_entity_ref}:${r.type}->${r.target_entity_ref}`,\n                  r.source_entity_ref,\n                ]),\n              );\n            });\n\n            const newRelationSources = new Map<string, string>(\n              result.relations.map(relation => {\n                const sourceEntityRef = stringifyEntityRef(relation.source);\n                const targetEntityRef = stringifyEntityRef(relation.target);\n                return [\n                  `${sourceEntityRef}:${relation.type}->${targetEntityRef}`,\n                  sourceEntityRef,\n                ];\n              }),\n            );\n\n            const setOfThingsToStitch = new Set<string>([\n              stringifyEntityRef(result.completedEntity),\n            ]);\n            newRelationSources.forEach((sourceEntityRef, uniqueKey) => {\n              if (!oldRelationSources.has(uniqueKey)) {\n                setOfThingsToStitch.add(sourceEntityRef);\n              }\n            });\n            oldRelationSources!.forEach((sourceEntityRef, uniqueKey) => {\n              if (!newRelationSources.has(uniqueKey)) {\n                setOfThingsToStitch.add(sourceEntityRef);\n              }\n            });\n\n            await this.stitcher.stitch({\n              entityRefs: setOfThingsToStitch,\n            });\n\n            track.markSuccessfulWithChanges();\n          } catch (error) {\n            assertError(error);\n            track.markFailed(error);\n          }\n        });\n      },\n    });\n  }\n\n  private startOrphanCleanup(): () => void {\n    const orphanStrategy =\n      this.config.getOptionalString('catalog.orphanStrategy') ?? 'delete';\n    if (orphanStrategy !== 'delete') {\n      return () => {};\n    }\n\n    const stitchingStrategy = stitchingStrategyFromConfig(this.config);\n\n    const runOnce = async () => {\n      try {\n        const n = await deleteOrphanedEntities({\n          knex: this.knex,\n          strategy: stitchingStrategy,\n        });\n        if (n > 0) {\n          this.logger.info(`Deleted ${n} orphaned entities`);\n        }\n      } catch (error) {\n        this.logger.warn(`Failed to delete orphaned entities`, error);\n      }\n    };\n\n    if (this.scheduler) {\n      const abortController = new AbortController();\n\n      this.scheduler.scheduleTask({\n        id: 'catalog_orphan_cleanup',\n        frequency: { milliseconds: this.orphanCleanupIntervalMs },\n        timeout: { milliseconds: this.orphanCleanupIntervalMs * 0.8 },\n        fn: runOnce,\n        signal: abortController.signal,\n      });\n\n      return () => {\n        abortController.abort();\n      };\n    }\n\n    const intervalKey = setInterval(runOnce, this.orphanCleanupIntervalMs);\n    return () => {\n      clearInterval(intervalKey);\n    };\n  }\n}\n\n// Helps wrap the timing and logging behaviors\nfunction progressTracker() {\n  // prom-client metrics are deprecated in favour of OpenTelemetry metrics.\n  const promProcessedEntities = createCounterMetric({\n    name: 'catalog_processed_entities_count',\n    help: 'Amount of entities processed, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingDuration = createSummaryMetric({\n    name: 'catalog_processing_duration_seconds',\n    help: 'Time spent executing the full processing flow, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessorsDuration = createSummaryMetric({\n    name: 'catalog_processors_duration_seconds',\n    help: 'Time spent executing catalog processors, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingQueueDelay = createSummaryMetric({\n    name: 'catalog_processing_queue_delay_seconds',\n    help: 'The amount of delay between being scheduled for processing, and the start of actually being processed, DEPRECATED, use OpenTelemetry metrics instead',\n  });\n\n  const meter = metrics.getMeter('default');\n  const processedEntities = meter.createCounter(\n    'catalog.processed.entities.count',\n    { description: 'Amount of entities processed' },\n  );\n\n  const processingDuration = meter.createHistogram(\n    'catalog.processing.duration',\n    {\n      description: 'Time spent executing the full processing flow',\n      unit: 'seconds',\n    },\n  );\n\n  const processorsDuration = meter.createHistogram(\n    'catalog.processors.duration',\n    {\n      description: 'Time spent executing catalog processors',\n      unit: 'seconds',\n    },\n  );\n\n  const processingQueueDelay = meter.createHistogram(\n    'catalog.processing.queue.delay',\n    {\n      description:\n        'The amount of delay between being scheduled for processing, and the start of actually being processed',\n      unit: 'seconds',\n    },\n  );\n\n  function processStart(item: RefreshStateItem, logger: LoggerService) {\n    const startTime = process.hrtime();\n    const endOverallTimer = promProcessingDuration.startTimer();\n    const endProcessorsTimer = promProcessorsDuration.startTimer();\n\n    logger.debug(`Processing ${item.entityRef}`);\n\n    if (item.nextUpdateAt) {\n      const seconds = -item.nextUpdateAt.diffNow().as('seconds');\n      promProcessingQueueDelay.observe(seconds);\n      processingQueueDelay.record(seconds);\n    }\n\n    function endTime() {\n      const delta = process.hrtime(startTime);\n      return delta[0] + delta[1] / 1e9;\n    }\n\n    function markProcessorsCompleted(result: EntityProcessingResult) {\n      endProcessorsTimer({ result: result.ok ? 'ok' : 'failed' });\n      processorsDuration.record(endTime(), {\n        result: result.ok ? 'ok' : 'failed',\n      });\n    }\n\n    function markSuccessfulWithNoChanges() {\n      endOverallTimer({ result: 'unchanged' });\n      promProcessedEntities.inc({ result: 'unchanged' }, 1);\n\n      processingDuration.record(endTime(), { result: 'unchanged' });\n      processedEntities.add(1, { result: 'unchanged' });\n    }\n\n    function markSuccessfulWithErrors() {\n      endOverallTimer({ result: 'errors' });\n      promProcessedEntities.inc({ result: 'errors' }, 1);\n\n      processingDuration.record(endTime(), { result: 'errors' });\n      processedEntities.add(1, { result: 'errors' });\n    }\n\n    function markSuccessfulWithChanges() {\n      endOverallTimer({ result: 'changed' });\n      promProcessedEntities.inc({ result: 'changed' }, 1);\n\n      processingDuration.record(endTime(), { result: 'changed' });\n      processedEntities.add(1, { result: 'changed' });\n    }\n\n    function markFailed(error: Error) {\n      promProcessedEntities.inc({ result: 'failed' }, 1);\n      processedEntities.add(1, { result: 'failed' });\n      logger.warn(`Processing of ${item.entityRef} failed`, error);\n    }\n\n    return {\n      markProcessorsCompleted,\n      markSuccessfulWithNoChanges,\n      markSuccessfulWithErrors,\n      markSuccessfulWithChanges,\n      markFailed,\n    };\n  }\n\n  return { processStart };\n}\n"],"names":["trace","TRACER_ID","stableStringify","startTaskPipeline","withActiveSpan","addEntityAttributes","ANNOTATION_LOCATION","CATALOG_ERRORS_TOPIC","serializeError","stringifyEntityRef","stringifyError","assertError","stitchingStrategyFromConfig","deleteOrphanedEntities","createCounterMetric","createSummaryMetric","metrics"],"mappings":";;;;;;;;;;;;;;;;;AA0CA,MAAM,SAAA,GAAY,CAAA;AAElB,MAAM,MAAA,GAASA,SAAA,CAAM,SAAA,CAAUC,uBAAS,CAAA;AAIxC,MAAM,oBAAA,GAAuB,CAAC,GAAA,KAAe;AAC3C,EAAA,MAAM,MAAA,GAAS,GAAA,CAAI,GAAA,CAAIC,gCAAe,EAAE,IAAA,EAAK;AAC7C,EAAA,OAAO,CAAA,CAAA,EAAI,MAAA,CAAO,IAAA,CAAK,GAAG,CAAC,CAAA,CAAA,CAAA;AAC7B,CAAA;AAQO,MAAM,8BAAA,CAA+B;AAAA,EACzB,MAAA;AAAA,EACA,SAAA;AAAA,EACA,MAAA;AAAA,EACA,IAAA;AAAA,EACA,kBAAA;AAAA,EACA,YAAA;AAAA,EACA,QAAA;AAAA,EACA,UAAA;AAAA,EACA,iBAAA;AAAA,EACA,uBAAA;AAAA,EACA,iBAAA;AAAA,EAIA,OAAA;AAAA,EACA,WAAA;AAAA,EAET,QAAA;AAAA,EAER,YAAY,OAAA,EAiBT;AACD,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,YAAY,OAAA,CAAQ,SAAA;AACzB,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,OAAO,OAAA,CAAQ,IAAA;AACpB,IAAA,IAAA,CAAK,qBAAqB,OAAA,CAAQ,kBAAA;AAClC,IAAA,IAAA,CAAK,eAAe,OAAA,CAAQ,YAAA;AAC5B,IAAA,IAAA,CAAK,WAAW,OAAA,CAAQ,QAAA;AACxB,IAAA,IAAA,CAAK,aAAa,OAAA,CAAQ,UAAA;AAC1B,IAAA,IAAA,CAAK,iBAAA,GAAoB,QAAQ,iBAAA,IAAqB,GAAA;AACtD,IAAA,IAAA,CAAK,uBAAA,GAA0B,QAAQ,uBAAA,IAA2B,GAAA;AAClE,IAAA,IAAA,CAAK,oBAAoB,OAAA,CAAQ,iBAAA;AACjC,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA,CAAQ,OAAA,IAAW,eAAA,EAAgB;AAClD,IAAA,IAAA,CAAK,cAAc,OAAA,CAAQ,WAAA;AAE3B,IAAA,IAAA,CAAK,QAAA,GAAW,MAAA;AAAA,EAClB;AAAA,EAEA,MAAM,KAAA,GAAQ;AACZ,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,MAAM,IAAI,MAAM,sCAAsC,CAAA;AAAA,IACxD;AAEA,IAAA,MAAM,YAAA,GAAe,KAAK,aAAA,EAAc;AACxC,IAAA,MAAM,WAAA,GAAc,KAAK,kBAAA,EAAmB;AAE5C,IAAA,IAAA,CAAK,WAAW,MAAM;AACpB,MAAA,YAAA,EAAa;AACb,MAAA,WAAA,EAAY;AAAA,IACd,CAAA;AAAA,EACF;AAAA,EAEA,MAAM,IAAA,GAAO;AACX,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,IAAA,CAAK,QAAA,EAAS;AACd,MAAA,IAAA,CAAK,QAAA,GAAW,MAAA;AAAA,IAClB;AAAA,EACF;AAAA,EAEQ,aAAA,GAA4B;AAClC,IAAA,OAAOC,8BAAA,CAAoC;AAAA,MACzC,YAAA,EAAc,CAAA;AAAA,MACd,aAAA,EAAe,EAAA;AAAA,MACf,mBAAmB,IAAA,CAAK,iBAAA;AAAA,MACxB,SAAA,EAAW,OAAM,KAAA,KAAS;AACxB,QAAA,IAAI;AACF,UAAA,MAAM,EAAE,OAAM,GACZ,MAAM,KAAK,kBAAA,CAAmB,sBAAA,CAAuB,KAAK,IAAA,EAAM;AAAA,YAC9D,gBAAA,EAAkB;AAAA,WACnB,CAAA;AACH,UAAA,OAAO,KAAA;AAAA,QACT,SAAS,KAAA,EAAO;AACd,UAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,iCAAA,EAAmC,KAAK,CAAA;AACzD,UAAA,OAAO,EAAC;AAAA,QACV;AAAA,MACF,CAAA;AAAA,MACA,WAAA,EAAa,OAAM,IAAA,KAAQ;AACzB,QAAA,MAAMC,4BAAA,CAAe,MAAA,EAAQ,eAAA,EAAiB,OAAM,IAAA,KAAQ;AAC1D,UAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,YAAA,CAAa,IAAA,EAAM,KAAK,MAAM,CAAA;AACzD,UAAAC,iCAAA,CAAoB,IAAA,EAAM,KAAK,iBAAiB,CAAA;AAEhD,UAAA,IAAI;AACF,YAAA,MAAM;AAAA,cACJ,EAAA;AAAA,cACA,KAAA;AAAA,cACA,iBAAA;AAAA,cACA,SAAA;AAAA,cACA,WAAA;AAAA,cACA,UAAA,EAAY;AAAA,aACd,GAAI,IAAA;AACJ,YAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,YAAA,CAAa,OAAA,CAAQ;AAAA,cAC7C,MAAA,EAAQ,iBAAA;AAAA,cACR;AAAA,aACD,CAAA;AAED,YAAA,KAAA,CAAM,wBAAwB,MAAM,CAAA;AAEpC,YAAA,IAAI,OAAO,EAAA,EAAI;AACb,cAAA,MAAM,EAAE,GAAA,EAAK,CAAA,EAAG,GAAG,eAAA,EAAgB,GAAI,SAAS,EAAC;AACjD,cAAA,IACEH,iCAAgB,eAAe,CAAA,KAC/BA,gCAAA,CAAgB,MAAA,CAAO,KAAK,CAAA,EAC5B;AACA,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,kBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,iBAAA,CAAkB,EAAA,EAAI;AAAA,oBAClD,EAAA;AAAA,oBACA,KAAA,EAAO;AAAA,sBACL,GAAA,EAAK,SAAA;AAAA,sBACL,GAAG,MAAA,CAAO;AAAA;AACZ,mBACD,CAAA;AAAA,gBACH,CAAC,CAAA;AAAA,cACH;AAAA,YACF,CAAA,MAAO;AACL,cAAA,MAAM,WAAW,KAAA,EAAO,GAAA;AACxB,cAAA,MAAM,GAAA,GAAM,MAAA,CAAO,SAAA,CAAU,QAAQ,IAAK,QAAA,GAAsB,CAAA;AAChE,cAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,iBAAA,CAAkB,EAAA,EAAI;AAAA,kBAClD,EAAA;AAAA,kBACA,KAAA,EAAO,GAAA,GAAM,CAAA,GAAI,EAAE,GAAG,OAAO,GAAA,EAAK,GAAA,GAAM,CAAA,EAAE,GAAI;AAAC,iBAChD,CAAA;AAAA,cACH,CAAC,CAAA;AAAA,YACH;AAEA,YAAA,MAAM,QAAA,GACJ,iBAAA,EAAmB,QAAA,EAAU,WAAA,GAAcI,gCAAmB,CAAA;AAChE,YAAA,IAAI,MAAA,CAAO,OAAO,MAAA,EAAQ;AACxB,cAAA,IAAA,CAAK,aAAa,OAAA,CAAQ;AAAA,gBACxB,KAAA,EAAOC,8BAAA;AAAA,gBACP,YAAA,EAAc;AAAA,kBACZ,MAAA,EAAQ,SAAA;AAAA,kBACR,QAAA;AAAA,kBACA,QAAQ,MAAA,CAAO;AAAA;AACjB,eACD,CAAA;AAAA,YACH;AACA,YAAA,MAAM,eAAe,IAAA,CAAK,SAAA;AAAA,cACxB,OAAO,MAAA,CAAO,GAAA,CAAI,CAAA,CAAA,KAAKC,qBAAA,CAAe,CAAC,CAAC;AAAA,aAC1C;AAEA,YAAA,IAAI,WAAA,GAAc,IAAA,CAAK,UAAA,EAAW,CAAE,OAAO,YAAY,CAAA;AAEvD,YAAA,IAAI,OAAO,EAAA,EAAI;AACb,cAAA,MAAM,EAAE,UAAA,EAAY,OAAA,EAAQ,GAC1B,MAAM,KAAK,kBAAA,CAAmB,WAAA;AAAA,gBAAY,CAAA,EAAA,KACxC,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,EAAA,EAAI;AAAA,kBACtC,UAAA,EAAY;AAAA,oBACV,SAAA;AAAA,oBACA,GAAG,OAAO,gBAAA,CAAiB,GAAA;AAAA,sBAAI,CAAA,CAAA,KAC7BC,+BAAA,CAAmB,CAAA,CAAE,MAAM;AAAA;AAC7B;AACF,iBACD;AAAA,eACH;AAEF,cAAA,WAAA,GAAc,YACX,MAAA,CAAOP,gCAAA,CAAgB,EAAE,GAAG,MAAA,CAAO,iBAAiB,CAAC,EACrD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,gBAAgB,CAAC,CAAC,EACzD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,SAAS,CAAC,CAAC,EAClD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,WAAW,CAAC,CAAC,EACpD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,OAAO,CAAC,CAAC,CAAA;AAAA,YAC9C;AAEA,YAAA,MAAM,UAAA,GAAa,WAAA,CAAY,MAAA,CAAO,KAAK,CAAA;AAC3C,YAAA,IAAI,eAAe,kBAAA,EAAoB;AAIrC,cAAA,KAAA,CAAM,2BAAA,EAA4B;AAClC,cAAA;AAAA,YACF;AASA,YAAA,IAAI,CAAC,OAAO,EAAA,EAAI;AAEd,cAAA,OAAA,CAAQ,OAAA,CAAQ,MAAS,CAAA,CACtB,IAAA;AAAA,gBAAK,MACJ,KAAK,iBAAA,GAAoB;AAAA,kBACvB,iBAAA;AAAA,kBACA,QAAQ,MAAA,CAAO;AAAA,iBAChB;AAAA,eACH,CACC,MAAM,CAAA,KAAA,KAAS;AACd,gBAAA,IAAA,CAAK,MAAA,CAAO,KAAA;AAAA,kBACV,CAAA,8CAAA,EAAiDQ,qBAAA;AAAA,oBAC/C;AAAA,mBACD,CAAA;AAAA,iBACH;AAAA,cACF,CAAC,CAAA;AAEH,cAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,2BAAA,CAA4B,EAAA,EAAI;AAAA,kBAC5D,EAAA;AAAA,kBACA,MAAA,EAAQ,YAAA;AAAA,kBACR;AAAA,iBACD,CAAA;AAAA,cACH,CAAC,CAAA;AAED,cAAA,MAAM,IAAA,CAAK,SAAS,MAAA,CAAO;AAAA,gBACzB,UAAA,EAAY,CAACD,+BAAA,CAAmB,iBAAiB,CAAC;AAAA,eACnD,CAAA;AAED,cAAA,KAAA,CAAM,wBAAA,EAAyB;AAC/B,cAAA;AAAA,YACF;AAEA,YAAA,MAAA,CAAO,eAAA,CAAgB,SAAS,GAAA,GAAM,EAAA;AACtC,YAAA,IAAI,kBAAA;AACJ,YAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,cAAA,MAAM,EAAE,QAAA,EAAS,GACf,MAAM,IAAA,CAAK,kBAAA,CAAmB,sBAAsB,EAAA,EAAI;AAAA,gBACtD,EAAA;AAAA,gBACA,iBAAiB,MAAA,CAAO,eAAA;AAAA,gBACxB,UAAA;AAAA,gBACA,MAAA,EAAQ,YAAA;AAAA,gBACR,WAAW,MAAA,CAAO,SAAA;AAAA,gBAClB,kBAAkB,MAAA,CAAO,gBAAA;AAAA,gBACzB,WAAA;AAAA,gBACA,aAAa,MAAA,CAAO;AAAA,eACrB,CAAA;AACH,cAAA,kBAAA,GAAqB,IAAI,GAAA;AAAA,gBACvB,QAAA,CAAS,SAAA,CAAU,GAAA,CAAI,CAAA,CAAA,KAAK;AAAA,kBAC1B,CAAA,EAAG,EAAE,iBAAiB,CAAA,CAAA,EAAI,EAAE,IAAI,CAAA,EAAA,EAAK,EAAE,iBAAiB,CAAA,CAAA;AAAA,kBACxD,CAAA,CAAE;AAAA,iBACH;AAAA,eACH;AAAA,YACF,CAAC,CAAA;AAED,YAAA,MAAM,qBAAqB,IAAI,GAAA;AAAA,cAC7B,MAAA,CAAO,SAAA,CAAU,GAAA,CAAI,CAAA,QAAA,KAAY;AAC/B,gBAAA,MAAM,eAAA,GAAkBA,+BAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAC1D,gBAAA,MAAM,eAAA,GAAkBA,+BAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAC1D,gBAAA,OAAO;AAAA,kBACL,GAAG,eAAe,CAAA,CAAA,EAAI,QAAA,CAAS,IAAI,KAAK,eAAe,CAAA,CAAA;AAAA,kBACvD;AAAA,iBACF;AAAA,cACF,CAAC;AAAA,aACH;AAEA,YAAA,MAAM,mBAAA,uBAA0B,GAAA,CAAY;AAAA,cAC1CA,+BAAA,CAAmB,OAAO,eAAe;AAAA,aAC1C,CAAA;AACD,YAAA,kBAAA,CAAmB,OAAA,CAAQ,CAAC,eAAA,EAAiB,SAAA,KAAc;AACzD,cAAA,IAAI,CAAC,kBAAA,CAAmB,GAAA,CAAI,SAAS,CAAA,EAAG;AACtC,gBAAA,mBAAA,CAAoB,IAAI,eAAe,CAAA;AAAA,cACzC;AAAA,YACF,CAAC,CAAA;AACD,YAAA,kBAAA,CAAoB,OAAA,CAAQ,CAAC,eAAA,EAAiB,SAAA,KAAc;AAC1D,cAAA,IAAI,CAAC,kBAAA,CAAmB,GAAA,CAAI,SAAS,CAAA,EAAG;AACtC,gBAAA,mBAAA,CAAoB,IAAI,eAAe,CAAA;AAAA,cACzC;AAAA,YACF,CAAC,CAAA;AAED,YAAA,MAAM,IAAA,CAAK,SAAS,MAAA,CAAO;AAAA,cACzB,UAAA,EAAY;AAAA,aACb,CAAA;AAED,YAAA,KAAA,CAAM,yBAAA,EAA0B;AAAA,UAClC,SAAS,KAAA,EAAO;AACd,YAAAE,kBAAA,CAAY,KAAK,CAAA;AACjB,YAAA,KAAA,CAAM,WAAW,KAAK,CAAA;AAAA,UACxB;AAAA,QACF,CAAC,CAAA;AAAA,MACH;AAAA,KACD,CAAA;AAAA,EACH;AAAA,EAEQ,kBAAA,GAAiC;AACvC,IAAA,MAAM,cAAA,GACJ,IAAA,CAAK,MAAA,CAAO,iBAAA,CAAkB,wBAAwB,CAAA,IAAK,QAAA;AAC7D,IAAA,IAAI,mBAAmB,QAAA,EAAU;AAC/B,MAAA,OAAO,MAAM;AAAA,MAAC,CAAA;AAAA,IAChB;AAEA,IAAA,MAAM,iBAAA,GAAoBC,iCAAA,CAA4B,IAAA,CAAK,MAAM,CAAA;AAEjE,IAAA,MAAM,UAAU,YAAY;AAC1B,MAAA,IAAI;AACF,QAAA,MAAM,CAAA,GAAI,MAAMC,6CAAA,CAAuB;AAAA,UACrC,MAAM,IAAA,CAAK,IAAA;AAAA,UACX,QAAA,EAAU;AAAA,SACX,CAAA;AACD,QAAA,IAAI,IAAI,CAAA,EAAG;AACT,UAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,CAAA,QAAA,EAAW,CAAC,CAAA,kBAAA,CAAoB,CAAA;AAAA,QACnD;AAAA,MACF,SAAS,KAAA,EAAO;AACd,QAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,CAAA,kCAAA,CAAA,EAAsC,KAAK,CAAA;AAAA,MAC9D;AAAA,IACF,CAAA;AAEA,IAAA,IAAI,KAAK,SAAA,EAAW;AAClB,MAAA,MAAM,eAAA,GAAkB,IAAI,eAAA,EAAgB;AAE5C,MAAA,IAAA,CAAK,UAAU,YAAA,CAAa;AAAA,QAC1B,EAAA,EAAI,wBAAA;AAAA,QACJ,SAAA,EAAW,EAAE,YAAA,EAAc,IAAA,CAAK,uBAAA,EAAwB;AAAA,QACxD,OAAA,EAAS,EAAE,YAAA,EAAc,IAAA,CAAK,0BAA0B,GAAA,EAAI;AAAA,QAC5D,EAAA,EAAI,OAAA;AAAA,QACJ,QAAQ,eAAA,CAAgB;AAAA,OACzB,CAAA;AAED,MAAA,OAAO,MAAM;AACX,QAAA,eAAA,CAAgB,KAAA,EAAM;AAAA,MACxB,CAAA;AAAA,IACF;AAEA,IAAA,MAAM,WAAA,GAAc,WAAA,CAAY,OAAA,EAAS,IAAA,CAAK,uBAAuB,CAAA;AACrE,IAAA,OAAO,MAAM;AACX,MAAA,aAAA,CAAc,WAAW,CAAA;AAAA,IAC3B,CAAA;AAAA,EACF;AACF;AAGA,SAAS,eAAA,GAAkB;AAEzB,EAAA,MAAM,wBAAwBC,2BAAA,CAAoB;AAAA,IAChD,IAAA,EAAM,kCAAA;AAAA,IACN,IAAA,EAAM,6EAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBC,2BAAA,CAAoB;AAAA,IACjD,IAAA,EAAM,qCAAA;AAAA,IACN,IAAA,EAAM,8FAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBA,2BAAA,CAAoB;AAAA,IACjD,IAAA,EAAM,qCAAA;AAAA,IACN,IAAA,EAAM,wFAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,2BAA2BA,2BAAA,CAAoB;AAAA,IACnD,IAAA,EAAM,wCAAA;AAAA,IACN,IAAA,EAAM;AAAA,GACP,CAAA;AAED,EAAA,MAAM,KAAA,GAAQC,WAAA,CAAQ,QAAA,CAAS,SAAS,CAAA;AACxC,EAAA,MAAM,oBAAoB,KAAA,CAAM,aAAA;AAAA,IAC9B,kCAAA;AAAA,IACA,EAAE,aAAa,8BAAA;AAA+B,GAChD;AAEA,EAAA,MAAM,qBAAqB,KAAA,CAAM,eAAA;AAAA,IAC/B,6BAAA;AAAA,IACA;AAAA,MACE,WAAA,EAAa,+CAAA;AAAA,MACb,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,MAAM,qBAAqB,KAAA,CAAM,eAAA;AAAA,IAC/B,6BAAA;AAAA,IACA;AAAA,MACE,WAAA,EAAa,yCAAA;AAAA,MACb,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,MAAM,uBAAuB,KAAA,CAAM,eAAA;AAAA,IACjC,gCAAA;AAAA,IACA;AAAA,MACE,WAAA,EACE,uGAAA;AAAA,MACF,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,SAAS,YAAA,CAAa,MAAwB,MAAA,EAAuB;AACnE,IAAA,MAAM,SAAA,GAAY,QAAQ,MAAA,EAAO;AACjC,IAAA,MAAM,eAAA,GAAkB,uBAAuB,UAAA,EAAW;AAC1D,IAAA,MAAM,kBAAA,GAAqB,uBAAuB,UAAA,EAAW;AAE7D,IAAA,MAAA,CAAO,KAAA,CAAM,CAAA,WAAA,EAAc,IAAA,CAAK,SAAS,CAAA,CAAE,CAAA;AAE3C,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,MAAM,UAAU,CAAC,IAAA,CAAK,aAAa,OAAA,EAAQ,CAAE,GAAG,SAAS,CAAA;AACzD,MAAA,wBAAA,CAAyB,QAAQ,OAAO,CAAA;AACxC,MAAA,oBAAA,CAAqB,OAAO,OAAO,CAAA;AAAA,IACrC;AAEA,IAAA,SAAS,OAAA,GAAU;AACjB,MAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,MAAA,CAAO,SAAS,CAAA;AACtC,MAAA,OAAO,KAAA,CAAM,CAAC,CAAA,GAAI,KAAA,CAAM,CAAC,CAAA,GAAI,GAAA;AAAA,IAC/B;AAEA,IAAA,SAAS,wBAAwB,MAAA,EAAgC;AAC/D,MAAA,kBAAA,CAAmB,EAAE,MAAA,EAAQ,MAAA,CAAO,EAAA,GAAK,IAAA,GAAO,UAAU,CAAA;AAC1D,MAAA,kBAAA,CAAmB,MAAA,CAAO,SAAQ,EAAG;AAAA,QACnC,MAAA,EAAQ,MAAA,CAAO,EAAA,GAAK,IAAA,GAAO;AAAA,OAC5B,CAAA;AAAA,IACH;AAEA,IAAA,SAAS,2BAAA,GAA8B;AACrC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,WAAA,EAAa,CAAA;AACvC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,WAAA,IAAe,CAAC,CAAA;AAEpD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAC5D,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAAA,IAClD;AAEA,IAAA,SAAS,wBAAA,GAA2B;AAClC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,QAAA,EAAU,CAAA;AACpC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,QAAA,IAAY,CAAC,CAAA;AAEjD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AACzD,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAAA,IAC/C;AAEA,IAAA,SAAS,yBAAA,GAA4B;AACnC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,SAAA,EAAW,CAAA;AACrC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,SAAA,IAAa,CAAC,CAAA;AAElD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAC1D,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAAA,IAChD;AAEA,IAAA,SAAS,WAAW,KAAA,EAAc;AAChC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,QAAA,IAAY,CAAC,CAAA;AACjD,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAC7C,MAAA,MAAA,CAAO,IAAA,CAAK,CAAA,cAAA,EAAiB,IAAA,CAAK,SAAS,WAAW,KAAK,CAAA;AAAA,IAC7D;AAEA,IAAA,OAAO;AAAA,MACL,uBAAA;AAAA,MACA,2BAAA;AAAA,MACA,wBAAA;AAAA,MACA,yBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF;AAEA,EAAA,OAAO,EAAE,YAAA,EAAa;AACxB;;;;"}
+{"version":3,"file":"DefaultCatalogProcessingEngine.cjs.js","sources":["../../src/processing/DefaultCatalogProcessingEngine.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  ANNOTATION_LOCATION,\n  Entity,\n  stringifyEntityRef,\n} from '@backstage/catalog-model';\nimport { assertError, serializeError, stringifyError } from '@backstage/errors';\nimport { Hash } from 'crypto';\nimport stableStringify from 'fast-json-stable-stringify';\nimport { Knex } from 'knex';\nimport { metrics, trace } from '@opentelemetry/api';\nimport { ProcessingDatabase, RefreshStateItem } from '../database/types';\nimport { createCounterMetric, createSummaryMetric } from '../util/metrics';\nimport { CatalogProcessingOrchestrator, EntityProcessingResult } from './types';\nimport { Stitcher, stitchingStrategyFromConfig } from '../stitching/types';\nimport { startTaskPipeline } from './TaskPipeline';\nimport { Config } from '@backstage/config';\nimport {\n  addEntityAttributes,\n  TRACER_ID,\n  withActiveSpan,\n} from '../util/opentelemetry';\nimport { deleteOrphanedEntities } from '../database/operations/util/deleteOrphanedEntities';\nimport { EventsService } from '@backstage/plugin-events-node';\nimport { CATALOG_ERRORS_TOPIC } from '../constants';\nimport { LoggerService, SchedulerService } from '@backstage/backend-plugin-api';\n\nconst CACHE_TTL = 5;\n\nconst tracer = trace.getTracer(TRACER_ID);\n\nexport type ProgressTracker = ReturnType<typeof progressTracker>;\n\nconst stableStringifyArray = (arr: any[]) => {\n  const sorted = arr.map(stableStringify).sort();\n  return `[${sorted.join(',')}]`;\n};\n\n// NOTE(freben): Perhaps surprisingly, this class does not implement the\n// CatalogProcessingEngine type. That type is externally visible and its name is\n// the way it is for historic reasons. This class has no particular reason to\n// implement that precise interface; nowadays there are several different\n// engines \"hiding\" behind the CatalogProcessingEngine interface, of which this\n// is just one.\nexport class DefaultCatalogProcessingEngine {\n  private readonly config: Config;\n  private readonly scheduler: SchedulerService;\n  private readonly logger: LoggerService;\n  private readonly knex: Knex;\n  private readonly processingDatabase: ProcessingDatabase;\n  private readonly orchestrator: CatalogProcessingOrchestrator;\n  private readonly stitcher: Stitcher;\n  private readonly createHash: () => Hash;\n  private readonly pollingIntervalMs: number;\n  private readonly orphanCleanupIntervalMs: number;\n  private readonly onProcessingError?: (event: {\n    unprocessedEntity: Entity;\n    errors: Error[];\n  }) => Promise<void> | void;\n  private readonly tracker: ProgressTracker;\n  private readonly events: EventsService;\n\n  private stopFunc?: () => void;\n\n  constructor(options: {\n    config: Config;\n    scheduler: SchedulerService;\n    logger: LoggerService;\n    knex: Knex;\n    processingDatabase: ProcessingDatabase;\n    orchestrator: CatalogProcessingOrchestrator;\n    stitcher: Stitcher;\n    createHash: () => Hash;\n    pollingIntervalMs?: number;\n    orphanCleanupIntervalMs?: number;\n    onProcessingError?: (event: {\n      unprocessedEntity: Entity;\n      errors: Error[];\n    }) => Promise<void> | void;\n    tracker?: ProgressTracker;\n    events: EventsService;\n  }) {\n    this.config = options.config;\n    this.scheduler = options.scheduler;\n    this.logger = options.logger;\n    this.knex = options.knex;\n    this.processingDatabase = options.processingDatabase;\n    this.orchestrator = options.orchestrator;\n    this.stitcher = options.stitcher;\n    this.createHash = options.createHash;\n    this.pollingIntervalMs = options.pollingIntervalMs ?? 1_000;\n    this.orphanCleanupIntervalMs = options.orphanCleanupIntervalMs ?? 30_000;\n    this.onProcessingError = options.onProcessingError;\n    this.tracker = options.tracker ?? progressTracker();\n    this.events = options.events;\n\n    this.stopFunc = undefined;\n  }\n\n  async start() {\n    if (this.stopFunc) {\n      throw new Error('Processing engine is already started');\n    }\n\n    const stopPipeline = this.startPipeline();\n    const stopCleanup = this.startOrphanCleanup();\n\n    this.stopFunc = () => {\n      stopPipeline();\n      stopCleanup();\n    };\n  }\n\n  async stop() {\n    if (this.stopFunc) {\n      this.stopFunc();\n      this.stopFunc = undefined;\n    }\n  }\n\n  private startPipeline(): () => void {\n    return startTaskPipeline<RefreshStateItem>({\n      lowWatermark: 5,\n      highWatermark: 10,\n      pollingIntervalMs: this.pollingIntervalMs,\n      loadTasks: async count => {\n        try {\n          const { items } =\n            await this.processingDatabase.getProcessableEntities(this.knex, {\n              processBatchSize: count,\n            });\n          return items;\n        } catch (error) {\n          this.logger.warn('Failed to load processing items', error);\n          return [];\n        }\n      },\n      processTask: async item => {\n        await withActiveSpan(tracer, 'ProcessingRun', async span => {\n          const track = this.tracker.processStart(item, this.logger);\n          addEntityAttributes(span, item.unprocessedEntity);\n\n          try {\n            const {\n              id,\n              state,\n              unprocessedEntity,\n              entityRef,\n              locationKey,\n              resultHash: previousResultHash,\n            } = item;\n            const result = await this.orchestrator.process({\n              entity: unprocessedEntity,\n              state,\n            });\n\n            track.markProcessorsCompleted(result);\n\n            if (result.ok) {\n              const { ttl: _, ...stateWithoutTtl } = state ?? {};\n              if (\n                stableStringify(stateWithoutTtl) !==\n                stableStringify(result.state)\n              ) {\n                await this.processingDatabase.transaction(async tx => {\n                  await this.processingDatabase.updateEntityCache(tx, {\n                    id,\n                    state: {\n                      ttl: CACHE_TTL,\n                      ...result.state,\n                    },\n                  });\n                });\n              }\n            } else {\n              const maybeTtl = state?.ttl;\n              const ttl = Number.isInteger(maybeTtl) ? (maybeTtl as number) : 0;\n              await this.processingDatabase.transaction(async tx => {\n                await this.processingDatabase.updateEntityCache(tx, {\n                  id,\n                  state: ttl > 0 ? { ...state, ttl: ttl - 1 } : {},\n                });\n              });\n            }\n\n            const location =\n              unprocessedEntity?.metadata?.annotations?.[ANNOTATION_LOCATION];\n            if (result.errors.length) {\n              this.events.publish({\n                topic: CATALOG_ERRORS_TOPIC,\n                eventPayload: {\n                  entity: entityRef,\n                  location,\n                  errors: result.errors,\n                },\n              });\n            }\n            const errorsString = JSON.stringify(\n              result.errors.map(e => serializeError(e)),\n            );\n\n            let hashBuilder = this.createHash().update(errorsString);\n\n            if (result.ok) {\n              const { entityRefs: parents } =\n                await this.processingDatabase.transaction(tx =>\n                  this.processingDatabase.listParents(tx, {\n                    entityRefs: [\n                      entityRef,\n                      ...result.deferredEntities.map(e =>\n                        stringifyEntityRef(e.entity),\n                      ),\n                    ],\n                  }),\n                );\n\n              hashBuilder = hashBuilder\n                .update(stableStringify({ ...result.completedEntity }))\n                .update(stableStringifyArray([...result.deferredEntities]))\n                .update(stableStringifyArray([...result.relations]))\n                .update(stableStringifyArray([...result.refreshKeys]))\n                .update(stableStringifyArray([...parents]));\n            }\n\n            const resultHash = hashBuilder.digest('hex');\n            if (resultHash === previousResultHash) {\n              // If nothing changed in our produced outputs, we cannot have any\n              // significant effect on our surroundings; therefore, we just abort\n              // without any updates / stitching.\n              track.markSuccessfulWithNoChanges();\n              return;\n            }\n\n            // If the result was marked as not OK, it signals that some part of the\n            // processing pipeline threw an exception. This can happen both as part of\n            // non-catastrophic things such as due to validation errors, as well as if\n            // something fatal happens inside the processing for other reasons. In any\n            // case, this means we can't trust that anything in the output is okay. So\n            // just store the errors and trigger a stich so that they become visible to\n            // the outside.\n            if (!result.ok) {\n              // notify the error listener if the entity can not be processed.\n              Promise.resolve(undefined)\n                .then(() =>\n                  this.onProcessingError?.({\n                    unprocessedEntity,\n                    errors: result.errors,\n                  }),\n                )\n                .catch(error => {\n                  this.logger.debug(\n                    `Processing error listener threw an exception, ${stringifyError(\n                      error,\n                    )}`,\n                  );\n                });\n\n              await this.processingDatabase.transaction(async tx => {\n                await this.processingDatabase.updateProcessedEntityErrors(tx, {\n                  id,\n                  errors: errorsString,\n                  resultHash,\n                });\n              });\n\n              await this.stitcher.stitch({\n                entityRefs: [stringifyEntityRef(unprocessedEntity)],\n              });\n\n              track.markSuccessfulWithErrors();\n              return;\n            }\n\n            result.completedEntity.metadata.uid = id;\n            let oldRelationSources: Map<string, string>;\n            await this.processingDatabase.transaction(async tx => {\n              const { previous } =\n                await this.processingDatabase.updateProcessedEntity(tx, {\n                  id,\n                  processedEntity: result.completedEntity,\n                  resultHash,\n                  errors: errorsString,\n                  relations: result.relations,\n                  deferredEntities: result.deferredEntities,\n                  locationKey,\n                  refreshKeys: result.refreshKeys,\n                });\n              oldRelationSources = new Map(\n                previous.relations.map(r => [\n                  `${r.source_entity_ref}:${r.type}->${r.target_entity_ref}`,\n                  r.source_entity_ref,\n                ]),\n              );\n            });\n\n            const newRelationSources = new Map<string, string>(\n              result.relations.map(relation => {\n                const sourceEntityRef = stringifyEntityRef(relation.source);\n                const targetEntityRef = stringifyEntityRef(relation.target);\n                return [\n                  `${sourceEntityRef}:${relation.type}->${targetEntityRef}`,\n                  sourceEntityRef,\n                ];\n              }),\n            );\n\n            const setOfThingsToStitch = new Set<string>([\n              stringifyEntityRef(result.completedEntity),\n            ]);\n            newRelationSources.forEach((sourceEntityRef, uniqueKey) => {\n              if (!oldRelationSources.has(uniqueKey)) {\n                setOfThingsToStitch.add(sourceEntityRef);\n              }\n            });\n            oldRelationSources!.forEach((sourceEntityRef, uniqueKey) => {\n              if (!newRelationSources.has(uniqueKey)) {\n                setOfThingsToStitch.add(sourceEntityRef);\n              }\n            });\n\n            await this.stitcher.stitch({\n              entityRefs: setOfThingsToStitch,\n            });\n\n            track.markSuccessfulWithChanges();\n          } catch (error) {\n            assertError(error);\n            track.markFailed(error);\n          }\n        });\n      },\n    });\n  }\n\n  private startOrphanCleanup(): () => void {\n    const orphanStrategy =\n      this.config.getOptionalString('catalog.orphanStrategy') ?? 'delete';\n    if (orphanStrategy !== 'delete') {\n      return () => {};\n    }\n\n    const stitchingStrategy = stitchingStrategyFromConfig(this.config);\n\n    const runOnce = async () => {\n      try {\n        const n = await deleteOrphanedEntities({\n          knex: this.knex,\n          strategy: stitchingStrategy,\n        });\n        if (n > 0) {\n          this.logger.info(`Deleted ${n} orphaned entities`);\n        }\n      } catch (error) {\n        this.logger.warn(`Failed to delete orphaned entities`, error);\n      }\n    };\n\n    const abortController = new AbortController();\n    this.scheduler.scheduleTask({\n      id: 'catalog_orphan_cleanup',\n      frequency: { milliseconds: this.orphanCleanupIntervalMs },\n      timeout: { milliseconds: this.orphanCleanupIntervalMs * 0.8 },\n      fn: runOnce,\n      signal: abortController.signal,\n    });\n\n    return () => {\n      abortController.abort();\n    };\n  }\n}\n\n// Helps wrap the timing and logging behaviors\nfunction progressTracker() {\n  // prom-client metrics are deprecated in favour of OpenTelemetry metrics.\n  const promProcessedEntities = createCounterMetric({\n    name: 'catalog_processed_entities_count',\n    help: 'Amount of entities processed, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingDuration = createSummaryMetric({\n    name: 'catalog_processing_duration_seconds',\n    help: 'Time spent executing the full processing flow, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessorsDuration = createSummaryMetric({\n    name: 'catalog_processors_duration_seconds',\n    help: 'Time spent executing catalog processors, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingQueueDelay = createSummaryMetric({\n    name: 'catalog_processing_queue_delay_seconds',\n    help: 'The amount of delay between being scheduled for processing, and the start of actually being processed, DEPRECATED, use OpenTelemetry metrics instead',\n  });\n\n  const meter = metrics.getMeter('default');\n  const processedEntities = meter.createCounter(\n    'catalog.processed.entities.count',\n    { description: 'Amount of entities processed' },\n  );\n\n  const processingDuration = meter.createHistogram(\n    'catalog.processing.duration',\n    {\n      description: 'Time spent executing the full processing flow',\n      unit: 'seconds',\n    },\n  );\n\n  const processorsDuration = meter.createHistogram(\n    'catalog.processors.duration',\n    {\n      description: 'Time spent executing catalog processors',\n      unit: 'seconds',\n    },\n  );\n\n  const processingQueueDelay = meter.createHistogram(\n    'catalog.processing.queue.delay',\n    {\n      description:\n        'The amount of delay between being scheduled for processing, and the start of actually being processed',\n      unit: 'seconds',\n    },\n  );\n\n  function processStart(item: RefreshStateItem, logger: LoggerService) {\n    const startTime = process.hrtime();\n    const endOverallTimer = promProcessingDuration.startTimer();\n    const endProcessorsTimer = promProcessorsDuration.startTimer();\n\n    logger.debug(`Processing ${item.entityRef}`);\n\n    if (item.nextUpdateAt) {\n      const seconds = -item.nextUpdateAt.diffNow().as('seconds');\n      promProcessingQueueDelay.observe(seconds);\n      processingQueueDelay.record(seconds);\n    }\n\n    function endTime() {\n      const delta = process.hrtime(startTime);\n      return delta[0] + delta[1] / 1e9;\n    }\n\n    function markProcessorsCompleted(result: EntityProcessingResult) {\n      endProcessorsTimer({ result: result.ok ? 'ok' : 'failed' });\n      processorsDuration.record(endTime(), {\n        result: result.ok ? 'ok' : 'failed',\n      });\n    }\n\n    function markSuccessfulWithNoChanges() {\n      endOverallTimer({ result: 'unchanged' });\n      promProcessedEntities.inc({ result: 'unchanged' }, 1);\n\n      processingDuration.record(endTime(), { result: 'unchanged' });\n      processedEntities.add(1, { result: 'unchanged' });\n    }\n\n    function markSuccessfulWithErrors() {\n      endOverallTimer({ result: 'errors' });\n      promProcessedEntities.inc({ result: 'errors' }, 1);\n\n      processingDuration.record(endTime(), { result: 'errors' });\n      processedEntities.add(1, { result: 'errors' });\n    }\n\n    function markSuccessfulWithChanges() {\n      endOverallTimer({ result: 'changed' });\n      promProcessedEntities.inc({ result: 'changed' }, 1);\n\n      processingDuration.record(endTime(), { result: 'changed' });\n      processedEntities.add(1, { result: 'changed' });\n    }\n\n    function markFailed(error: Error) {\n      promProcessedEntities.inc({ result: 'failed' }, 1);\n      processedEntities.add(1, { result: 'failed' });\n      logger.warn(`Processing of ${item.entityRef} failed`, error);\n    }\n\n    return {\n      markProcessorsCompleted,\n      markSuccessfulWithNoChanges,\n      markSuccessfulWithErrors,\n      markSuccessfulWithChanges,\n      markFailed,\n    };\n  }\n\n  return { processStart };\n}\n"],"names":["trace","TRACER_ID","stableStringify","startTaskPipeline","withActiveSpan","addEntityAttributes","ANNOTATION_LOCATION","CATALOG_ERRORS_TOPIC","serializeError","stringifyEntityRef","stringifyError","assertError","stitchingStrategyFromConfig","deleteOrphanedEntities","createCounterMetric","createSummaryMetric","metrics"],"mappings":";;;;;;;;;;;;;;;;;AA0CA,MAAM,SAAA,GAAY,CAAA;AAElB,MAAM,MAAA,GAASA,SAAA,CAAM,SAAA,CAAUC,uBAAS,CAAA;AAIxC,MAAM,oBAAA,GAAuB,CAAC,GAAA,KAAe;AAC3C,EAAA,MAAM,MAAA,GAAS,GAAA,CAAI,GAAA,CAAIC,gCAAe,EAAE,IAAA,EAAK;AAC7C,EAAA,OAAO,CAAA,CAAA,EAAI,MAAA,CAAO,IAAA,CAAK,GAAG,CAAC,CAAA,CAAA,CAAA;AAC7B,CAAA;AAQO,MAAM,8BAAA,CAA+B;AAAA,EACzB,MAAA;AAAA,EACA,SAAA;AAAA,EACA,MAAA;AAAA,EACA,IAAA;AAAA,EACA,kBAAA;AAAA,EACA,YAAA;AAAA,EACA,QAAA;AAAA,EACA,UAAA;AAAA,EACA,iBAAA;AAAA,EACA,uBAAA;AAAA,EACA,iBAAA;AAAA,EAIA,OAAA;AAAA,EACA,MAAA;AAAA,EAET,QAAA;AAAA,EAER,YAAY,OAAA,EAiBT;AACD,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,YAAY,OAAA,CAAQ,SAAA;AACzB,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,OAAO,OAAA,CAAQ,IAAA;AACpB,IAAA,IAAA,CAAK,qBAAqB,OAAA,CAAQ,kBAAA;AAClC,IAAA,IAAA,CAAK,eAAe,OAAA,CAAQ,YAAA;AAC5B,IAAA,IAAA,CAAK,WAAW,OAAA,CAAQ,QAAA;AACxB,IAAA,IAAA,CAAK,aAAa,OAAA,CAAQ,UAAA;AAC1B,IAAA,IAAA,CAAK,iBAAA,GAAoB,QAAQ,iBAAA,IAAqB,GAAA;AACtD,IAAA,IAAA,CAAK,uBAAA,GAA0B,QAAQ,uBAAA,IAA2B,GAAA;AAClE,IAAA,IAAA,CAAK,oBAAoB,OAAA,CAAQ,iBAAA;AACjC,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA,CAAQ,OAAA,IAAW,eAAA,EAAgB;AAClD,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AAEtB,IAAA,IAAA,CAAK,QAAA,GAAW,MAAA;AAAA,EAClB;AAAA,EAEA,MAAM,KAAA,GAAQ;AACZ,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,MAAM,IAAI,MAAM,sCAAsC,CAAA;AAAA,IACxD;AAEA,IAAA,MAAM,YAAA,GAAe,KAAK,aAAA,EAAc;AACxC,IAAA,MAAM,WAAA,GAAc,KAAK,kBAAA,EAAmB;AAE5C,IAAA,IAAA,CAAK,WAAW,MAAM;AACpB,MAAA,YAAA,EAAa;AACb,MAAA,WAAA,EAAY;AAAA,IACd,CAAA;AAAA,EACF;AAAA,EAEA,MAAM,IAAA,GAAO;AACX,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,IAAA,CAAK,QAAA,EAAS;AACd,MAAA,IAAA,CAAK,QAAA,GAAW,MAAA;AAAA,IAClB;AAAA,EACF;AAAA,EAEQ,aAAA,GAA4B;AAClC,IAAA,OAAOC,8BAAA,CAAoC;AAAA,MACzC,YAAA,EAAc,CAAA;AAAA,MACd,aAAA,EAAe,EAAA;AAAA,MACf,mBAAmB,IAAA,CAAK,iBAAA;AAAA,MACxB,SAAA,EAAW,OAAM,KAAA,KAAS;AACxB,QAAA,IAAI;AACF,UAAA,MAAM,EAAE,OAAM,GACZ,MAAM,KAAK,kBAAA,CAAmB,sBAAA,CAAuB,KAAK,IAAA,EAAM;AAAA,YAC9D,gBAAA,EAAkB;AAAA,WACnB,CAAA;AACH,UAAA,OAAO,KAAA;AAAA,QACT,SAAS,KAAA,EAAO;AACd,UAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,iCAAA,EAAmC,KAAK,CAAA;AACzD,UAAA,OAAO,EAAC;AAAA,QACV;AAAA,MACF,CAAA;AAAA,MACA,WAAA,EAAa,OAAM,IAAA,KAAQ;AACzB,QAAA,MAAMC,4BAAA,CAAe,MAAA,EAAQ,eAAA,EAAiB,OAAM,IAAA,KAAQ;AAC1D,UAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,YAAA,CAAa,IAAA,EAAM,KAAK,MAAM,CAAA;AACzD,UAAAC,iCAAA,CAAoB,IAAA,EAAM,KAAK,iBAAiB,CAAA;AAEhD,UAAA,IAAI;AACF,YAAA,MAAM;AAAA,cACJ,EAAA;AAAA,cACA,KAAA;AAAA,cACA,iBAAA;AAAA,cACA,SAAA;AAAA,cACA,WAAA;AAAA,cACA,UAAA,EAAY;AAAA,aACd,GAAI,IAAA;AACJ,YAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,YAAA,CAAa,OAAA,CAAQ;AAAA,cAC7C,MAAA,EAAQ,iBAAA;AAAA,cACR;AAAA,aACD,CAAA;AAED,YAAA,KAAA,CAAM,wBAAwB,MAAM,CAAA;AAEpC,YAAA,IAAI,OAAO,EAAA,EAAI;AACb,cAAA,MAAM,EAAE,GAAA,EAAK,CAAA,EAAG,GAAG,eAAA,EAAgB,GAAI,SAAS,EAAC;AACjD,cAAA,IACEH,iCAAgB,eAAe,CAAA,KAC/BA,gCAAA,CAAgB,MAAA,CAAO,KAAK,CAAA,EAC5B;AACA,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,kBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,iBAAA,CAAkB,EAAA,EAAI;AAAA,oBAClD,EAAA;AAAA,oBACA,KAAA,EAAO;AAAA,sBACL,GAAA,EAAK,SAAA;AAAA,sBACL,GAAG,MAAA,CAAO;AAAA;AACZ,mBACD,CAAA;AAAA,gBACH,CAAC,CAAA;AAAA,cACH;AAAA,YACF,CAAA,MAAO;AACL,cAAA,MAAM,WAAW,KAAA,EAAO,GAAA;AACxB,cAAA,MAAM,GAAA,GAAM,MAAA,CAAO,SAAA,CAAU,QAAQ,IAAK,QAAA,GAAsB,CAAA;AAChE,cAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,iBAAA,CAAkB,EAAA,EAAI;AAAA,kBAClD,EAAA;AAAA,kBACA,KAAA,EAAO,GAAA,GAAM,CAAA,GAAI,EAAE,GAAG,OAAO,GAAA,EAAK,GAAA,GAAM,CAAA,EAAE,GAAI;AAAC,iBAChD,CAAA;AAAA,cACH,CAAC,CAAA;AAAA,YACH;AAEA,YAAA,MAAM,QAAA,GACJ,iBAAA,EAAmB,QAAA,EAAU,WAAA,GAAcI,gCAAmB,CAAA;AAChE,YAAA,IAAI,MAAA,CAAO,OAAO,MAAA,EAAQ;AACxB,cAAA,IAAA,CAAK,OAAO,OAAA,CAAQ;AAAA,gBAClB,KAAA,EAAOC,8BAAA;AAAA,gBACP,YAAA,EAAc;AAAA,kBACZ,MAAA,EAAQ,SAAA;AAAA,kBACR,QAAA;AAAA,kBACA,QAAQ,MAAA,CAAO;AAAA;AACjB,eACD,CAAA;AAAA,YACH;AACA,YAAA,MAAM,eAAe,IAAA,CAAK,SAAA;AAAA,cACxB,OAAO,MAAA,CAAO,GAAA,CAAI,CAAA,CAAA,KAAKC,qBAAA,CAAe,CAAC,CAAC;AAAA,aAC1C;AAEA,YAAA,IAAI,WAAA,GAAc,IAAA,CAAK,UAAA,EAAW,CAAE,OAAO,YAAY,CAAA;AAEvD,YAAA,IAAI,OAAO,EAAA,EAAI;AACb,cAAA,MAAM,EAAE,UAAA,EAAY,OAAA,EAAQ,GAC1B,MAAM,KAAK,kBAAA,CAAmB,WAAA;AAAA,gBAAY,CAAA,EAAA,KACxC,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,EAAA,EAAI;AAAA,kBACtC,UAAA,EAAY;AAAA,oBACV,SAAA;AAAA,oBACA,GAAG,OAAO,gBAAA,CAAiB,GAAA;AAAA,sBAAI,CAAA,CAAA,KAC7BC,+BAAA,CAAmB,CAAA,CAAE,MAAM;AAAA;AAC7B;AACF,iBACD;AAAA,eACH;AAEF,cAAA,WAAA,GAAc,YACX,MAAA,CAAOP,gCAAA,CAAgB,EAAE,GAAG,MAAA,CAAO,iBAAiB,CAAC,EACrD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,gBAAgB,CAAC,CAAC,EACzD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,SAAS,CAAC,CAAC,EAClD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,WAAW,CAAC,CAAC,EACpD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,OAAO,CAAC,CAAC,CAAA;AAAA,YAC9C;AAEA,YAAA,MAAM,UAAA,GAAa,WAAA,CAAY,MAAA,CAAO,KAAK,CAAA;AAC3C,YAAA,IAAI,eAAe,kBAAA,EAAoB;AAIrC,cAAA,KAAA,CAAM,2BAAA,EAA4B;AAClC,cAAA;AAAA,YACF;AASA,YAAA,IAAI,CAAC,OAAO,EAAA,EAAI;AAEd,cAAA,OAAA,CAAQ,OAAA,CAAQ,MAAS,CAAA,CACtB,IAAA;AAAA,gBAAK,MACJ,KAAK,iBAAA,GAAoB;AAAA,kBACvB,iBAAA;AAAA,kBACA,QAAQ,MAAA,CAAO;AAAA,iBAChB;AAAA,eACH,CACC,MAAM,CAAA,KAAA,KAAS;AACd,gBAAA,IAAA,CAAK,MAAA,CAAO,KAAA;AAAA,kBACV,CAAA,8CAAA,EAAiDQ,qBAAA;AAAA,oBAC/C;AAAA,mBACD,CAAA;AAAA,iBACH;AAAA,cACF,CAAC,CAAA;AAEH,cAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,2BAAA,CAA4B,EAAA,EAAI;AAAA,kBAC5D,EAAA;AAAA,kBACA,MAAA,EAAQ,YAAA;AAAA,kBACR;AAAA,iBACD,CAAA;AAAA,cACH,CAAC,CAAA;AAED,cAAA,MAAM,IAAA,CAAK,SAAS,MAAA,CAAO;AAAA,gBACzB,UAAA,EAAY,CAACD,+BAAA,CAAmB,iBAAiB,CAAC;AAAA,eACnD,CAAA;AAED,cAAA,KAAA,CAAM,wBAAA,EAAyB;AAC/B,cAAA;AAAA,YACF;AAEA,YAAA,MAAA,CAAO,eAAA,CAAgB,SAAS,GAAA,GAAM,EAAA;AACtC,YAAA,IAAI,kBAAA;AACJ,YAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,cAAA,MAAM,EAAE,QAAA,EAAS,GACf,MAAM,IAAA,CAAK,kBAAA,CAAmB,sBAAsB,EAAA,EAAI;AAAA,gBACtD,EAAA;AAAA,gBACA,iBAAiB,MAAA,CAAO,eAAA;AAAA,gBACxB,UAAA;AAAA,gBACA,MAAA,EAAQ,YAAA;AAAA,gBACR,WAAW,MAAA,CAAO,SAAA;AAAA,gBAClB,kBAAkB,MAAA,CAAO,gBAAA;AAAA,gBACzB,WAAA;AAAA,gBACA,aAAa,MAAA,CAAO;AAAA,eACrB,CAAA;AACH,cAAA,kBAAA,GAAqB,IAAI,GAAA;AAAA,gBACvB,QAAA,CAAS,SAAA,CAAU,GAAA,CAAI,CAAA,CAAA,KAAK;AAAA,kBAC1B,CAAA,EAAG,EAAE,iBAAiB,CAAA,CAAA,EAAI,EAAE,IAAI,CAAA,EAAA,EAAK,EAAE,iBAAiB,CAAA,CAAA;AAAA,kBACxD,CAAA,CAAE;AAAA,iBACH;AAAA,eACH;AAAA,YACF,CAAC,CAAA;AAED,YAAA,MAAM,qBAAqB,IAAI,GAAA;AAAA,cAC7B,MAAA,CAAO,SAAA,CAAU,GAAA,CAAI,CAAA,QAAA,KAAY;AAC/B,gBAAA,MAAM,eAAA,GAAkBA,+BAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAC1D,gBAAA,MAAM,eAAA,GAAkBA,+BAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAC1D,gBAAA,OAAO;AAAA,kBACL,GAAG,eAAe,CAAA,CAAA,EAAI,QAAA,CAAS,IAAI,KAAK,eAAe,CAAA,CAAA;AAAA,kBACvD;AAAA,iBACF;AAAA,cACF,CAAC;AAAA,aACH;AAEA,YAAA,MAAM,mBAAA,uBAA0B,GAAA,CAAY;AAAA,cAC1CA,+BAAA,CAAmB,OAAO,eAAe;AAAA,aAC1C,CAAA;AACD,YAAA,kBAAA,CAAmB,OAAA,CAAQ,CAAC,eAAA,EAAiB,SAAA,KAAc;AACzD,cAAA,IAAI,CAAC,kBAAA,CAAmB,GAAA,CAAI,SAAS,CAAA,EAAG;AACtC,gBAAA,mBAAA,CAAoB,IAAI,eAAe,CAAA;AAAA,cACzC;AAAA,YACF,CAAC,CAAA;AACD,YAAA,kBAAA,CAAoB,OAAA,CAAQ,CAAC,eAAA,EAAiB,SAAA,KAAc;AAC1D,cAAA,IAAI,CAAC,kBAAA,CAAmB,GAAA,CAAI,SAAS,CAAA,EAAG;AACtC,gBAAA,mBAAA,CAAoB,IAAI,eAAe,CAAA;AAAA,cACzC;AAAA,YACF,CAAC,CAAA;AAED,YAAA,MAAM,IAAA,CAAK,SAAS,MAAA,CAAO;AAAA,cACzB,UAAA,EAAY;AAAA,aACb,CAAA;AAED,YAAA,KAAA,CAAM,yBAAA,EAA0B;AAAA,UAClC,SAAS,KAAA,EAAO;AACd,YAAAE,kBAAA,CAAY,KAAK,CAAA;AACjB,YAAA,KAAA,CAAM,WAAW,KAAK,CAAA;AAAA,UACxB;AAAA,QACF,CAAC,CAAA;AAAA,MACH;AAAA,KACD,CAAA;AAAA,EACH;AAAA,EAEQ,kBAAA,GAAiC;AACvC,IAAA,MAAM,cAAA,GACJ,IAAA,CAAK,MAAA,CAAO,iBAAA,CAAkB,wBAAwB,CAAA,IAAK,QAAA;AAC7D,IAAA,IAAI,mBAAmB,QAAA,EAAU;AAC/B,MAAA,OAAO,MAAM;AAAA,MAAC,CAAA;AAAA,IAChB;AAEA,IAAA,MAAM,iBAAA,GAAoBC,iCAAA,CAA4B,IAAA,CAAK,MAAM,CAAA;AAEjE,IAAA,MAAM,UAAU,YAAY;AAC1B,MAAA,IAAI;AACF,QAAA,MAAM,CAAA,GAAI,MAAMC,6CAAA,CAAuB;AAAA,UACrC,MAAM,IAAA,CAAK,IAAA;AAAA,UACX,QAAA,EAAU;AAAA,SACX,CAAA;AACD,QAAA,IAAI,IAAI,CAAA,EAAG;AACT,UAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,CAAA,QAAA,EAAW,CAAC,CAAA,kBAAA,CAAoB,CAAA;AAAA,QACnD;AAAA,MACF,SAAS,KAAA,EAAO;AACd,QAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,CAAA,kCAAA,CAAA,EAAsC,KAAK,CAAA;AAAA,MAC9D;AAAA,IACF,CAAA;AAEA,IAAA,MAAM,eAAA,GAAkB,IAAI,eAAA,EAAgB;AAC5C,IAAA,IAAA,CAAK,UAAU,YAAA,CAAa;AAAA,MAC1B,EAAA,EAAI,wBAAA;AAAA,MACJ,SAAA,EAAW,EAAE,YAAA,EAAc,IAAA,CAAK,uBAAA,EAAwB;AAAA,MACxD,OAAA,EAAS,EAAE,YAAA,EAAc,IAAA,CAAK,0BAA0B,GAAA,EAAI;AAAA,MAC5D,EAAA,EAAI,OAAA;AAAA,MACJ,QAAQ,eAAA,CAAgB;AAAA,KACzB,CAAA;AAED,IAAA,OAAO,MAAM;AACX,MAAA,eAAA,CAAgB,KAAA,EAAM;AAAA,IACxB,CAAA;AAAA,EACF;AACF;AAGA,SAAS,eAAA,GAAkB;AAEzB,EAAA,MAAM,wBAAwBC,2BAAA,CAAoB;AAAA,IAChD,IAAA,EAAM,kCAAA;AAAA,IACN,IAAA,EAAM,6EAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBC,2BAAA,CAAoB;AAAA,IACjD,IAAA,EAAM,qCAAA;AAAA,IACN,IAAA,EAAM,8FAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBA,2BAAA,CAAoB;AAAA,IACjD,IAAA,EAAM,qCAAA;AAAA,IACN,IAAA,EAAM,wFAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,2BAA2BA,2BAAA,CAAoB;AAAA,IACnD,IAAA,EAAM,wCAAA;AAAA,IACN,IAAA,EAAM;AAAA,GACP,CAAA;AAED,EAAA,MAAM,KAAA,GAAQC,WAAA,CAAQ,QAAA,CAAS,SAAS,CAAA;AACxC,EAAA,MAAM,oBAAoB,KAAA,CAAM,aAAA;AAAA,IAC9B,kCAAA;AAAA,IACA,EAAE,aAAa,8BAAA;AAA+B,GAChD;AAEA,EAAA,MAAM,qBAAqB,KAAA,CAAM,eAAA;AAAA,IAC/B,6BAAA;AAAA,IACA;AAAA,MACE,WAAA,EAAa,+CAAA;AAAA,MACb,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,MAAM,qBAAqB,KAAA,CAAM,eAAA;AAAA,IAC/B,6BAAA;AAAA,IACA;AAAA,MACE,WAAA,EAAa,yCAAA;AAAA,MACb,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,MAAM,uBAAuB,KAAA,CAAM,eAAA;AAAA,IACjC,gCAAA;AAAA,IACA;AAAA,MACE,WAAA,EACE,uGAAA;AAAA,MACF,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,SAAS,YAAA,CAAa,MAAwB,MAAA,EAAuB;AACnE,IAAA,MAAM,SAAA,GAAY,QAAQ,MAAA,EAAO;AACjC,IAAA,MAAM,eAAA,GAAkB,uBAAuB,UAAA,EAAW;AAC1D,IAAA,MAAM,kBAAA,GAAqB,uBAAuB,UAAA,EAAW;AAE7D,IAAA,MAAA,CAAO,KAAA,CAAM,CAAA,WAAA,EAAc,IAAA,CAAK,SAAS,CAAA,CAAE,CAAA;AAE3C,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,MAAM,UAAU,CAAC,IAAA,CAAK,aAAa,OAAA,EAAQ,CAAE,GAAG,SAAS,CAAA;AACzD,MAAA,wBAAA,CAAyB,QAAQ,OAAO,CAAA;AACxC,MAAA,oBAAA,CAAqB,OAAO,OAAO,CAAA;AAAA,IACrC;AAEA,IAAA,SAAS,OAAA,GAAU;AACjB,MAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,MAAA,CAAO,SAAS,CAAA;AACtC,MAAA,OAAO,KAAA,CAAM,CAAC,CAAA,GAAI,KAAA,CAAM,CAAC,CAAA,GAAI,GAAA;AAAA,IAC/B;AAEA,IAAA,SAAS,wBAAwB,MAAA,EAAgC;AAC/D,MAAA,kBAAA,CAAmB,EAAE,MAAA,EAAQ,MAAA,CAAO,EAAA,GAAK,IAAA,GAAO,UAAU,CAAA;AAC1D,MAAA,kBAAA,CAAmB,MAAA,CAAO,SAAQ,EAAG;AAAA,QACnC,MAAA,EAAQ,MAAA,CAAO,EAAA,GAAK,IAAA,GAAO;AAAA,OAC5B,CAAA;AAAA,IACH;AAEA,IAAA,SAAS,2BAAA,GAA8B;AACrC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,WAAA,EAAa,CAAA;AACvC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,WAAA,IAAe,CAAC,CAAA;AAEpD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAC5D,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAAA,IAClD;AAEA,IAAA,SAAS,wBAAA,GAA2B;AAClC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,QAAA,EAAU,CAAA;AACpC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,QAAA,IAAY,CAAC,CAAA;AAEjD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AACzD,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAAA,IAC/C;AAEA,IAAA,SAAS,yBAAA,GAA4B;AACnC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,SAAA,EAAW,CAAA;AACrC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,SAAA,IAAa,CAAC,CAAA;AAElD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAC1D,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAAA,IAChD;AAEA,IAAA,SAAS,WAAW,KAAA,EAAc;AAChC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,QAAA,IAAY,CAAC,CAAA;AACjD,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAC7C,MAAA,MAAA,CAAO,IAAA,CAAK,CAAA,cAAA,EAAiB,IAAA,CAAK,SAAS,WAAW,KAAK,CAAA;AAAA,IAC7D;AAEA,IAAA,OAAO;AAAA,MACL,uBAAA;AAAA,MACA,2BAAA;AAAA,MACA,wBAAA;AAAA,MACA,yBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF;AAEA,EAAA,OAAO,EAAE,YAAA,EAAa;AACxB;;;;"}

package/dist/processing/DefaultCatalogProcessingOrchestrator.cjs.js

@@ -190,9 +190,6 @@ class DefaultCatalogProcessingOrchestrator {
             );
             if (thisValid) {
               valid = true;
-              if (this.options.legacySingleProcessorValidation) {
-                break;
-              }
             }
           } catch (e) {
             throw new errors.InputError(