@backstage/plugin-catalog-backend 3.1.0 → 3.1.1-next.0

package/CHANGELOG.md

@@ -1,5 +1,48 @@
 # @backstage/plugin-catalog-backend
 
+## 3.1.1-next.0
+
+### Patch Changes
+
+- 9890488: Internal refactor to remove remnants of the old backend system
+- 2aaf01a: Fix for duplicate search results in entity-facets API call
+- e489661: Moved catalog processor and provider disabling and priorities under own config objects.
+
+  This is due to issue with some existing providers, such as GitHub, using array syntax for the provider configuration.
+
+  The new config format is not backwards compatible, so users will need to update their config files. The new format
+  is as follows:
+
+  ```yaml
+  catalog:
+    providerOptions:
+      providerA:
+        disabled: false
+      providerB:
+        disabled: true
+    processorOptions:
+      processorA:
+        disabled: false
+        priority: 10
+      processorB:
+        disabled: true
+  ```
+
+- Updated dependencies
+  - @backstage/integration@1.18.1-next.0
+  - @backstage/plugin-permission-node@0.10.4
+  - @backstage/backend-openapi-utils@0.6.1
+  - @backstage/backend-plugin-api@1.4.3
+  - @backstage/catalog-client@1.12.0
+  - @backstage/catalog-model@1.7.5
+  - @backstage/config@1.3.3
+  - @backstage/errors@1.2.7
+  - @backstage/types@1.2.2
+  - @backstage/plugin-catalog-common@1.1.5
+  - @backstage/plugin-catalog-node@1.19.0
+  - @backstage/plugin-events-node@0.4.15
+  - @backstage/plugin-permission-common@0.9.1
+
 ## 3.1.0
 
 ### Minor Changes

package/config.d.ts

@@ -223,38 +223,36 @@ export interface Config {
     processingInterval?: HumanDuration | false;
 
     /**
-     * Catalog provide specific configuration.
-     * Additional configuration for providers are specified in the catalog
-     * modules.
+     * Provider-specific additional configuration options.
      */
-    providers?: {
+    providerOptions?: {
       /**
-       * Name is the provider ID, e.g. "bitbucketServer"
+       * Key is the provider name, value is an object with additional configuration
        */
       [name: string]: {
         /**
-         * Whether the provider is enabled or not. Defaults to true.
+         * Determines whether this provider is disabled or not. If not specified,
+         * defaults to false.
          */
-        enabled?: boolean;
+        disabled?: boolean;
       };
     };
+
     /**
-     * Configuration for entity processors. Additional configuration for
-     * processors are specified in the catalog modules.
+     * Processor-specific additional configuration options.
      */
-    processors?: {
+    processorOptions?: {
       /**
-       * Name is the processor ID, e.g. "catalog-processor"
+       * Key is the processor name, value is an object with additional configuration
        */
       [name: string]: {
         /**
-         * Whether the processor is enabled or not. Defaults to true.
+         * Determines whether this processor is disabled or not. If not specified,
+         * defaults to false.
          */
-        enabled?: boolean;
+        disabled?: boolean;
         /**
-         * The priority of the processor, which is used to determine the order in which
-         * processors are run. The default priority is 20, and lower value means
-         * that the processor runs earlier.
+         * The default priority is 20, and lower value means that the processor runs earlier.
          */
         priority?: number;
       };

package/dist/database/DefaultProcessingDatabase.cjs.js

@@ -222,7 +222,7 @@ class DefaultProcessingDatabase {
         this.options.logger.warn(
           `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`
         );
-        if (this.options.eventBroker && locationKey) {
+        if (locationKey) {
           const eventParams = {
             topic: constants.CATALOG_CONFLICTS_TOPIC,
             eventPayload: {
@@ -233,7 +233,7 @@ class DefaultProcessingDatabase {
               lastConflictAt: luxon.DateTime.now().toISO()
             }
           };
-          await this.options.eventBroker?.publish(eventParams);
+          await this.options.events.publish(eventParams);
         }
       }
     }

package/dist/database/DefaultProcessingDatabase.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultProcessingDatabase.cjs.js","sources":["../../src/database/DefaultProcessingDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError } from '@backstage/errors';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { ProcessingIntervalFunction } from '../processing/refresh';\nimport { rethrowError, timestampToDateTime } from './conversion';\nimport { initDatabaseMetrics } from './metrics';\nimport {\n  DbRefreshKeysRow,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n} from './tables';\nimport {\n  GetProcessableEntitiesResult,\n  ListParentsOptions,\n  ListParentsResult,\n  ProcessingDatabase,\n  RefreshStateItem,\n  Transaction,\n  UpdateEntityCacheOptions,\n  UpdateProcessedEntityOptions,\n} from './types';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { generateStableHash, generateTargetKey } from './util';\nimport {\n  EventBroker,\n  EventParams,\n  EventsService,\n} from '@backstage/plugin-events-node';\nimport { DateTime } from 'luxon';\nimport { CATALOG_CONFLICTS_TOPIC } from '../constants';\nimport { CatalogConflictEventPayload } from '../catalog/types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProcessingDatabase implements ProcessingDatabase {\n  constructor(\n    private readonly options: {\n      database: Knex;\n      logger: LoggerService;\n      refreshInterval: ProcessingIntervalFunction;\n      eventBroker?: EventBroker | EventsService;\n    },\n  ) {\n    initDatabaseMetrics(options.database);\n  }\n\n  async updateProcessedEntity(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<{ previous: { relations: DbRelationsRow[] } }> {\n    const tx = txOpaque as Knex.Transaction;\n    const {\n      id,\n      processedEntity,\n      resultHash,\n      errors,\n      relations,\n      deferredEntities,\n      refreshKeys,\n      locationKey,\n    } = options;\n    const configClient = tx.client.config.client;\n    const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        processed_entity: JSON.stringify(processedEntity),\n        result_hash: resultHash,\n        errors,\n        location_key: locationKey,\n      })\n      .where('entity_id', id)\n      .andWhere(inner => {\n        if (!locationKey) {\n          return inner.whereNull('location_key');\n        }\n        return inner\n          .where('location_key', locationKey)\n          .orWhereNull('location_key');\n      });\n    if (refreshResult === 0) {\n      throw new ConflictError(\n        `Conflicting write of processing result for ${id} with location key '${locationKey}'`,\n      );\n    }\n    const sourceEntityRef = stringifyEntityRef(processedEntity);\n\n    // Schedule all deferred entities for future processing.\n    await this.addUnprocessedEntities(tx, {\n      entities: deferredEntities,\n      sourceEntityRef,\n    });\n\n    // Delete old relations\n    // NOTE(freben): knex implemented support for returning() on update queries for sqlite, but at the current time of writing (Sep 2022) not for delete() queries.\n    let previousRelationRows: DbRelationsRow[];\n    if (configClient.includes('sqlite3') || configClient.includes('mysql')) {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .select('*')\n        .where({ originating_entity_id: id });\n      await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete();\n    } else {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete()\n        .returning('*');\n    }\n\n    // Batch insert new relations\n    const relationRows: DbRelationsRow[] = relations.map(\n      ({ source, target, type }) => ({\n        originating_entity_id: id,\n        source_entity_ref: stringifyEntityRef(source),\n        target_entity_ref: stringifyEntityRef(target),\n        type,\n      }),\n    );\n\n    await tx.batchInsert(\n      'relations',\n      this.deduplicateRelations(relationRows),\n      BATCH_SIZE,\n    );\n\n    // Delete old refresh keys\n    await tx<DbRefreshKeysRow>('refresh_keys')\n      .where({ entity_id: id })\n      .delete();\n\n    // Insert the refresh keys for the processed entity\n    await tx.batchInsert(\n      'refresh_keys',\n      refreshKeys.map(k => ({\n        entity_id: id,\n        key: generateTargetKey(k.key),\n      })),\n      BATCH_SIZE,\n    );\n\n    return {\n      previous: {\n        relations: previousRelationRows,\n      },\n    };\n  }\n\n  async updateProcessedEntityErrors(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, errors, resultHash } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        errors,\n        result_hash: resultHash,\n      })\n      .where('entity_id', id);\n  }\n\n  async updateEntityCache(\n    txOpaque: Transaction,\n    options: UpdateEntityCacheOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, state } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({ cache: JSON.stringify(state ?? {}) })\n      .where('entity_id', id);\n  }\n\n  async getProcessableEntities(\n    maybeTx: Transaction | Knex,\n    request: { processBatchSize: number },\n  ): Promise<GetProcessableEntitiesResult> {\n    const knex = maybeTx as Knex.Transaction | Knex;\n\n    let itemsQuery = knex<DbRefreshStateRow>('refresh_state').select([\n      'entity_id',\n      'entity_ref',\n      'unprocessed_entity',\n      'result_hash',\n      'cache',\n      'errors',\n      'location_key',\n      'next_update_at',\n    ]);\n\n    // This avoids duplication of work because of race conditions and is\n    // also fast because locked rows are ignored rather than blocking.\n    // It's only available in MySQL and PostgreSQL\n    if (['mysql', 'mysql2', 'pg'].includes(knex.client.config.client)) {\n      itemsQuery = itemsQuery.forUpdate().skipLocked();\n    }\n\n    const items = await itemsQuery\n      .where('next_update_at', '<=', knex.fn.now())\n      .limit(request.processBatchSize)\n      .orderBy('next_update_at', 'asc');\n\n    const interval = this.options.refreshInterval();\n\n    const nextUpdateAt = (refreshInterval: number) => {\n      if (knex.client.config.client.includes('sqlite3')) {\n        return knex.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);\n      } else if (knex.client.config.client.includes('mysql')) {\n        return knex.raw(`now() + interval ${refreshInterval} second`);\n      }\n      return knex.raw(`now() + interval '${refreshInterval} seconds'`);\n    };\n\n    await knex<DbRefreshStateRow>('refresh_state')\n      .whereIn(\n        'entity_ref',\n        items.map(i => i.entity_ref),\n      )\n      .update({\n        next_update_at: nextUpdateAt(interval),\n      });\n\n    return {\n      items: items.map(\n        i =>\n          ({\n            id: i.entity_id,\n            entityRef: i.entity_ref,\n            unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity,\n            resultHash: i.result_hash || '',\n            nextUpdateAt: timestampToDateTime(i.next_update_at),\n            state: i.cache ? JSON.parse(i.cache) : undefined,\n            errors: i.errors,\n            locationKey: i.location_key,\n          } satisfies RefreshStateItem),\n      ),\n    };\n  }\n\n  async listParents(\n    txOpaque: Transaction,\n    options: ListParentsOptions,\n  ): Promise<ListParentsResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .whereIn('target_entity_ref', options.entityRefs)\n      .select();\n\n    const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);\n\n    return { entityRefs };\n  }\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] {\n    return lodash.uniqBy(\n      rows,\n      r => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`,\n    );\n  }\n\n  /**\n   * Add a set of deferred entities for processing.\n   * The entities will be added at the front of the processing queue.\n   */\n  private async addUnprocessedEntities(\n    txOpaque: Transaction,\n    options: {\n      sourceEntityRef: string;\n      entities: DeferredEntity[];\n    },\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n\n    // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards\n    const stateReferences = new Array<string>();\n\n    // Upsert all of the unprocessed entities into the refresh_state table, by\n    // their entity ref.\n    for (const { entity, locationKey } of options.entities) {\n      const entityRef = stringifyEntityRef(entity);\n      const hash = generateStableHash(entity);\n\n      const updated = await updateUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n      });\n      if (updated) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      const inserted = await insertUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n        logger: this.options.logger,\n      });\n      if (inserted) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      // If the row can't be inserted, we have a conflict, but it could be either\n      // because of a conflicting locationKey or a race with another instance, so check\n      // whether the conflicting entity has the same entityRef but a different locationKey\n      const conflictingKey = await checkLocationKeyConflict({\n        tx,\n        entityRef,\n        locationKey,\n      });\n      if (conflictingKey) {\n        this.options.logger.warn(\n          `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n        );\n        if (this.options.eventBroker && locationKey) {\n          const eventParams: EventParams<CatalogConflictEventPayload> = {\n            topic: CATALOG_CONFLICTS_TOPIC,\n            eventPayload: {\n              unprocessedEntity: entity,\n              entityRef,\n              newLocationKey: locationKey,\n              existingLocationKey: conflictingKey,\n              lastConflictAt: DateTime.now().toISO()!,\n            },\n          };\n          await this.options.eventBroker?.publish(eventParams);\n        }\n      }\n    }\n\n    // Lastly, replace refresh state references for the originating entity and any successfully added entities\n    await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n      // Remove all existing references from the originating entity\n      .where({ source_entity_ref: options.sourceEntityRef })\n      // And remove any existing references to entities that we're inserting new references for\n      .orWhereIn('target_entity_ref', stateReferences)\n      .delete();\n    await tx.batchInsert(\n      'refresh_state_references',\n      stateReferences.map(entityRef => ({\n        source_entity_ref: options.sourceEntityRef,\n        target_entity_ref: entityRef,\n      })),\n      BATCH_SIZE,\n    );\n  }\n}\n"],"names":["initDatabaseMetrics","errors","ConflictError","stringifyEntityRef","generateTargetKey","timestampToDateTime","rethrowError","lodash","generateStableHash","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","CATALOG_CONFLICTS_TOPIC","DateTime"],"mappings":";;;;;;;;;;;;;;;;;;AA0DA,MAAM,UAAA,GAAa,EAAA;AAEZ,MAAM,yBAAA,CAAwD;AAAA,EACnE,YACmB,OAAA,EAMjB;AANiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAOjB,IAAAA,2BAAA,CAAoB,QAAQ,QAAQ,CAAA;AAAA,EACtC;AAAA,EAEA,MAAM,qBAAA,CACJ,QAAA,EACA,OAAA,EACwD;AACxD,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM;AAAA,MACJ,EAAA;AAAA,MACA,eAAA;AAAA,MACA,UAAA;AAAA,cACAC,QAAA;AAAA,MACA,SAAA;AAAA,MACA,gBAAA;AAAA,MACA,WAAA;AAAA,MACA;AAAA,KACF,GAAI,OAAA;AACJ,IAAA,MAAM,YAAA,GAAe,EAAA,CAAG,MAAA,CAAO,MAAA,CAAO,MAAA;AACtC,IAAA,MAAM,aAAA,GAAgB,MAAM,EAAA,CAAsB,eAAe,EAC9D,MAAA,CAAO;AAAA,MACN,gBAAA,EAAkB,IAAA,CAAK,SAAA,CAAU,eAAe,CAAA;AAAA,MAChD,WAAA,EAAa,UAAA;AAAA,cACbA,QAAA;AAAA,MACA,YAAA,EAAc;AAAA,KACf,CAAA,CACA,KAAA,CAAM,aAAa,EAAE,CAAA,CACrB,SAAS,CAAA,KAAA,KAAS;AACjB,MAAA,IAAI,CAAC,WAAA,EAAa;AAChB,QAAA,OAAO,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA,MACvC;AACA,MAAA,OAAO,MACJ,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,IAC/B,CAAC,CAAA;AACH,IAAA,IAAI,kBAAkB,CAAA,EAAG;AACvB,MAAA,MAAM,IAAIC,oBAAA;AAAA,QACR,CAAA,2CAAA,EAA8C,EAAE,CAAA,oBAAA,EAAuB,WAAW,CAAA,CAAA;AAAA,OACpF;AAAA,IACF;AACA,IAAA,MAAM,eAAA,GAAkBC,gCAAmB,eAAe,CAAA;AAG1D,IAAA,MAAM,IAAA,CAAK,uBAAuB,EAAA,EAAI;AAAA,MACpC,QAAA,EAAU,gBAAA;AAAA,MACV;AAAA,KACD,CAAA;AAID,IAAA,IAAI,oBAAA;AACJ,IAAA,IAAI,aAAa,QAAA,CAAS,SAAS,KAAK,YAAA,CAAa,QAAA,CAAS,OAAO,CAAA,EAAG;AACtE,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,MAAA,CAAO,GAAG,CAAA,CACV,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA;AACtC,MAAA,MAAM,EAAA,CAAmB,WAAW,CAAA,CACjC,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO;AAAA,IACZ,CAAA,MAAO;AACL,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO,CACP,UAAU,GAAG,CAAA;AAAA,IAClB;AAGA,IAAA,MAAM,eAAiC,SAAA,CAAU,GAAA;AAAA,MAC/C,CAAC,EAAE,MAAA,EAAQ,MAAA,EAAQ,MAAK,MAAO;AAAA,QAC7B,qBAAA,EAAuB,EAAA;AAAA,QACvB,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C;AAAA,OACF;AAAA,KACF;AAEA,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,WAAA;AAAA,MACA,IAAA,CAAK,qBAAqB,YAAY,CAAA;AAAA,MACtC;AAAA,KACF;AAGA,IAAA,MAAM,EAAA,CAAqB,cAAc,CAAA,CACtC,KAAA,CAAM,EAAE,SAAA,EAAW,EAAA,EAAI,CAAA,CACvB,MAAA,EAAO;AAGV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,cAAA;AAAA,MACA,WAAA,CAAY,IAAI,CAAA,CAAA,MAAM;AAAA,QACpB,SAAA,EAAW,EAAA;AAAA,QACX,GAAA,EAAKC,sBAAA,CAAkB,CAAA,CAAE,GAAG;AAAA,OAC9B,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAEA,IAAA,OAAO;AAAA,MACL,QAAA,EAAU;AAAA,QACR,SAAA,EAAW;AAAA;AACb,KACF;AAAA,EACF;AAAA,EAEA,MAAM,2BAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,MAAA,EAAQ,UAAA,EAAW,GAAI,OAAA;AAEnC,IAAA,MAAM,EAAA,CAAsB,eAAe,CAAA,CACxC,MAAA,CAAO;AAAA,MACN,MAAA;AAAA,MACA,WAAA,EAAa;AAAA,KACd,CAAA,CACA,KAAA,CAAM,WAAA,EAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,iBAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,KAAA,EAAM,GAAI,OAAA;AAEtB,IAAA,MAAM,GAAsB,eAAe,CAAA,CACxC,MAAA,CAAO,EAAE,OAAO,IAAA,CAAK,SAAA,CAAU,KAAA,IAAS,EAAE,CAAA,EAAG,CAAA,CAC7C,KAAA,CAAM,aAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,sBAAA,CACJ,OAAA,EACA,OAAA,EACuC;AACvC,IAAA,MAAM,IAAA,GAAO,OAAA;AAEb,IAAA,IAAI,UAAA,GAAa,IAAA,CAAwB,eAAe,CAAA,CAAE,MAAA,CAAO;AAAA,MAC/D,WAAA;AAAA,MACA,YAAA;AAAA,MACA,oBAAA;AAAA,MACA,aAAA;AAAA,MACA,OAAA;AAAA,MACA,QAAA;AAAA,MACA,cAAA;AAAA,MACA;AAAA,KACD,CAAA;AAKD,IAAA,IAAI,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,SAAS,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,MAAM,CAAA,EAAG;AACjE,MAAA,UAAA,GAAa,UAAA,CAAW,SAAA,EAAU,CAAE,UAAA,EAAW;AAAA,IACjD;AAEA,IAAA,MAAM,QAAQ,MAAM,UAAA,CACjB,KAAA,CAAM,gBAAA,EAAkB,MAAM,IAAA,CAAK,EAAA,CAAG,GAAA,EAAK,EAC3C,KAAA,CAAM,OAAA,CAAQ,gBAAgB,CAAA,CAC9B,OAAA,CAAQ,kBAAkB,KAAK,CAAA;AAElC,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,OAAA,CAAQ,eAAA,EAAgB;AAE9C,IAAA,MAAM,YAAA,GAAe,CAAC,eAAA,KAA4B;AAChD,MAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AACjD,QAAA,OAAO,KAAK,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,eAAe,UAAU,CAAC,CAAA;AAAA,MACtE,WAAW,IAAA,CAAK,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACtD,QAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,iBAAA,EAAoB,eAAe,CAAA,OAAA,CAAS,CAAA;AAAA,MAC9D;AACA,MAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,kBAAA,EAAqB,eAAe,CAAA,SAAA,CAAW,CAAA;AAAA,IACjE,CAAA;AAEA,IAAA,MAAM,IAAA,CAAwB,eAAe,CAAA,CAC1C,OAAA;AAAA,MACC,YAAA;AAAA,MACA,KAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,MAE5B,MAAA,CAAO;AAAA,MACN,cAAA,EAAgB,aAAa,QAAQ;AAAA,KACtC,CAAA;AAEH,IAAA,OAAO;AAAA,MACL,OAAO,KAAA,CAAM,GAAA;AAAA,QACX,CAAA,CAAA,MACG;AAAA,UACC,IAAI,CAAA,CAAE,SAAA;AAAA,UACN,WAAW,CAAA,CAAE,UAAA;AAAA,UACb,iBAAA,EAAmB,IAAA,CAAK,KAAA,CAAM,CAAA,CAAE,kBAAkB,CAAA;AAAA,UAClD,UAAA,EAAY,EAAE,WAAA,IAAe,EAAA;AAAA,UAC7B,YAAA,EAAcC,8BAAA,CAAoB,CAAA,CAAE,cAAc,CAAA;AAAA,UAClD,OAAO,CAAA,CAAE,KAAA,GAAQ,KAAK,KAAA,CAAM,CAAA,CAAE,KAAK,CAAA,GAAI,MAAA;AAAA,UACvC,QAAQ,CAAA,CAAE,MAAA;AAAA,UACV,aAAa,CAAA,CAAE;AAAA,SACjB;AAAA;AACJ,KACF;AAAA,EACF;AAAA,EAEA,MAAM,WAAA,CACJ,QAAA,EACA,OAAA,EAC4B;AAC5B,IAAA,MAAM,EAAA,GAAK,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,MAEC,OAAA,CAAQ,mBAAA,EAAqB,OAAA,CAAQ,UAAU,EAC/C,MAAA,EAAO;AAEV,IAAA,MAAM,UAAA,GAAa,KAAK,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,iBAAkB,CAAA,CAAE,OAAO,OAAO,CAAA;AAErE,IAAA,OAAO,EAAE,UAAA,EAAW;AAAA,EACtB;AAAA,EAEA,MAAM,YAAe,EAAA,EAAiD;AACpE,IAAA,IAAI;AACF,MAAA,IAAI,MAAA,GAAwB,KAAA,CAAA;AAE5B,MAAA,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,WAAA;AAAA,QAC1B,OAAM,EAAA,KAAM;AAGV,UAAA,MAAA,GAAS,MAAM,GAAG,EAAE,CAAA;AAAA,QACtB,CAAA;AAAA,QACA;AAAA;AAAA,UAEE,qBAAA,EAAuB;AAAA;AACzB,OACF;AAEA,MAAA,OAAO,MAAA;AAAA,IACT,SAAS,CAAA,EAAG;AACV,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,0BAAA,EAA6B,CAAC,CAAA,CAAE,CAAA;AAC1D,MAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA,IACtB;AAAA,EACF;AAAA,EAEQ,qBAAqB,IAAA,EAA0C;AACrE,IAAA,OAAOC,uBAAA,CAAO,MAAA;AAAA,MACZ,IAAA;AAAA,MACA,CAAA,CAAA,KAAK,GAAG,CAAA,CAAE,iBAAiB,IAAI,CAAA,CAAE,iBAAiB,CAAA,CAAA,EAAI,CAAA,CAAE,IAAI,CAAA;AAAA,KAC9D;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,sBAAA,CACZ,QAAA,EACA,OAAA,EAIe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AAGX,IAAA,MAAM,eAAA,GAAkB,IAAI,KAAA,EAAc;AAI1C,IAAA,KAAA,MAAW,EAAE,MAAA,EAAQ,WAAA,EAAY,IAAK,QAAQ,QAAA,EAAU;AACtD,MAAA,MAAM,SAAA,GAAYJ,gCAAmB,MAAM,CAAA;AAC3C,MAAA,MAAM,IAAA,GAAOK,wBAAmB,MAAM,CAAA;AAEtC,MAAA,MAAM,OAAA,GAAU,MAAMC,+CAAA,CAAwB;AAAA,QAC5C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,OAAA,EAAS;AACX,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,QAAA,GAAW,MAAMC,+CAAA,CAAwB;AAAA,QAC7C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA,WAAA;AAAA,QACA,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAKA,MAAA,MAAM,cAAA,GAAiB,MAAMC,iDAAA,CAAyB;AAAA,QACpD,EAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,cAAA,EAAgB;AAClB,QAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,UAClB,CAAA,+BAAA,EAAkC,SAAS,CAAA,uBAAA,EAA0B,cAAc,iBAAiB,WAAW,CAAA;AAAA,SACjH;AACA,QAAA,IAAI,IAAA,CAAK,OAAA,CAAQ,WAAA,IAAe,WAAA,EAAa;AAC3C,UAAA,MAAM,WAAA,GAAwD;AAAA,YAC5D,KAAA,EAAOC,iCAAA;AAAA,YACP,YAAA,EAAc;AAAA,cACZ,iBAAA,EAAmB,MAAA;AAAA,cACnB,SAAA;AAAA,cACA,cAAA,EAAgB,WAAA;AAAA,cAChB,mBAAA,EAAqB,cAAA;AAAA,cACrB,cAAA,EAAgBC,cAAA,CAAS,GAAA,EAAI,CAAE,KAAA;AAAM;AACvC,WACF;AACA,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,WAAA,EAAa,OAAA,CAAQ,WAAW,CAAA;AAAA,QACrD;AAAA,MACF;AAAA,IACF;AAGA,IAAA,MAAM,EAAA,CAAgC,0BAA0B,CAAA,CAE7D,KAAA,CAAM,EAAE,iBAAA,EAAmB,OAAA,CAAQ,eAAA,EAAiB,CAAA,CAEpD,SAAA,CAAU,mBAAA,EAAqB,eAAe,EAC9C,MAAA,EAAO;AACV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,0BAAA;AAAA,MACA,eAAA,CAAgB,IAAI,CAAA,SAAA,MAAc;AAAA,QAChC,mBAAmB,OAAA,CAAQ,eAAA;AAAA,QAC3B,iBAAA,EAAmB;AAAA,OACrB,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAAA,EACF;AACF;;;;"}
+{"version":3,"file":"DefaultProcessingDatabase.cjs.js","sources":["../../src/database/DefaultProcessingDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError } from '@backstage/errors';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { ProcessingIntervalFunction } from '../processing/refresh';\nimport { rethrowError, timestampToDateTime } from './conversion';\nimport { initDatabaseMetrics } from './metrics';\nimport {\n  DbRefreshKeysRow,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n} from './tables';\nimport {\n  GetProcessableEntitiesResult,\n  ListParentsOptions,\n  ListParentsResult,\n  ProcessingDatabase,\n  RefreshStateItem,\n  Transaction,\n  UpdateEntityCacheOptions,\n  UpdateProcessedEntityOptions,\n} from './types';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { generateStableHash, generateTargetKey } from './util';\nimport { EventParams, EventsService } from '@backstage/plugin-events-node';\nimport { DateTime } from 'luxon';\nimport { CATALOG_CONFLICTS_TOPIC } from '../constants';\nimport { CatalogConflictEventPayload } from '../catalog/types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProcessingDatabase implements ProcessingDatabase {\n  constructor(\n    private readonly options: {\n      database: Knex;\n      logger: LoggerService;\n      refreshInterval: ProcessingIntervalFunction;\n      events: EventsService;\n    },\n  ) {\n    initDatabaseMetrics(options.database);\n  }\n\n  async updateProcessedEntity(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<{ previous: { relations: DbRelationsRow[] } }> {\n    const tx = txOpaque as Knex.Transaction;\n    const {\n      id,\n      processedEntity,\n      resultHash,\n      errors,\n      relations,\n      deferredEntities,\n      refreshKeys,\n      locationKey,\n    } = options;\n    const configClient = tx.client.config.client;\n    const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        processed_entity: JSON.stringify(processedEntity),\n        result_hash: resultHash,\n        errors,\n        location_key: locationKey,\n      })\n      .where('entity_id', id)\n      .andWhere(inner => {\n        if (!locationKey) {\n          return inner.whereNull('location_key');\n        }\n        return inner\n          .where('location_key', locationKey)\n          .orWhereNull('location_key');\n      });\n    if (refreshResult === 0) {\n      throw new ConflictError(\n        `Conflicting write of processing result for ${id} with location key '${locationKey}'`,\n      );\n    }\n    const sourceEntityRef = stringifyEntityRef(processedEntity);\n\n    // Schedule all deferred entities for future processing.\n    await this.addUnprocessedEntities(tx, {\n      entities: deferredEntities,\n      sourceEntityRef,\n    });\n\n    // Delete old relations\n    // NOTE(freben): knex implemented support for returning() on update queries for sqlite, but at the current time of writing (Sep 2022) not for delete() queries.\n    let previousRelationRows: DbRelationsRow[];\n    if (configClient.includes('sqlite3') || configClient.includes('mysql')) {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .select('*')\n        .where({ originating_entity_id: id });\n      await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete();\n    } else {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete()\n        .returning('*');\n    }\n\n    // Batch insert new relations\n    const relationRows: DbRelationsRow[] = relations.map(\n      ({ source, target, type }) => ({\n        originating_entity_id: id,\n        source_entity_ref: stringifyEntityRef(source),\n        target_entity_ref: stringifyEntityRef(target),\n        type,\n      }),\n    );\n\n    await tx.batchInsert(\n      'relations',\n      this.deduplicateRelations(relationRows),\n      BATCH_SIZE,\n    );\n\n    // Delete old refresh keys\n    await tx<DbRefreshKeysRow>('refresh_keys')\n      .where({ entity_id: id })\n      .delete();\n\n    // Insert the refresh keys for the processed entity\n    await tx.batchInsert(\n      'refresh_keys',\n      refreshKeys.map(k => ({\n        entity_id: id,\n        key: generateTargetKey(k.key),\n      })),\n      BATCH_SIZE,\n    );\n\n    return {\n      previous: {\n        relations: previousRelationRows,\n      },\n    };\n  }\n\n  async updateProcessedEntityErrors(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, errors, resultHash } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        errors,\n        result_hash: resultHash,\n      })\n      .where('entity_id', id);\n  }\n\n  async updateEntityCache(\n    txOpaque: Transaction,\n    options: UpdateEntityCacheOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, state } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({ cache: JSON.stringify(state ?? {}) })\n      .where('entity_id', id);\n  }\n\n  async getProcessableEntities(\n    maybeTx: Transaction | Knex,\n    request: { processBatchSize: number },\n  ): Promise<GetProcessableEntitiesResult> {\n    const knex = maybeTx as Knex.Transaction | Knex;\n\n    let itemsQuery = knex<DbRefreshStateRow>('refresh_state').select([\n      'entity_id',\n      'entity_ref',\n      'unprocessed_entity',\n      'result_hash',\n      'cache',\n      'errors',\n      'location_key',\n      'next_update_at',\n    ]);\n\n    // This avoids duplication of work because of race conditions and is\n    // also fast because locked rows are ignored rather than blocking.\n    // It's only available in MySQL and PostgreSQL\n    if (['mysql', 'mysql2', 'pg'].includes(knex.client.config.client)) {\n      itemsQuery = itemsQuery.forUpdate().skipLocked();\n    }\n\n    const items = await itemsQuery\n      .where('next_update_at', '<=', knex.fn.now())\n      .limit(request.processBatchSize)\n      .orderBy('next_update_at', 'asc');\n\n    const interval = this.options.refreshInterval();\n\n    const nextUpdateAt = (refreshInterval: number) => {\n      if (knex.client.config.client.includes('sqlite3')) {\n        return knex.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);\n      } else if (knex.client.config.client.includes('mysql')) {\n        return knex.raw(`now() + interval ${refreshInterval} second`);\n      }\n      return knex.raw(`now() + interval '${refreshInterval} seconds'`);\n    };\n\n    await knex<DbRefreshStateRow>('refresh_state')\n      .whereIn(\n        'entity_ref',\n        items.map(i => i.entity_ref),\n      )\n      .update({\n        next_update_at: nextUpdateAt(interval),\n      });\n\n    return {\n      items: items.map(\n        i =>\n          ({\n            id: i.entity_id,\n            entityRef: i.entity_ref,\n            unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity,\n            resultHash: i.result_hash || '',\n            nextUpdateAt: timestampToDateTime(i.next_update_at),\n            state: i.cache ? JSON.parse(i.cache) : undefined,\n            errors: i.errors,\n            locationKey: i.location_key,\n          } satisfies RefreshStateItem),\n      ),\n    };\n  }\n\n  async listParents(\n    txOpaque: Transaction,\n    options: ListParentsOptions,\n  ): Promise<ListParentsResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .whereIn('target_entity_ref', options.entityRefs)\n      .select();\n\n    const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);\n\n    return { entityRefs };\n  }\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] {\n    return lodash.uniqBy(\n      rows,\n      r => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`,\n    );\n  }\n\n  /**\n   * Add a set of deferred entities for processing.\n   * The entities will be added at the front of the processing queue.\n   */\n  private async addUnprocessedEntities(\n    txOpaque: Transaction,\n    options: {\n      sourceEntityRef: string;\n      entities: DeferredEntity[];\n    },\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n\n    // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards\n    const stateReferences = new Array<string>();\n\n    // Upsert all of the unprocessed entities into the refresh_state table, by\n    // their entity ref.\n    for (const { entity, locationKey } of options.entities) {\n      const entityRef = stringifyEntityRef(entity);\n      const hash = generateStableHash(entity);\n\n      const updated = await updateUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n      });\n      if (updated) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      const inserted = await insertUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n        logger: this.options.logger,\n      });\n      if (inserted) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      // If the row can't be inserted, we have a conflict, but it could be either\n      // because of a conflicting locationKey or a race with another instance, so check\n      // whether the conflicting entity has the same entityRef but a different locationKey\n      const conflictingKey = await checkLocationKeyConflict({\n        tx,\n        entityRef,\n        locationKey,\n      });\n      if (conflictingKey) {\n        this.options.logger.warn(\n          `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n        );\n        if (locationKey) {\n          const eventParams: EventParams<CatalogConflictEventPayload> = {\n            topic: CATALOG_CONFLICTS_TOPIC,\n            eventPayload: {\n              unprocessedEntity: entity,\n              entityRef,\n              newLocationKey: locationKey,\n              existingLocationKey: conflictingKey,\n              lastConflictAt: DateTime.now().toISO()!,\n            },\n          };\n          await this.options.events.publish(eventParams);\n        }\n      }\n    }\n\n    // Lastly, replace refresh state references for the originating entity and any successfully added entities\n    await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n      // Remove all existing references from the originating entity\n      .where({ source_entity_ref: options.sourceEntityRef })\n      // And remove any existing references to entities that we're inserting new references for\n      .orWhereIn('target_entity_ref', stateReferences)\n      .delete();\n    await tx.batchInsert(\n      'refresh_state_references',\n      stateReferences.map(entityRef => ({\n        source_entity_ref: options.sourceEntityRef,\n        target_entity_ref: entityRef,\n      })),\n      BATCH_SIZE,\n    );\n  }\n}\n"],"names":["initDatabaseMetrics","errors","ConflictError","stringifyEntityRef","generateTargetKey","timestampToDateTime","rethrowError","lodash","generateStableHash","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","CATALOG_CONFLICTS_TOPIC","DateTime"],"mappings":";;;;;;;;;;;;;;;;;;AAsDA,MAAM,UAAA,GAAa,EAAA;AAEZ,MAAM,yBAAA,CAAwD;AAAA,EACnE,YACmB,OAAA,EAMjB;AANiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAOjB,IAAAA,2BAAA,CAAoB,QAAQ,QAAQ,CAAA;AAAA,EACtC;AAAA,EAEA,MAAM,qBAAA,CACJ,QAAA,EACA,OAAA,EACwD;AACxD,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM;AAAA,MACJ,EAAA;AAAA,MACA,eAAA;AAAA,MACA,UAAA;AAAA,cACAC,QAAA;AAAA,MACA,SAAA;AAAA,MACA,gBAAA;AAAA,MACA,WAAA;AAAA,MACA;AAAA,KACF,GAAI,OAAA;AACJ,IAAA,MAAM,YAAA,GAAe,EAAA,CAAG,MAAA,CAAO,MAAA,CAAO,MAAA;AACtC,IAAA,MAAM,aAAA,GAAgB,MAAM,EAAA,CAAsB,eAAe,EAC9D,MAAA,CAAO;AAAA,MACN,gBAAA,EAAkB,IAAA,CAAK,SAAA,CAAU,eAAe,CAAA;AAAA,MAChD,WAAA,EAAa,UAAA;AAAA,cACbA,QAAA;AAAA,MACA,YAAA,EAAc;AAAA,KACf,CAAA,CACA,KAAA,CAAM,aAAa,EAAE,CAAA,CACrB,SAAS,CAAA,KAAA,KAAS;AACjB,MAAA,IAAI,CAAC,WAAA,EAAa;AAChB,QAAA,OAAO,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA,MACvC;AACA,MAAA,OAAO,MACJ,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,IAC/B,CAAC,CAAA;AACH,IAAA,IAAI,kBAAkB,CAAA,EAAG;AACvB,MAAA,MAAM,IAAIC,oBAAA;AAAA,QACR,CAAA,2CAAA,EAA8C,EAAE,CAAA,oBAAA,EAAuB,WAAW,CAAA,CAAA;AAAA,OACpF;AAAA,IACF;AACA,IAAA,MAAM,eAAA,GAAkBC,gCAAmB,eAAe,CAAA;AAG1D,IAAA,MAAM,IAAA,CAAK,uBAAuB,EAAA,EAAI;AAAA,MACpC,QAAA,EAAU,gBAAA;AAAA,MACV;AAAA,KACD,CAAA;AAID,IAAA,IAAI,oBAAA;AACJ,IAAA,IAAI,aAAa,QAAA,CAAS,SAAS,KAAK,YAAA,CAAa,QAAA,CAAS,OAAO,CAAA,EAAG;AACtE,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,MAAA,CAAO,GAAG,CAAA,CACV,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA;AACtC,MAAA,MAAM,EAAA,CAAmB,WAAW,CAAA,CACjC,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO;AAAA,IACZ,CAAA,MAAO;AACL,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO,CACP,UAAU,GAAG,CAAA;AAAA,IAClB;AAGA,IAAA,MAAM,eAAiC,SAAA,CAAU,GAAA;AAAA,MAC/C,CAAC,EAAE,MAAA,EAAQ,MAAA,EAAQ,MAAK,MAAO;AAAA,QAC7B,qBAAA,EAAuB,EAAA;AAAA,QACvB,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C;AAAA,OACF;AAAA,KACF;AAEA,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,WAAA;AAAA,MACA,IAAA,CAAK,qBAAqB,YAAY,CAAA;AAAA,MACtC;AAAA,KACF;AAGA,IAAA,MAAM,EAAA,CAAqB,cAAc,CAAA,CACtC,KAAA,CAAM,EAAE,SAAA,EAAW,EAAA,EAAI,CAAA,CACvB,MAAA,EAAO;AAGV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,cAAA;AAAA,MACA,WAAA,CAAY,IAAI,CAAA,CAAA,MAAM;AAAA,QACpB,SAAA,EAAW,EAAA;AAAA,QACX,GAAA,EAAKC,sBAAA,CAAkB,CAAA,CAAE,GAAG;AAAA,OAC9B,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAEA,IAAA,OAAO;AAAA,MACL,QAAA,EAAU;AAAA,QACR,SAAA,EAAW;AAAA;AACb,KACF;AAAA,EACF;AAAA,EAEA,MAAM,2BAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,MAAA,EAAQ,UAAA,EAAW,GAAI,OAAA;AAEnC,IAAA,MAAM,EAAA,CAAsB,eAAe,CAAA,CACxC,MAAA,CAAO;AAAA,MACN,MAAA;AAAA,MACA,WAAA,EAAa;AAAA,KACd,CAAA,CACA,KAAA,CAAM,WAAA,EAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,iBAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,KAAA,EAAM,GAAI,OAAA;AAEtB,IAAA,MAAM,GAAsB,eAAe,CAAA,CACxC,MAAA,CAAO,EAAE,OAAO,IAAA,CAAK,SAAA,CAAU,KAAA,IAAS,EAAE,CAAA,EAAG,CAAA,CAC7C,KAAA,CAAM,aAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,sBAAA,CACJ,OAAA,EACA,OAAA,EACuC;AACvC,IAAA,MAAM,IAAA,GAAO,OAAA;AAEb,IAAA,IAAI,UAAA,GAAa,IAAA,CAAwB,eAAe,CAAA,CAAE,MAAA,CAAO;AAAA,MAC/D,WAAA;AAAA,MACA,YAAA;AAAA,MACA,oBAAA;AAAA,MACA,aAAA;AAAA,MACA,OAAA;AAAA,MACA,QAAA;AAAA,MACA,cAAA;AAAA,MACA;AAAA,KACD,CAAA;AAKD,IAAA,IAAI,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,SAAS,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,MAAM,CAAA,EAAG;AACjE,MAAA,UAAA,GAAa,UAAA,CAAW,SAAA,EAAU,CAAE,UAAA,EAAW;AAAA,IACjD;AAEA,IAAA,MAAM,QAAQ,MAAM,UAAA,CACjB,KAAA,CAAM,gBAAA,EAAkB,MAAM,IAAA,CAAK,EAAA,CAAG,GAAA,EAAK,EAC3C,KAAA,CAAM,OAAA,CAAQ,gBAAgB,CAAA,CAC9B,OAAA,CAAQ,kBAAkB,KAAK,CAAA;AAElC,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,OAAA,CAAQ,eAAA,EAAgB;AAE9C,IAAA,MAAM,YAAA,GAAe,CAAC,eAAA,KAA4B;AAChD,MAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AACjD,QAAA,OAAO,KAAK,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,eAAe,UAAU,CAAC,CAAA;AAAA,MACtE,WAAW,IAAA,CAAK,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACtD,QAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,iBAAA,EAAoB,eAAe,CAAA,OAAA,CAAS,CAAA;AAAA,MAC9D;AACA,MAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,kBAAA,EAAqB,eAAe,CAAA,SAAA,CAAW,CAAA;AAAA,IACjE,CAAA;AAEA,IAAA,MAAM,IAAA,CAAwB,eAAe,CAAA,CAC1C,OAAA;AAAA,MACC,YAAA;AAAA,MACA,KAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,MAE5B,MAAA,CAAO;AAAA,MACN,cAAA,EAAgB,aAAa,QAAQ;AAAA,KACtC,CAAA;AAEH,IAAA,OAAO;AAAA,MACL,OAAO,KAAA,CAAM,GAAA;AAAA,QACX,CAAA,CAAA,MACG;AAAA,UACC,IAAI,CAAA,CAAE,SAAA;AAAA,UACN,WAAW,CAAA,CAAE,UAAA;AAAA,UACb,iBAAA,EAAmB,IAAA,CAAK,KAAA,CAAM,CAAA,CAAE,kBAAkB,CAAA;AAAA,UAClD,UAAA,EAAY,EAAE,WAAA,IAAe,EAAA;AAAA,UAC7B,YAAA,EAAcC,8BAAA,CAAoB,CAAA,CAAE,cAAc,CAAA;AAAA,UAClD,OAAO,CAAA,CAAE,KAAA,GAAQ,KAAK,KAAA,CAAM,CAAA,CAAE,KAAK,CAAA,GAAI,MAAA;AAAA,UACvC,QAAQ,CAAA,CAAE,MAAA;AAAA,UACV,aAAa,CAAA,CAAE;AAAA,SACjB;AAAA;AACJ,KACF;AAAA,EACF;AAAA,EAEA,MAAM,WAAA,CACJ,QAAA,EACA,OAAA,EAC4B;AAC5B,IAAA,MAAM,EAAA,GAAK,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,MAEC,OAAA,CAAQ,mBAAA,EAAqB,OAAA,CAAQ,UAAU,EAC/C,MAAA,EAAO;AAEV,IAAA,MAAM,UAAA,GAAa,KAAK,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,iBAAkB,CAAA,CAAE,OAAO,OAAO,CAAA;AAErE,IAAA,OAAO,EAAE,UAAA,EAAW;AAAA,EACtB;AAAA,EAEA,MAAM,YAAe,EAAA,EAAiD;AACpE,IAAA,IAAI;AACF,MAAA,IAAI,MAAA,GAAwB,KAAA,CAAA;AAE5B,MAAA,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,WAAA;AAAA,QAC1B,OAAM,EAAA,KAAM;AAGV,UAAA,MAAA,GAAS,MAAM,GAAG,EAAE,CAAA;AAAA,QACtB,CAAA;AAAA,QACA;AAAA;AAAA,UAEE,qBAAA,EAAuB;AAAA;AACzB,OACF;AAEA,MAAA,OAAO,MAAA;AAAA,IACT,SAAS,CAAA,EAAG;AACV,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,0BAAA,EAA6B,CAAC,CAAA,CAAE,CAAA;AAC1D,MAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA,IACtB;AAAA,EACF;AAAA,EAEQ,qBAAqB,IAAA,EAA0C;AACrE,IAAA,OAAOC,uBAAA,CAAO,MAAA;AAAA,MACZ,IAAA;AAAA,MACA,CAAA,CAAA,KAAK,GAAG,CAAA,CAAE,iBAAiB,IAAI,CAAA,CAAE,iBAAiB,CAAA,CAAA,EAAI,CAAA,CAAE,IAAI,CAAA;AAAA,KAC9D;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,sBAAA,CACZ,QAAA,EACA,OAAA,EAIe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AAGX,IAAA,MAAM,eAAA,GAAkB,IAAI,KAAA,EAAc;AAI1C,IAAA,KAAA,MAAW,EAAE,MAAA,EAAQ,WAAA,EAAY,IAAK,QAAQ,QAAA,EAAU;AACtD,MAAA,MAAM,SAAA,GAAYJ,gCAAmB,MAAM,CAAA;AAC3C,MAAA,MAAM,IAAA,GAAOK,wBAAmB,MAAM,CAAA;AAEtC,MAAA,MAAM,OAAA,GAAU,MAAMC,+CAAA,CAAwB;AAAA,QAC5C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,OAAA,EAAS;AACX,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,QAAA,GAAW,MAAMC,+CAAA,CAAwB;AAAA,QAC7C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA,WAAA;AAAA,QACA,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAKA,MAAA,MAAM,cAAA,GAAiB,MAAMC,iDAAA,CAAyB;AAAA,QACpD,EAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,cAAA,EAAgB;AAClB,QAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,UAClB,CAAA,+BAAA,EAAkC,SAAS,CAAA,uBAAA,EAA0B,cAAc,iBAAiB,WAAW,CAAA;AAAA,SACjH;AACA,QAAA,IAAI,WAAA,EAAa;AACf,UAAA,MAAM,WAAA,GAAwD;AAAA,YAC5D,KAAA,EAAOC,iCAAA;AAAA,YACP,YAAA,EAAc;AAAA,cACZ,iBAAA,EAAmB,MAAA;AAAA,cACnB,SAAA;AAAA,cACA,cAAA,EAAgB,WAAA;AAAA,cAChB,mBAAA,EAAqB,cAAA;AAAA,cACrB,cAAA,EAAgBC,cAAA,CAAS,GAAA,EAAI,CAAE,KAAA;AAAM;AACvC,WACF;AACA,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,OAAA,CAAQ,WAAW,CAAA;AAAA,QAC/C;AAAA,MACF;AAAA,IACF;AAGA,IAAA,MAAM,EAAA,CAAgC,0BAA0B,CAAA,CAE7D,KAAA,CAAM,EAAE,iBAAA,EAAmB,OAAA,CAAQ,eAAA,EAAiB,CAAA,CAEpD,SAAA,CAAU,mBAAA,EAAqB,eAAe,EAC9C,MAAA,EAAO;AACV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,0BAAA;AAAA,MACA,eAAA,CAAgB,IAAI,CAAA,SAAA,MAAc;AAAA,QAChC,mBAAmB,OAAA,CAAQ,eAAA;AAAA,QAC3B,iBAAA,EAAmB;AAAA,OACrB,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAAA,EACF;AACF;;;;"}

package/dist/processing/DefaultCatalogProcessingEngine.cjs.js

@@ -34,7 +34,7 @@ class DefaultCatalogProcessingEngine {
   orphanCleanupIntervalMs;
   onProcessingError;
   tracker;
-  eventBroker;
+  events;
   stopFunc;
   constructor(options) {
     this.config = options.config;
@@ -49,7 +49,7 @@ class DefaultCatalogProcessingEngine {
     this.orphanCleanupIntervalMs = options.orphanCleanupIntervalMs ?? 3e4;
     this.onProcessingError = options.onProcessingError;
     this.tracker = options.tracker ?? progressTracker();
-    this.eventBroker = options.eventBroker;
+    this.events = options.events;
     this.stopFunc = void 0;
   }
   async start() {
@@ -128,7 +128,7 @@ class DefaultCatalogProcessingEngine {
             }
             const location = unprocessedEntity?.metadata?.annotations?.[catalogModel.ANNOTATION_LOCATION];
             if (result.errors.length) {
-              this.eventBroker?.publish({
+              this.events.publish({
                 topic: constants.CATALOG_ERRORS_TOPIC,
                 eventPayload: {
                   entity: entityRef,
@@ -260,22 +260,16 @@ class DefaultCatalogProcessingEngine {
         this.logger.warn(`Failed to delete orphaned entities`, error);
       }
     };
-    if (this.scheduler) {
-      const abortController = new AbortController();
-      this.scheduler.scheduleTask({
-        id: "catalog_orphan_cleanup",
-        frequency: { milliseconds: this.orphanCleanupIntervalMs },
-        timeout: { milliseconds: this.orphanCleanupIntervalMs * 0.8 },
-        fn: runOnce,
-        signal: abortController.signal
-      });
-      return () => {
-        abortController.abort();
-      };
-    }
-    const intervalKey = setInterval(runOnce, this.orphanCleanupIntervalMs);
+    const abortController = new AbortController();
+    this.scheduler.scheduleTask({
+      id: "catalog_orphan_cleanup",
+      frequency: { milliseconds: this.orphanCleanupIntervalMs },
+      timeout: { milliseconds: this.orphanCleanupIntervalMs * 0.8 },
+      fn: runOnce,
+      signal: abortController.signal
+    });
     return () => {
-      clearInterval(intervalKey);
+      abortController.abort();
     };
   }
 }

package/dist/processing/DefaultCatalogProcessingEngine.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultCatalogProcessingEngine.cjs.js","sources":["../../src/processing/DefaultCatalogProcessingEngine.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  ANNOTATION_LOCATION,\n  Entity,\n  stringifyEntityRef,\n} from '@backstage/catalog-model';\nimport { assertError, serializeError, stringifyError } from '@backstage/errors';\nimport { Hash } from 'crypto';\nimport stableStringify from 'fast-json-stable-stringify';\nimport { Knex } from 'knex';\nimport { metrics, trace } from '@opentelemetry/api';\nimport { ProcessingDatabase, RefreshStateItem } from '../database/types';\nimport { createCounterMetric, createSummaryMetric } from '../util/metrics';\nimport { CatalogProcessingOrchestrator, EntityProcessingResult } from './types';\nimport { Stitcher, stitchingStrategyFromConfig } from '../stitching/types';\nimport { startTaskPipeline } from './TaskPipeline';\nimport { Config } from '@backstage/config';\nimport {\n  addEntityAttributes,\n  TRACER_ID,\n  withActiveSpan,\n} from '../util/opentelemetry';\nimport { deleteOrphanedEntities } from '../database/operations/util/deleteOrphanedEntities';\nimport { EventBroker, EventsService } from '@backstage/plugin-events-node';\nimport { CATALOG_ERRORS_TOPIC } from '../constants';\nimport { LoggerService, SchedulerService } from '@backstage/backend-plugin-api';\n\nconst CACHE_TTL = 5;\n\nconst tracer = trace.getTracer(TRACER_ID);\n\nexport type ProgressTracker = ReturnType<typeof progressTracker>;\n\nconst stableStringifyArray = (arr: any[]) => {\n  const sorted = arr.map(stableStringify).sort();\n  return `[${sorted.join(',')}]`;\n};\n\n// NOTE(freben): Perhaps surprisingly, this class does not implement the\n// CatalogProcessingEngine type. That type is externally visible and its name is\n// the way it is for historic reasons. This class has no particular reason to\n// implement that precise interface; nowadays there are several different\n// engines \"hiding\" behind the CatalogProcessingEngine interface, of which this\n// is just one.\nexport class DefaultCatalogProcessingEngine {\n  private readonly config: Config;\n  private readonly scheduler?: SchedulerService;\n  private readonly logger: LoggerService;\n  private readonly knex: Knex;\n  private readonly processingDatabase: ProcessingDatabase;\n  private readonly orchestrator: CatalogProcessingOrchestrator;\n  private readonly stitcher: Stitcher;\n  private readonly createHash: () => Hash;\n  private readonly pollingIntervalMs: number;\n  private readonly orphanCleanupIntervalMs: number;\n  private readonly onProcessingError?: (event: {\n    unprocessedEntity: Entity;\n    errors: Error[];\n  }) => Promise<void> | void;\n  private readonly tracker: ProgressTracker;\n  private readonly eventBroker?: EventBroker | EventsService;\n\n  private stopFunc?: () => void;\n\n  constructor(options: {\n    config: Config;\n    scheduler?: SchedulerService;\n    logger: LoggerService;\n    knex: Knex;\n    processingDatabase: ProcessingDatabase;\n    orchestrator: CatalogProcessingOrchestrator;\n    stitcher: Stitcher;\n    createHash: () => Hash;\n    pollingIntervalMs?: number;\n    orphanCleanupIntervalMs?: number;\n    onProcessingError?: (event: {\n      unprocessedEntity: Entity;\n      errors: Error[];\n    }) => Promise<void> | void;\n    tracker?: ProgressTracker;\n    eventBroker?: EventBroker | EventsService;\n  }) {\n    this.config = options.config;\n    this.scheduler = options.scheduler;\n    this.logger = options.logger;\n    this.knex = options.knex;\n    this.processingDatabase = options.processingDatabase;\n    this.orchestrator = options.orchestrator;\n    this.stitcher = options.stitcher;\n    this.createHash = options.createHash;\n    this.pollingIntervalMs = options.pollingIntervalMs ?? 1_000;\n    this.orphanCleanupIntervalMs = options.orphanCleanupIntervalMs ?? 30_000;\n    this.onProcessingError = options.onProcessingError;\n    this.tracker = options.tracker ?? progressTracker();\n    this.eventBroker = options.eventBroker;\n\n    this.stopFunc = undefined;\n  }\n\n  async start() {\n    if (this.stopFunc) {\n      throw new Error('Processing engine is already started');\n    }\n\n    const stopPipeline = this.startPipeline();\n    const stopCleanup = this.startOrphanCleanup();\n\n    this.stopFunc = () => {\n      stopPipeline();\n      stopCleanup();\n    };\n  }\n\n  async stop() {\n    if (this.stopFunc) {\n      this.stopFunc();\n      this.stopFunc = undefined;\n    }\n  }\n\n  private startPipeline(): () => void {\n    return startTaskPipeline<RefreshStateItem>({\n      lowWatermark: 5,\n      highWatermark: 10,\n      pollingIntervalMs: this.pollingIntervalMs,\n      loadTasks: async count => {\n        try {\n          const { items } =\n            await this.processingDatabase.getProcessableEntities(this.knex, {\n              processBatchSize: count,\n            });\n          return items;\n        } catch (error) {\n          this.logger.warn('Failed to load processing items', error);\n          return [];\n        }\n      },\n      processTask: async item => {\n        await withActiveSpan(tracer, 'ProcessingRun', async span => {\n          const track = this.tracker.processStart(item, this.logger);\n          addEntityAttributes(span, item.unprocessedEntity);\n\n          try {\n            const {\n              id,\n              state,\n              unprocessedEntity,\n              entityRef,\n              locationKey,\n              resultHash: previousResultHash,\n            } = item;\n            const result = await this.orchestrator.process({\n              entity: unprocessedEntity,\n              state,\n            });\n\n            track.markProcessorsCompleted(result);\n\n            if (result.ok) {\n              const { ttl: _, ...stateWithoutTtl } = state ?? {};\n              if (\n                stableStringify(stateWithoutTtl) !==\n                stableStringify(result.state)\n              ) {\n                await this.processingDatabase.transaction(async tx => {\n                  await this.processingDatabase.updateEntityCache(tx, {\n                    id,\n                    state: {\n                      ttl: CACHE_TTL,\n                      ...result.state,\n                    },\n                  });\n                });\n              }\n            } else {\n              const maybeTtl = state?.ttl;\n              const ttl = Number.isInteger(maybeTtl) ? (maybeTtl as number) : 0;\n              await this.processingDatabase.transaction(async tx => {\n                await this.processingDatabase.updateEntityCache(tx, {\n                  id,\n                  state: ttl > 0 ? { ...state, ttl: ttl - 1 } : {},\n                });\n              });\n            }\n\n            const location =\n              unprocessedEntity?.metadata?.annotations?.[ANNOTATION_LOCATION];\n            if (result.errors.length) {\n              this.eventBroker?.publish({\n                topic: CATALOG_ERRORS_TOPIC,\n                eventPayload: {\n                  entity: entityRef,\n                  location,\n                  errors: result.errors,\n                },\n              });\n            }\n            const errorsString = JSON.stringify(\n              result.errors.map(e => serializeError(e)),\n            );\n\n            let hashBuilder = this.createHash().update(errorsString);\n\n            if (result.ok) {\n              const { entityRefs: parents } =\n                await this.processingDatabase.transaction(tx =>\n                  this.processingDatabase.listParents(tx, {\n                    entityRefs: [\n                      entityRef,\n                      ...result.deferredEntities.map(e =>\n                        stringifyEntityRef(e.entity),\n                      ),\n                    ],\n                  }),\n                );\n\n              hashBuilder = hashBuilder\n                .update(stableStringify({ ...result.completedEntity }))\n                .update(stableStringifyArray([...result.deferredEntities]))\n                .update(stableStringifyArray([...result.relations]))\n                .update(stableStringifyArray([...result.refreshKeys]))\n                .update(stableStringifyArray([...parents]));\n            }\n\n            const resultHash = hashBuilder.digest('hex');\n            if (resultHash === previousResultHash) {\n              // If nothing changed in our produced outputs, we cannot have any\n              // significant effect on our surroundings; therefore, we just abort\n              // without any updates / stitching.\n              track.markSuccessfulWithNoChanges();\n              return;\n            }\n\n            // If the result was marked as not OK, it signals that some part of the\n            // processing pipeline threw an exception. This can happen both as part of\n            // non-catastrophic things such as due to validation errors, as well as if\n            // something fatal happens inside the processing for other reasons. In any\n            // case, this means we can't trust that anything in the output is okay. So\n            // just store the errors and trigger a stich so that they become visible to\n            // the outside.\n            if (!result.ok) {\n              // notify the error listener if the entity can not be processed.\n              Promise.resolve(undefined)\n                .then(() =>\n                  this.onProcessingError?.({\n                    unprocessedEntity,\n                    errors: result.errors,\n                  }),\n                )\n                .catch(error => {\n                  this.logger.debug(\n                    `Processing error listener threw an exception, ${stringifyError(\n                      error,\n                    )}`,\n                  );\n                });\n\n              await this.processingDatabase.transaction(async tx => {\n                await this.processingDatabase.updateProcessedEntityErrors(tx, {\n                  id,\n                  errors: errorsString,\n                  resultHash,\n                });\n              });\n\n              await this.stitcher.stitch({\n                entityRefs: [stringifyEntityRef(unprocessedEntity)],\n              });\n\n              track.markSuccessfulWithErrors();\n              return;\n            }\n\n            result.completedEntity.metadata.uid = id;\n            let oldRelationSources: Map<string, string>;\n            await this.processingDatabase.transaction(async tx => {\n              const { previous } =\n                await this.processingDatabase.updateProcessedEntity(tx, {\n                  id,\n                  processedEntity: result.completedEntity,\n                  resultHash,\n                  errors: errorsString,\n                  relations: result.relations,\n                  deferredEntities: result.deferredEntities,\n                  locationKey,\n                  refreshKeys: result.refreshKeys,\n                });\n              oldRelationSources = new Map(\n                previous.relations.map(r => [\n                  `${r.source_entity_ref}:${r.type}->${r.target_entity_ref}`,\n                  r.source_entity_ref,\n                ]),\n              );\n            });\n\n            const newRelationSources = new Map<string, string>(\n              result.relations.map(relation => {\n                const sourceEntityRef = stringifyEntityRef(relation.source);\n                const targetEntityRef = stringifyEntityRef(relation.target);\n                return [\n                  `${sourceEntityRef}:${relation.type}->${targetEntityRef}`,\n                  sourceEntityRef,\n                ];\n              }),\n            );\n\n            const setOfThingsToStitch = new Set<string>([\n              stringifyEntityRef(result.completedEntity),\n            ]);\n            newRelationSources.forEach((sourceEntityRef, uniqueKey) => {\n              if (!oldRelationSources.has(uniqueKey)) {\n                setOfThingsToStitch.add(sourceEntityRef);\n              }\n            });\n            oldRelationSources!.forEach((sourceEntityRef, uniqueKey) => {\n              if (!newRelationSources.has(uniqueKey)) {\n                setOfThingsToStitch.add(sourceEntityRef);\n              }\n            });\n\n            await this.stitcher.stitch({\n              entityRefs: setOfThingsToStitch,\n            });\n\n            track.markSuccessfulWithChanges();\n          } catch (error) {\n            assertError(error);\n            track.markFailed(error);\n          }\n        });\n      },\n    });\n  }\n\n  private startOrphanCleanup(): () => void {\n    const orphanStrategy =\n      this.config.getOptionalString('catalog.orphanStrategy') ?? 'delete';\n    if (orphanStrategy !== 'delete') {\n      return () => {};\n    }\n\n    const stitchingStrategy = stitchingStrategyFromConfig(this.config);\n\n    const runOnce = async () => {\n      try {\n        const n = await deleteOrphanedEntities({\n          knex: this.knex,\n          strategy: stitchingStrategy,\n        });\n        if (n > 0) {\n          this.logger.info(`Deleted ${n} orphaned entities`);\n        }\n      } catch (error) {\n        this.logger.warn(`Failed to delete orphaned entities`, error);\n      }\n    };\n\n    if (this.scheduler) {\n      const abortController = new AbortController();\n\n      this.scheduler.scheduleTask({\n        id: 'catalog_orphan_cleanup',\n        frequency: { milliseconds: this.orphanCleanupIntervalMs },\n        timeout: { milliseconds: this.orphanCleanupIntervalMs * 0.8 },\n        fn: runOnce,\n        signal: abortController.signal,\n      });\n\n      return () => {\n        abortController.abort();\n      };\n    }\n\n    const intervalKey = setInterval(runOnce, this.orphanCleanupIntervalMs);\n    return () => {\n      clearInterval(intervalKey);\n    };\n  }\n}\n\n// Helps wrap the timing and logging behaviors\nfunction progressTracker() {\n  // prom-client metrics are deprecated in favour of OpenTelemetry metrics.\n  const promProcessedEntities = createCounterMetric({\n    name: 'catalog_processed_entities_count',\n    help: 'Amount of entities processed, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingDuration = createSummaryMetric({\n    name: 'catalog_processing_duration_seconds',\n    help: 'Time spent executing the full processing flow, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessorsDuration = createSummaryMetric({\n    name: 'catalog_processors_duration_seconds',\n    help: 'Time spent executing catalog processors, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingQueueDelay = createSummaryMetric({\n    name: 'catalog_processing_queue_delay_seconds',\n    help: 'The amount of delay between being scheduled for processing, and the start of actually being processed, DEPRECATED, use OpenTelemetry metrics instead',\n  });\n\n  const meter = metrics.getMeter('default');\n  const processedEntities = meter.createCounter(\n    'catalog.processed.entities.count',\n    { description: 'Amount of entities processed' },\n  );\n\n  const processingDuration = meter.createHistogram(\n    'catalog.processing.duration',\n    {\n      description: 'Time spent executing the full processing flow',\n      unit: 'seconds',\n    },\n  );\n\n  const processorsDuration = meter.createHistogram(\n    'catalog.processors.duration',\n    {\n      description: 'Time spent executing catalog processors',\n      unit: 'seconds',\n    },\n  );\n\n  const processingQueueDelay = meter.createHistogram(\n    'catalog.processing.queue.delay',\n    {\n      description:\n        'The amount of delay between being scheduled for processing, and the start of actually being processed',\n      unit: 'seconds',\n    },\n  );\n\n  function processStart(item: RefreshStateItem, logger: LoggerService) {\n    const startTime = process.hrtime();\n    const endOverallTimer = promProcessingDuration.startTimer();\n    const endProcessorsTimer = promProcessorsDuration.startTimer();\n\n    logger.debug(`Processing ${item.entityRef}`);\n\n    if (item.nextUpdateAt) {\n      const seconds = -item.nextUpdateAt.diffNow().as('seconds');\n      promProcessingQueueDelay.observe(seconds);\n      processingQueueDelay.record(seconds);\n    }\n\n    function endTime() {\n      const delta = process.hrtime(startTime);\n      return delta[0] + delta[1] / 1e9;\n    }\n\n    function markProcessorsCompleted(result: EntityProcessingResult) {\n      endProcessorsTimer({ result: result.ok ? 'ok' : 'failed' });\n      processorsDuration.record(endTime(), {\n        result: result.ok ? 'ok' : 'failed',\n      });\n    }\n\n    function markSuccessfulWithNoChanges() {\n      endOverallTimer({ result: 'unchanged' });\n      promProcessedEntities.inc({ result: 'unchanged' }, 1);\n\n      processingDuration.record(endTime(), { result: 'unchanged' });\n      processedEntities.add(1, { result: 'unchanged' });\n    }\n\n    function markSuccessfulWithErrors() {\n      endOverallTimer({ result: 'errors' });\n      promProcessedEntities.inc({ result: 'errors' }, 1);\n\n      processingDuration.record(endTime(), { result: 'errors' });\n      processedEntities.add(1, { result: 'errors' });\n    }\n\n    function markSuccessfulWithChanges() {\n      endOverallTimer({ result: 'changed' });\n      promProcessedEntities.inc({ result: 'changed' }, 1);\n\n      processingDuration.record(endTime(), { result: 'changed' });\n      processedEntities.add(1, { result: 'changed' });\n    }\n\n    function markFailed(error: Error) {\n      promProcessedEntities.inc({ result: 'failed' }, 1);\n      processedEntities.add(1, { result: 'failed' });\n      logger.warn(`Processing of ${item.entityRef} failed`, error);\n    }\n\n    return {\n      markProcessorsCompleted,\n      markSuccessfulWithNoChanges,\n      markSuccessfulWithErrors,\n      markSuccessfulWithChanges,\n      markFailed,\n    };\n  }\n\n  return { processStart };\n}\n"],"names":["trace","TRACER_ID","stableStringify","startTaskPipeline","withActiveSpan","addEntityAttributes","ANNOTATION_LOCATION","CATALOG_ERRORS_TOPIC","serializeError","stringifyEntityRef","stringifyError","assertError","stitchingStrategyFromConfig","deleteOrphanedEntities","createCounterMetric","createSummaryMetric","metrics"],"mappings":";;;;;;;;;;;;;;;;;AA0CA,MAAM,SAAA,GAAY,CAAA;AAElB,MAAM,MAAA,GAASA,SAAA,CAAM,SAAA,CAAUC,uBAAS,CAAA;AAIxC,MAAM,oBAAA,GAAuB,CAAC,GAAA,KAAe;AAC3C,EAAA,MAAM,MAAA,GAAS,GAAA,CAAI,GAAA,CAAIC,gCAAe,EAAE,IAAA,EAAK;AAC7C,EAAA,OAAO,CAAA,CAAA,EAAI,MAAA,CAAO,IAAA,CAAK,GAAG,CAAC,CAAA,CAAA,CAAA;AAC7B,CAAA;AAQO,MAAM,8BAAA,CAA+B;AAAA,EACzB,MAAA;AAAA,EACA,SAAA;AAAA,EACA,MAAA;AAAA,EACA,IAAA;AAAA,EACA,kBAAA;AAAA,EACA,YAAA;AAAA,EACA,QAAA;AAAA,EACA,UAAA;AAAA,EACA,iBAAA;AAAA,EACA,uBAAA;AAAA,EACA,iBAAA;AAAA,EAIA,OAAA;AAAA,EACA,WAAA;AAAA,EAET,QAAA;AAAA,EAER,YAAY,OAAA,EAiBT;AACD,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,YAAY,OAAA,CAAQ,SAAA;AACzB,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,OAAO,OAAA,CAAQ,IAAA;AACpB,IAAA,IAAA,CAAK,qBAAqB,OAAA,CAAQ,kBAAA;AAClC,IAAA,IAAA,CAAK,eAAe,OAAA,CAAQ,YAAA;AAC5B,IAAA,IAAA,CAAK,WAAW,OAAA,CAAQ,QAAA;AACxB,IAAA,IAAA,CAAK,aAAa,OAAA,CAAQ,UAAA;AAC1B,IAAA,IAAA,CAAK,iBAAA,GAAoB,QAAQ,iBAAA,IAAqB,GAAA;AACtD,IAAA,IAAA,CAAK,uBAAA,GAA0B,QAAQ,uBAAA,IAA2B,GAAA;AAClE,IAAA,IAAA,CAAK,oBAAoB,OAAA,CAAQ,iBAAA;AACjC,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA,CAAQ,OAAA,IAAW,eAAA,EAAgB;AAClD,IAAA,IAAA,CAAK,cAAc,OAAA,CAAQ,WAAA;AAE3B,IAAA,IAAA,CAAK,QAAA,GAAW,MAAA;AAAA,EAClB;AAAA,EAEA,MAAM,KAAA,GAAQ;AACZ,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,MAAM,IAAI,MAAM,sCAAsC,CAAA;AAAA,IACxD;AAEA,IAAA,MAAM,YAAA,GAAe,KAAK,aAAA,EAAc;AACxC,IAAA,MAAM,WAAA,GAAc,KAAK,kBAAA,EAAmB;AAE5C,IAAA,IAAA,CAAK,WAAW,MAAM;AACpB,MAAA,YAAA,EAAa;AACb,MAAA,WAAA,EAAY;AAAA,IACd,CAAA;AAAA,EACF;AAAA,EAEA,MAAM,IAAA,GAAO;AACX,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,IAAA,CAAK,QAAA,EAAS;AACd,MAAA,IAAA,CAAK,QAAA,GAAW,MAAA;AAAA,IAClB;AAAA,EACF;AAAA,EAEQ,aAAA,GAA4B;AAClC,IAAA,OAAOC,8BAAA,CAAoC;AAAA,MACzC,YAAA,EAAc,CAAA;AAAA,MACd,aAAA,EAAe,EAAA;AAAA,MACf,mBAAmB,IAAA,CAAK,iBAAA;AAAA,MACxB,SAAA,EAAW,OAAM,KAAA,KAAS;AACxB,QAAA,IAAI;AACF,UAAA,MAAM,EAAE,OAAM,GACZ,MAAM,KAAK,kBAAA,CAAmB,sBAAA,CAAuB,KAAK,IAAA,EAAM;AAAA,YAC9D,gBAAA,EAAkB;AAAA,WACnB,CAAA;AACH,UAAA,OAAO,KAAA;AAAA,QACT,SAAS,KAAA,EAAO;AACd,UAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,iCAAA,EAAmC,KAAK,CAAA;AACzD,UAAA,OAAO,EAAC;AAAA,QACV;AAAA,MACF,CAAA;AAAA,MACA,WAAA,EAAa,OAAM,IAAA,KAAQ;AACzB,QAAA,MAAMC,4BAAA,CAAe,MAAA,EAAQ,eAAA,EAAiB,OAAM,IAAA,KAAQ;AAC1D,UAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,YAAA,CAAa,IAAA,EAAM,KAAK,MAAM,CAAA;AACzD,UAAAC,iCAAA,CAAoB,IAAA,EAAM,KAAK,iBAAiB,CAAA;AAEhD,UAAA,IAAI;AACF,YAAA,MAAM;AAAA,cACJ,EAAA;AAAA,cACA,KAAA;AAAA,cACA,iBAAA;AAAA,cACA,SAAA;AAAA,cACA,WAAA;AAAA,cACA,UAAA,EAAY;AAAA,aACd,GAAI,IAAA;AACJ,YAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,YAAA,CAAa,OAAA,CAAQ;AAAA,cAC7C,MAAA,EAAQ,iBAAA;AAAA,cACR;AAAA,aACD,CAAA;AAED,YAAA,KAAA,CAAM,wBAAwB,MAAM,CAAA;AAEpC,YAAA,IAAI,OAAO,EAAA,EAAI;AACb,cAAA,MAAM,EAAE,GAAA,EAAK,CAAA,EAAG,GAAG,eAAA,EAAgB,GAAI,SAAS,EAAC;AACjD,cAAA,IACEH,iCAAgB,eAAe,CAAA,KAC/BA,gCAAA,CAAgB,MAAA,CAAO,KAAK,CAAA,EAC5B;AACA,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,kBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,iBAAA,CAAkB,EAAA,EAAI;AAAA,oBAClD,EAAA;AAAA,oBACA,KAAA,EAAO;AAAA,sBACL,GAAA,EAAK,SAAA;AAAA,sBACL,GAAG,MAAA,CAAO;AAAA;AACZ,mBACD,CAAA;AAAA,gBACH,CAAC,CAAA;AAAA,cACH;AAAA,YACF,CAAA,MAAO;AACL,cAAA,MAAM,WAAW,KAAA,EAAO,GAAA;AACxB,cAAA,MAAM,GAAA,GAAM,MAAA,CAAO,SAAA,CAAU,QAAQ,IAAK,QAAA,GAAsB,CAAA;AAChE,cAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,iBAAA,CAAkB,EAAA,EAAI;AAAA,kBAClD,EAAA;AAAA,kBACA,KAAA,EAAO,GAAA,GAAM,CAAA,GAAI,EAAE,GAAG,OAAO,GAAA,EAAK,GAAA,GAAM,CAAA,EAAE,GAAI;AAAC,iBAChD,CAAA;AAAA,cACH,CAAC,CAAA;AAAA,YACH;AAEA,YAAA,MAAM,QAAA,GACJ,iBAAA,EAAmB,QAAA,EAAU,WAAA,GAAcI,gCAAmB,CAAA;AAChE,YAAA,IAAI,MAAA,CAAO,OAAO,MAAA,EAAQ;AACxB,cAAA,IAAA,CAAK,aAAa,OAAA,CAAQ;AAAA,gBACxB,KAAA,EAAOC,8BAAA;AAAA,gBACP,YAAA,EAAc;AAAA,kBACZ,MAAA,EAAQ,SAAA;AAAA,kBACR,QAAA;AAAA,kBACA,QAAQ,MAAA,CAAO;AAAA;AACjB,eACD,CAAA;AAAA,YACH;AACA,YAAA,MAAM,eAAe,IAAA,CAAK,SAAA;AAAA,cACxB,OAAO,MAAA,CAAO,GAAA,CAAI,CAAA,CAAA,KAAKC,qBAAA,CAAe,CAAC,CAAC;AAAA,aAC1C;AAEA,YAAA,IAAI,WAAA,GAAc,IAAA,CAAK,UAAA,EAAW,CAAE,OAAO,YAAY,CAAA;AAEvD,YAAA,IAAI,OAAO,EAAA,EAAI;AACb,cAAA,MAAM,EAAE,UAAA,EAAY,OAAA,EAAQ,GAC1B,MAAM,KAAK,kBAAA,CAAmB,WAAA;AAAA,gBAAY,CAAA,EAAA,KACxC,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,EAAA,EAAI;AAAA,kBACtC,UAAA,EAAY;AAAA,oBACV,SAAA;AAAA,oBACA,GAAG,OAAO,gBAAA,CAAiB,GAAA;AAAA,sBAAI,CAAA,CAAA,KAC7BC,+BAAA,CAAmB,CAAA,CAAE,MAAM;AAAA;AAC7B;AACF,iBACD;AAAA,eACH;AAEF,cAAA,WAAA,GAAc,YACX,MAAA,CAAOP,gCAAA,CAAgB,EAAE,GAAG,MAAA,CAAO,iBAAiB,CAAC,EACrD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,gBAAgB,CAAC,CAAC,EACzD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,SAAS,CAAC,CAAC,EAClD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,WAAW,CAAC,CAAC,EACpD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,OAAO,CAAC,CAAC,CAAA;AAAA,YAC9C;AAEA,YAAA,MAAM,UAAA,GAAa,WAAA,CAAY,MAAA,CAAO,KAAK,CAAA;AAC3C,YAAA,IAAI,eAAe,kBAAA,EAAoB;AAIrC,cAAA,KAAA,CAAM,2BAAA,EAA4B;AAClC,cAAA;AAAA,YACF;AASA,YAAA,IAAI,CAAC,OAAO,EAAA,EAAI;AAEd,cAAA,OAAA,CAAQ,OAAA,CAAQ,MAAS,CAAA,CACtB,IAAA;AAAA,gBAAK,MACJ,KAAK,iBAAA,GAAoB;AAAA,kBACvB,iBAAA;AAAA,kBACA,QAAQ,MAAA,CAAO;AAAA,iBAChB;AAAA,eACH,CACC,MAAM,CAAA,KAAA,KAAS;AACd,gBAAA,IAAA,CAAK,MAAA,CAAO,KAAA;AAAA,kBACV,CAAA,8CAAA,EAAiDQ,qBAAA;AAAA,oBAC/C;AAAA,mBACD,CAAA;AAAA,iBACH;AAAA,cACF,CAAC,CAAA;AAEH,cAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,2BAAA,CAA4B,EAAA,EAAI;AAAA,kBAC5D,EAAA;AAAA,kBACA,MAAA,EAAQ,YAAA;AAAA,kBACR;AAAA,iBACD,CAAA;AAAA,cACH,CAAC,CAAA;AAED,cAAA,MAAM,IAAA,CAAK,SAAS,MAAA,CAAO;AAAA,gBACzB,UAAA,EAAY,CAACD,+BAAA,CAAmB,iBAAiB,CAAC;AAAA,eACnD,CAAA;AAED,cAAA,KAAA,CAAM,wBAAA,EAAyB;AAC/B,cAAA;AAAA,YACF;AAEA,YAAA,MAAA,CAAO,eAAA,CAAgB,SAAS,GAAA,GAAM,EAAA;AACtC,YAAA,IAAI,kBAAA;AACJ,YAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,cAAA,MAAM,EAAE,QAAA,EAAS,GACf,MAAM,IAAA,CAAK,kBAAA,CAAmB,sBAAsB,EAAA,EAAI;AAAA,gBACtD,EAAA;AAAA,gBACA,iBAAiB,MAAA,CAAO,eAAA;AAAA,gBACxB,UAAA;AAAA,gBACA,MAAA,EAAQ,YAAA;AAAA,gBACR,WAAW,MAAA,CAAO,SAAA;AAAA,gBAClB,kBAAkB,MAAA,CAAO,gBAAA;AAAA,gBACzB,WAAA;AAAA,gBACA,aAAa,MAAA,CAAO;AAAA,eACrB,CAAA;AACH,cAAA,kBAAA,GAAqB,IAAI,GAAA;AAAA,gBACvB,QAAA,CAAS,SAAA,CAAU,GAAA,CAAI,CAAA,CAAA,KAAK;AAAA,kBAC1B,CAAA,EAAG,EAAE,iBAAiB,CAAA,CAAA,EAAI,EAAE,IAAI,CAAA,EAAA,EAAK,EAAE,iBAAiB,CAAA,CAAA;AAAA,kBACxD,CAAA,CAAE;AAAA,iBACH;AAAA,eACH;AAAA,YACF,CAAC,CAAA;AAED,YAAA,MAAM,qBAAqB,IAAI,GAAA;AAAA,cAC7B,MAAA,CAAO,SAAA,CAAU,GAAA,CAAI,CAAA,QAAA,KAAY;AAC/B,gBAAA,MAAM,eAAA,GAAkBA,+BAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAC1D,gBAAA,MAAM,eAAA,GAAkBA,+BAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAC1D,gBAAA,OAAO;AAAA,kBACL,GAAG,eAAe,CAAA,CAAA,EAAI,QAAA,CAAS,IAAI,KAAK,eAAe,CAAA,CAAA;AAAA,kBACvD;AAAA,iBACF;AAAA,cACF,CAAC;AAAA,aACH;AAEA,YAAA,MAAM,mBAAA,uBAA0B,GAAA,CAAY;AAAA,cAC1CA,+BAAA,CAAmB,OAAO,eAAe;AAAA,aAC1C,CAAA;AACD,YAAA,kBAAA,CAAmB,OAAA,CAAQ,CAAC,eAAA,EAAiB,SAAA,KAAc;AACzD,cAAA,IAAI,CAAC,kBAAA,CAAmB,GAAA,CAAI,SAAS,CAAA,EAAG;AACtC,gBAAA,mBAAA,CAAoB,IAAI,eAAe,CAAA;AAAA,cACzC;AAAA,YACF,CAAC,CAAA;AACD,YAAA,kBAAA,CAAoB,OAAA,CAAQ,CAAC,eAAA,EAAiB,SAAA,KAAc;AAC1D,cAAA,IAAI,CAAC,kBAAA,CAAmB,GAAA,CAAI,SAAS,CAAA,EAAG;AACtC,gBAAA,mBAAA,CAAoB,IAAI,eAAe,CAAA;AAAA,cACzC;AAAA,YACF,CAAC,CAAA;AAED,YAAA,MAAM,IAAA,CAAK,SAAS,MAAA,CAAO;AAAA,cACzB,UAAA,EAAY;AAAA,aACb,CAAA;AAED,YAAA,KAAA,CAAM,yBAAA,EAA0B;AAAA,UAClC,SAAS,KAAA,EAAO;AACd,YAAAE,kBAAA,CAAY,KAAK,CAAA;AACjB,YAAA,KAAA,CAAM,WAAW,KAAK,CAAA;AAAA,UACxB;AAAA,QACF,CAAC,CAAA;AAAA,MACH;AAAA,KACD,CAAA;AAAA,EACH;AAAA,EAEQ,kBAAA,GAAiC;AACvC,IAAA,MAAM,cAAA,GACJ,IAAA,CAAK,MAAA,CAAO,iBAAA,CAAkB,wBAAwB,CAAA,IAAK,QAAA;AAC7D,IAAA,IAAI,mBAAmB,QAAA,EAAU;AAC/B,MAAA,OAAO,MAAM;AAAA,MAAC,CAAA;AAAA,IAChB;AAEA,IAAA,MAAM,iBAAA,GAAoBC,iCAAA,CAA4B,IAAA,CAAK,MAAM,CAAA;AAEjE,IAAA,MAAM,UAAU,YAAY;AAC1B,MAAA,IAAI;AACF,QAAA,MAAM,CAAA,GAAI,MAAMC,6CAAA,CAAuB;AAAA,UACrC,MAAM,IAAA,CAAK,IAAA;AAAA,UACX,QAAA,EAAU;AAAA,SACX,CAAA;AACD,QAAA,IAAI,IAAI,CAAA,EAAG;AACT,UAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,CAAA,QAAA,EAAW,CAAC,CAAA,kBAAA,CAAoB,CAAA;AAAA,QACnD;AAAA,MACF,SAAS,KAAA,EAAO;AACd,QAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,CAAA,kCAAA,CAAA,EAAsC,KAAK,CAAA;AAAA,MAC9D;AAAA,IACF,CAAA;AAEA,IAAA,IAAI,KAAK,SAAA,EAAW;AAClB,MAAA,MAAM,eAAA,GAAkB,IAAI,eAAA,EAAgB;AAE5C,MAAA,IAAA,CAAK,UAAU,YAAA,CAAa;AAAA,QAC1B,EAAA,EAAI,wBAAA;AAAA,QACJ,SAAA,EAAW,EAAE,YAAA,EAAc,IAAA,CAAK,uBAAA,EAAwB;AAAA,QACxD,OAAA,EAAS,EAAE,YAAA,EAAc,IAAA,CAAK,0BAA0B,GAAA,EAAI;AAAA,QAC5D,EAAA,EAAI,OAAA;AAAA,QACJ,QAAQ,eAAA,CAAgB;AAAA,OACzB,CAAA;AAED,MAAA,OAAO,MAAM;AACX,QAAA,eAAA,CAAgB,KAAA,EAAM;AAAA,MACxB,CAAA;AAAA,IACF;AAEA,IAAA,MAAM,WAAA,GAAc,WAAA,CAAY,OAAA,EAAS,IAAA,CAAK,uBAAuB,CAAA;AACrE,IAAA,OAAO,MAAM;AACX,MAAA,aAAA,CAAc,WAAW,CAAA;AAAA,IAC3B,CAAA;AAAA,EACF;AACF;AAGA,SAAS,eAAA,GAAkB;AAEzB,EAAA,MAAM,wBAAwBC,2BAAA,CAAoB;AAAA,IAChD,IAAA,EAAM,kCAAA;AAAA,IACN,IAAA,EAAM,6EAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBC,2BAAA,CAAoB;AAAA,IACjD,IAAA,EAAM,qCAAA;AAAA,IACN,IAAA,EAAM,8FAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBA,2BAAA,CAAoB;AAAA,IACjD,IAAA,EAAM,qCAAA;AAAA,IACN,IAAA,EAAM,wFAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,2BAA2BA,2BAAA,CAAoB;AAAA,IACnD,IAAA,EAAM,wCAAA;AAAA,IACN,IAAA,EAAM;AAAA,GACP,CAAA;AAED,EAAA,MAAM,KAAA,GAAQC,WAAA,CAAQ,QAAA,CAAS,SAAS,CAAA;AACxC,EAAA,MAAM,oBAAoB,KAAA,CAAM,aAAA;AAAA,IAC9B,kCAAA;AAAA,IACA,EAAE,aAAa,8BAAA;AAA+B,GAChD;AAEA,EAAA,MAAM,qBAAqB,KAAA,CAAM,eAAA;AAAA,IAC/B,6BAAA;AAAA,IACA;AAAA,MACE,WAAA,EAAa,+CAAA;AAAA,MACb,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,MAAM,qBAAqB,KAAA,CAAM,eAAA;AAAA,IAC/B,6BAAA;AAAA,IACA;AAAA,MACE,WAAA,EAAa,yCAAA;AAAA,MACb,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,MAAM,uBAAuB,KAAA,CAAM,eAAA;AAAA,IACjC,gCAAA;AAAA,IACA;AAAA,MACE,WAAA,EACE,uGAAA;AAAA,MACF,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,SAAS,YAAA,CAAa,MAAwB,MAAA,EAAuB;AACnE,IAAA,MAAM,SAAA,GAAY,QAAQ,MAAA,EAAO;AACjC,IAAA,MAAM,eAAA,GAAkB,uBAAuB,UAAA,EAAW;AAC1D,IAAA,MAAM,kBAAA,GAAqB,uBAAuB,UAAA,EAAW;AAE7D,IAAA,MAAA,CAAO,KAAA,CAAM,CAAA,WAAA,EAAc,IAAA,CAAK,SAAS,CAAA,CAAE,CAAA;AAE3C,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,MAAM,UAAU,CAAC,IAAA,CAAK,aAAa,OAAA,EAAQ,CAAE,GAAG,SAAS,CAAA;AACzD,MAAA,wBAAA,CAAyB,QAAQ,OAAO,CAAA;AACxC,MAAA,oBAAA,CAAqB,OAAO,OAAO,CAAA;AAAA,IACrC;AAEA,IAAA,SAAS,OAAA,GAAU;AACjB,MAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,MAAA,CAAO,SAAS,CAAA;AACtC,MAAA,OAAO,KAAA,CAAM,CAAC,CAAA,GAAI,KAAA,CAAM,CAAC,CAAA,GAAI,GAAA;AAAA,IAC/B;AAEA,IAAA,SAAS,wBAAwB,MAAA,EAAgC;AAC/D,MAAA,kBAAA,CAAmB,EAAE,MAAA,EAAQ,MAAA,CAAO,EAAA,GAAK,IAAA,GAAO,UAAU,CAAA;AAC1D,MAAA,kBAAA,CAAmB,MAAA,CAAO,SAAQ,EAAG;AAAA,QACnC,MAAA,EAAQ,MAAA,CAAO,EAAA,GAAK,IAAA,GAAO;AAAA,OAC5B,CAAA;AAAA,IACH;AAEA,IAAA,SAAS,2BAAA,GAA8B;AACrC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,WAAA,EAAa,CAAA;AACvC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,WAAA,IAAe,CAAC,CAAA;AAEpD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAC5D,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAAA,IAClD;AAEA,IAAA,SAAS,wBAAA,GAA2B;AAClC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,QAAA,EAAU,CAAA;AACpC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,QAAA,IAAY,CAAC,CAAA;AAEjD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AACzD,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAAA,IAC/C;AAEA,IAAA,SAAS,yBAAA,GAA4B;AACnC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,SAAA,EAAW,CAAA;AACrC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,SAAA,IAAa,CAAC,CAAA;AAElD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAC1D,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAAA,IAChD;AAEA,IAAA,SAAS,WAAW,KAAA,EAAc;AAChC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,QAAA,IAAY,CAAC,CAAA;AACjD,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAC7C,MAAA,MAAA,CAAO,IAAA,CAAK,CAAA,cAAA,EAAiB,IAAA,CAAK,SAAS,WAAW,KAAK,CAAA;AAAA,IAC7D;AAEA,IAAA,OAAO;AAAA,MACL,uBAAA;AAAA,MACA,2BAAA;AAAA,MACA,wBAAA;AAAA,MACA,yBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF;AAEA,EAAA,OAAO,EAAE,YAAA,EAAa;AACxB;;;;"}
+{"version":3,"file":"DefaultCatalogProcessingEngine.cjs.js","sources":["../../src/processing/DefaultCatalogProcessingEngine.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  ANNOTATION_LOCATION,\n  Entity,\n  stringifyEntityRef,\n} from '@backstage/catalog-model';\nimport { assertError, serializeError, stringifyError } from '@backstage/errors';\nimport { Hash } from 'crypto';\nimport stableStringify from 'fast-json-stable-stringify';\nimport { Knex } from 'knex';\nimport { metrics, trace } from '@opentelemetry/api';\nimport { ProcessingDatabase, RefreshStateItem } from '../database/types';\nimport { createCounterMetric, createSummaryMetric } from '../util/metrics';\nimport { CatalogProcessingOrchestrator, EntityProcessingResult } from './types';\nimport { Stitcher, stitchingStrategyFromConfig } from '../stitching/types';\nimport { startTaskPipeline } from './TaskPipeline';\nimport { Config } from '@backstage/config';\nimport {\n  addEntityAttributes,\n  TRACER_ID,\n  withActiveSpan,\n} from '../util/opentelemetry';\nimport { deleteOrphanedEntities } from '../database/operations/util/deleteOrphanedEntities';\nimport { EventsService } from '@backstage/plugin-events-node';\nimport { CATALOG_ERRORS_TOPIC } from '../constants';\nimport { LoggerService, SchedulerService } from '@backstage/backend-plugin-api';\n\nconst CACHE_TTL = 5;\n\nconst tracer = trace.getTracer(TRACER_ID);\n\nexport type ProgressTracker = ReturnType<typeof progressTracker>;\n\nconst stableStringifyArray = (arr: any[]) => {\n  const sorted = arr.map(stableStringify).sort();\n  return `[${sorted.join(',')}]`;\n};\n\n// NOTE(freben): Perhaps surprisingly, this class does not implement the\n// CatalogProcessingEngine type. That type is externally visible and its name is\n// the way it is for historic reasons. This class has no particular reason to\n// implement that precise interface; nowadays there are several different\n// engines \"hiding\" behind the CatalogProcessingEngine interface, of which this\n// is just one.\nexport class DefaultCatalogProcessingEngine {\n  private readonly config: Config;\n  private readonly scheduler: SchedulerService;\n  private readonly logger: LoggerService;\n  private readonly knex: Knex;\n  private readonly processingDatabase: ProcessingDatabase;\n  private readonly orchestrator: CatalogProcessingOrchestrator;\n  private readonly stitcher: Stitcher;\n  private readonly createHash: () => Hash;\n  private readonly pollingIntervalMs: number;\n  private readonly orphanCleanupIntervalMs: number;\n  private readonly onProcessingError?: (event: {\n    unprocessedEntity: Entity;\n    errors: Error[];\n  }) => Promise<void> | void;\n  private readonly tracker: ProgressTracker;\n  private readonly events: EventsService;\n\n  private stopFunc?: () => void;\n\n  constructor(options: {\n    config: Config;\n    scheduler: SchedulerService;\n    logger: LoggerService;\n    knex: Knex;\n    processingDatabase: ProcessingDatabase;\n    orchestrator: CatalogProcessingOrchestrator;\n    stitcher: Stitcher;\n    createHash: () => Hash;\n    pollingIntervalMs?: number;\n    orphanCleanupIntervalMs?: number;\n    onProcessingError?: (event: {\n      unprocessedEntity: Entity;\n      errors: Error[];\n    }) => Promise<void> | void;\n    tracker?: ProgressTracker;\n    events: EventsService;\n  }) {\n    this.config = options.config;\n    this.scheduler = options.scheduler;\n    this.logger = options.logger;\n    this.knex = options.knex;\n    this.processingDatabase = options.processingDatabase;\n    this.orchestrator = options.orchestrator;\n    this.stitcher = options.stitcher;\n    this.createHash = options.createHash;\n    this.pollingIntervalMs = options.pollingIntervalMs ?? 1_000;\n    this.orphanCleanupIntervalMs = options.orphanCleanupIntervalMs ?? 30_000;\n    this.onProcessingError = options.onProcessingError;\n    this.tracker = options.tracker ?? progressTracker();\n    this.events = options.events;\n\n    this.stopFunc = undefined;\n  }\n\n  async start() {\n    if (this.stopFunc) {\n      throw new Error('Processing engine is already started');\n    }\n\n    const stopPipeline = this.startPipeline();\n    const stopCleanup = this.startOrphanCleanup();\n\n    this.stopFunc = () => {\n      stopPipeline();\n      stopCleanup();\n    };\n  }\n\n  async stop() {\n    if (this.stopFunc) {\n      this.stopFunc();\n      this.stopFunc = undefined;\n    }\n  }\n\n  private startPipeline(): () => void {\n    return startTaskPipeline<RefreshStateItem>({\n      lowWatermark: 5,\n      highWatermark: 10,\n      pollingIntervalMs: this.pollingIntervalMs,\n      loadTasks: async count => {\n        try {\n          const { items } =\n            await this.processingDatabase.getProcessableEntities(this.knex, {\n              processBatchSize: count,\n            });\n          return items;\n        } catch (error) {\n          this.logger.warn('Failed to load processing items', error);\n          return [];\n        }\n      },\n      processTask: async item => {\n        await withActiveSpan(tracer, 'ProcessingRun', async span => {\n          const track = this.tracker.processStart(item, this.logger);\n          addEntityAttributes(span, item.unprocessedEntity);\n\n          try {\n            const {\n              id,\n              state,\n              unprocessedEntity,\n              entityRef,\n              locationKey,\n              resultHash: previousResultHash,\n            } = item;\n            const result = await this.orchestrator.process({\n              entity: unprocessedEntity,\n              state,\n            });\n\n            track.markProcessorsCompleted(result);\n\n            if (result.ok) {\n              const { ttl: _, ...stateWithoutTtl } = state ?? {};\n              if (\n                stableStringify(stateWithoutTtl) !==\n                stableStringify(result.state)\n              ) {\n                await this.processingDatabase.transaction(async tx => {\n                  await this.processingDatabase.updateEntityCache(tx, {\n                    id,\n                    state: {\n                      ttl: CACHE_TTL,\n                      ...result.state,\n                    },\n                  });\n                });\n              }\n            } else {\n              const maybeTtl = state?.ttl;\n              const ttl = Number.isInteger(maybeTtl) ? (maybeTtl as number) : 0;\n              await this.processingDatabase.transaction(async tx => {\n                await this.processingDatabase.updateEntityCache(tx, {\n                  id,\n                  state: ttl > 0 ? { ...state, ttl: ttl - 1 } : {},\n                });\n              });\n            }\n\n            const location =\n              unprocessedEntity?.metadata?.annotations?.[ANNOTATION_LOCATION];\n            if (result.errors.length) {\n              this.events.publish({\n                topic: CATALOG_ERRORS_TOPIC,\n                eventPayload: {\n                  entity: entityRef,\n                  location,\n                  errors: result.errors,\n                },\n              });\n            }\n            const errorsString = JSON.stringify(\n              result.errors.map(e => serializeError(e)),\n            );\n\n            let hashBuilder = this.createHash().update(errorsString);\n\n            if (result.ok) {\n              const { entityRefs: parents } =\n                await this.processingDatabase.transaction(tx =>\n                  this.processingDatabase.listParents(tx, {\n                    entityRefs: [\n                      entityRef,\n                      ...result.deferredEntities.map(e =>\n                        stringifyEntityRef(e.entity),\n                      ),\n                    ],\n                  }),\n                );\n\n              hashBuilder = hashBuilder\n                .update(stableStringify({ ...result.completedEntity }))\n                .update(stableStringifyArray([...result.deferredEntities]))\n                .update(stableStringifyArray([...result.relations]))\n                .update(stableStringifyArray([...result.refreshKeys]))\n                .update(stableStringifyArray([...parents]));\n            }\n\n            const resultHash = hashBuilder.digest('hex');\n            if (resultHash === previousResultHash) {\n              // If nothing changed in our produced outputs, we cannot have any\n              // significant effect on our surroundings; therefore, we just abort\n              // without any updates / stitching.\n              track.markSuccessfulWithNoChanges();\n              return;\n            }\n\n            // If the result was marked as not OK, it signals that some part of the\n            // processing pipeline threw an exception. This can happen both as part of\n            // non-catastrophic things such as due to validation errors, as well as if\n            // something fatal happens inside the processing for other reasons. In any\n            // case, this means we can't trust that anything in the output is okay. So\n            // just store the errors and trigger a stich so that they become visible to\n            // the outside.\n            if (!result.ok) {\n              // notify the error listener if the entity can not be processed.\n              Promise.resolve(undefined)\n                .then(() =>\n                  this.onProcessingError?.({\n                    unprocessedEntity,\n                    errors: result.errors,\n                  }),\n                )\n                .catch(error => {\n                  this.logger.debug(\n                    `Processing error listener threw an exception, ${stringifyError(\n                      error,\n                    )}`,\n                  );\n                });\n\n              await this.processingDatabase.transaction(async tx => {\n                await this.processingDatabase.updateProcessedEntityErrors(tx, {\n                  id,\n                  errors: errorsString,\n                  resultHash,\n                });\n              });\n\n              await this.stitcher.stitch({\n                entityRefs: [stringifyEntityRef(unprocessedEntity)],\n              });\n\n              track.markSuccessfulWithErrors();\n              return;\n            }\n\n            result.completedEntity.metadata.uid = id;\n            let oldRelationSources: Map<string, string>;\n            await this.processingDatabase.transaction(async tx => {\n              const { previous } =\n                await this.processingDatabase.updateProcessedEntity(tx, {\n                  id,\n                  processedEntity: result.completedEntity,\n                  resultHash,\n                  errors: errorsString,\n                  relations: result.relations,\n                  deferredEntities: result.deferredEntities,\n                  locationKey,\n                  refreshKeys: result.refreshKeys,\n                });\n              oldRelationSources = new Map(\n                previous.relations.map(r => [\n                  `${r.source_entity_ref}:${r.type}->${r.target_entity_ref}`,\n                  r.source_entity_ref,\n                ]),\n              );\n            });\n\n            const newRelationSources = new Map<string, string>(\n              result.relations.map(relation => {\n                const sourceEntityRef = stringifyEntityRef(relation.source);\n                const targetEntityRef = stringifyEntityRef(relation.target);\n                return [\n                  `${sourceEntityRef}:${relation.type}->${targetEntityRef}`,\n                  sourceEntityRef,\n                ];\n              }),\n            );\n\n            const setOfThingsToStitch = new Set<string>([\n              stringifyEntityRef(result.completedEntity),\n            ]);\n            newRelationSources.forEach((sourceEntityRef, uniqueKey) => {\n              if (!oldRelationSources.has(uniqueKey)) {\n                setOfThingsToStitch.add(sourceEntityRef);\n              }\n            });\n            oldRelationSources!.forEach((sourceEntityRef, uniqueKey) => {\n              if (!newRelationSources.has(uniqueKey)) {\n                setOfThingsToStitch.add(sourceEntityRef);\n              }\n            });\n\n            await this.stitcher.stitch({\n              entityRefs: setOfThingsToStitch,\n            });\n\n            track.markSuccessfulWithChanges();\n          } catch (error) {\n            assertError(error);\n            track.markFailed(error);\n          }\n        });\n      },\n    });\n  }\n\n  private startOrphanCleanup(): () => void {\n    const orphanStrategy =\n      this.config.getOptionalString('catalog.orphanStrategy') ?? 'delete';\n    if (orphanStrategy !== 'delete') {\n      return () => {};\n    }\n\n    const stitchingStrategy = stitchingStrategyFromConfig(this.config);\n\n    const runOnce = async () => {\n      try {\n        const n = await deleteOrphanedEntities({\n          knex: this.knex,\n          strategy: stitchingStrategy,\n        });\n        if (n > 0) {\n          this.logger.info(`Deleted ${n} orphaned entities`);\n        }\n      } catch (error) {\n        this.logger.warn(`Failed to delete orphaned entities`, error);\n      }\n    };\n\n    const abortController = new AbortController();\n    this.scheduler.scheduleTask({\n      id: 'catalog_orphan_cleanup',\n      frequency: { milliseconds: this.orphanCleanupIntervalMs },\n      timeout: { milliseconds: this.orphanCleanupIntervalMs * 0.8 },\n      fn: runOnce,\n      signal: abortController.signal,\n    });\n\n    return () => {\n      abortController.abort();\n    };\n  }\n}\n\n// Helps wrap the timing and logging behaviors\nfunction progressTracker() {\n  // prom-client metrics are deprecated in favour of OpenTelemetry metrics.\n  const promProcessedEntities = createCounterMetric({\n    name: 'catalog_processed_entities_count',\n    help: 'Amount of entities processed, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingDuration = createSummaryMetric({\n    name: 'catalog_processing_duration_seconds',\n    help: 'Time spent executing the full processing flow, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessorsDuration = createSummaryMetric({\n    name: 'catalog_processors_duration_seconds',\n    help: 'Time spent executing catalog processors, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingQueueDelay = createSummaryMetric({\n    name: 'catalog_processing_queue_delay_seconds',\n    help: 'The amount of delay between being scheduled for processing, and the start of actually being processed, DEPRECATED, use OpenTelemetry metrics instead',\n  });\n\n  const meter = metrics.getMeter('default');\n  const processedEntities = meter.createCounter(\n    'catalog.processed.entities.count',\n    { description: 'Amount of entities processed' },\n  );\n\n  const processingDuration = meter.createHistogram(\n    'catalog.processing.duration',\n    {\n      description: 'Time spent executing the full processing flow',\n      unit: 'seconds',\n    },\n  );\n\n  const processorsDuration = meter.createHistogram(\n    'catalog.processors.duration',\n    {\n      description: 'Time spent executing catalog processors',\n      unit: 'seconds',\n    },\n  );\n\n  const processingQueueDelay = meter.createHistogram(\n    'catalog.processing.queue.delay',\n    {\n      description:\n        'The amount of delay between being scheduled for processing, and the start of actually being processed',\n      unit: 'seconds',\n    },\n  );\n\n  function processStart(item: RefreshStateItem, logger: LoggerService) {\n    const startTime = process.hrtime();\n    const endOverallTimer = promProcessingDuration.startTimer();\n    const endProcessorsTimer = promProcessorsDuration.startTimer();\n\n    logger.debug(`Processing ${item.entityRef}`);\n\n    if (item.nextUpdateAt) {\n      const seconds = -item.nextUpdateAt.diffNow().as('seconds');\n      promProcessingQueueDelay.observe(seconds);\n      processingQueueDelay.record(seconds);\n    }\n\n    function endTime() {\n      const delta = process.hrtime(startTime);\n      return delta[0] + delta[1] / 1e9;\n    }\n\n    function markProcessorsCompleted(result: EntityProcessingResult) {\n      endProcessorsTimer({ result: result.ok ? 'ok' : 'failed' });\n      processorsDuration.record(endTime(), {\n        result: result.ok ? 'ok' : 'failed',\n      });\n    }\n\n    function markSuccessfulWithNoChanges() {\n      endOverallTimer({ result: 'unchanged' });\n      promProcessedEntities.inc({ result: 'unchanged' }, 1);\n\n      processingDuration.record(endTime(), { result: 'unchanged' });\n      processedEntities.add(1, { result: 'unchanged' });\n    }\n\n    function markSuccessfulWithErrors() {\n      endOverallTimer({ result: 'errors' });\n      promProcessedEntities.inc({ result: 'errors' }, 1);\n\n      processingDuration.record(endTime(), { result: 'errors' });\n      processedEntities.add(1, { result: 'errors' });\n    }\n\n    function markSuccessfulWithChanges() {\n      endOverallTimer({ result: 'changed' });\n      promProcessedEntities.inc({ result: 'changed' }, 1);\n\n      processingDuration.record(endTime(), { result: 'changed' });\n      processedEntities.add(1, { result: 'changed' });\n    }\n\n    function markFailed(error: Error) {\n      promProcessedEntities.inc({ result: 'failed' }, 1);\n      processedEntities.add(1, { result: 'failed' });\n      logger.warn(`Processing of ${item.entityRef} failed`, error);\n    }\n\n    return {\n      markProcessorsCompleted,\n      markSuccessfulWithNoChanges,\n      markSuccessfulWithErrors,\n      markSuccessfulWithChanges,\n      markFailed,\n    };\n  }\n\n  return { processStart };\n}\n"],"names":["trace","TRACER_ID","stableStringify","startTaskPipeline","withActiveSpan","addEntityAttributes","ANNOTATION_LOCATION","CATALOG_ERRORS_TOPIC","serializeError","stringifyEntityRef","stringifyError","assertError","stitchingStrategyFromConfig","deleteOrphanedEntities","createCounterMetric","createSummaryMetric","metrics"],"mappings":";;;;;;;;;;;;;;;;;AA0CA,MAAM,SAAA,GAAY,CAAA;AAElB,MAAM,MAAA,GAASA,SAAA,CAAM,SAAA,CAAUC,uBAAS,CAAA;AAIxC,MAAM,oBAAA,GAAuB,CAAC,GAAA,KAAe;AAC3C,EAAA,MAAM,MAAA,GAAS,GAAA,CAAI,GAAA,CAAIC,gCAAe,EAAE,IAAA,EAAK;AAC7C,EAAA,OAAO,CAAA,CAAA,EAAI,MAAA,CAAO,IAAA,CAAK,GAAG,CAAC,CAAA,CAAA,CAAA;AAC7B,CAAA;AAQO,MAAM,8BAAA,CAA+B;AAAA,EACzB,MAAA;AAAA,EACA,SAAA;AAAA,EACA,MAAA;AAAA,EACA,IAAA;AAAA,EACA,kBAAA;AAAA,EACA,YAAA;AAAA,EACA,QAAA;AAAA,EACA,UAAA;AAAA,EACA,iBAAA;AAAA,EACA,uBAAA;AAAA,EACA,iBAAA;AAAA,EAIA,OAAA;AAAA,EACA,MAAA;AAAA,EAET,QAAA;AAAA,EAER,YAAY,OAAA,EAiBT;AACD,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,YAAY,OAAA,CAAQ,SAAA;AACzB,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,OAAO,OAAA,CAAQ,IAAA;AACpB,IAAA,IAAA,CAAK,qBAAqB,OAAA,CAAQ,kBAAA;AAClC,IAAA,IAAA,CAAK,eAAe,OAAA,CAAQ,YAAA;AAC5B,IAAA,IAAA,CAAK,WAAW,OAAA,CAAQ,QAAA;AACxB,IAAA,IAAA,CAAK,aAAa,OAAA,CAAQ,UAAA;AAC1B,IAAA,IAAA,CAAK,iBAAA,GAAoB,QAAQ,iBAAA,IAAqB,GAAA;AACtD,IAAA,IAAA,CAAK,uBAAA,GAA0B,QAAQ,uBAAA,IAA2B,GAAA;AAClE,IAAA,IAAA,CAAK,oBAAoB,OAAA,CAAQ,iBAAA;AACjC,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA,CAAQ,OAAA,IAAW,eAAA,EAAgB;AAClD,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AAEtB,IAAA,IAAA,CAAK,QAAA,GAAW,MAAA;AAAA,EAClB;AAAA,EAEA,MAAM,KAAA,GAAQ;AACZ,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,MAAM,IAAI,MAAM,sCAAsC,CAAA;AAAA,IACxD;AAEA,IAAA,MAAM,YAAA,GAAe,KAAK,aAAA,EAAc;AACxC,IAAA,MAAM,WAAA,GAAc,KAAK,kBAAA,EAAmB;AAE5C,IAAA,IAAA,CAAK,WAAW,MAAM;AACpB,MAAA,YAAA,EAAa;AACb,MAAA,WAAA,EAAY;AAAA,IACd,CAAA;AAAA,EACF;AAAA,EAEA,MAAM,IAAA,GAAO;AACX,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,IAAA,CAAK,QAAA,EAAS;AACd,MAAA,IAAA,CAAK,QAAA,GAAW,MAAA;AAAA,IAClB;AAAA,EACF;AAAA,EAEQ,aAAA,GAA4B;AAClC,IAAA,OAAOC,8BAAA,CAAoC;AAAA,MACzC,YAAA,EAAc,CAAA;AAAA,MACd,aAAA,EAAe,EAAA;AAAA,MACf,mBAAmB,IAAA,CAAK,iBAAA;AAAA,MACxB,SAAA,EAAW,OAAM,KAAA,KAAS;AACxB,QAAA,IAAI;AACF,UAAA,MAAM,EAAE,OAAM,GACZ,MAAM,KAAK,kBAAA,CAAmB,sBAAA,CAAuB,KAAK,IAAA,EAAM;AAAA,YAC9D,gBAAA,EAAkB;AAAA,WACnB,CAAA;AACH,UAAA,OAAO,KAAA;AAAA,QACT,SAAS,KAAA,EAAO;AACd,UAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,iCAAA,EAAmC,KAAK,CAAA;AACzD,UAAA,OAAO,EAAC;AAAA,QACV;AAAA,MACF,CAAA;AAAA,MACA,WAAA,EAAa,OAAM,IAAA,KAAQ;AACzB,QAAA,MAAMC,4BAAA,CAAe,MAAA,EAAQ,eAAA,EAAiB,OAAM,IAAA,KAAQ;AAC1D,UAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,YAAA,CAAa,IAAA,EAAM,KAAK,MAAM,CAAA;AACzD,UAAAC,iCAAA,CAAoB,IAAA,EAAM,KAAK,iBAAiB,CAAA;AAEhD,UAAA,IAAI;AACF,YAAA,MAAM;AAAA,cACJ,EAAA;AAAA,cACA,KAAA;AAAA,cACA,iBAAA;AAAA,cACA,SAAA;AAAA,cACA,WAAA;AAAA,cACA,UAAA,EAAY;AAAA,aACd,GAAI,IAAA;AACJ,YAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,YAAA,CAAa,OAAA,CAAQ;AAAA,cAC7C,MAAA,EAAQ,iBAAA;AAAA,cACR;AAAA,aACD,CAAA;AAED,YAAA,KAAA,CAAM,wBAAwB,MAAM,CAAA;AAEpC,YAAA,IAAI,OAAO,EAAA,EAAI;AACb,cAAA,MAAM,EAAE,GAAA,EAAK,CAAA,EAAG,GAAG,eAAA,EAAgB,GAAI,SAAS,EAAC;AACjD,cAAA,IACEH,iCAAgB,eAAe,CAAA,KAC/BA,gCAAA,CAAgB,MAAA,CAAO,KAAK,CAAA,EAC5B;AACA,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,kBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,iBAAA,CAAkB,EAAA,EAAI;AAAA,oBAClD,EAAA;AAAA,oBACA,KAAA,EAAO;AAAA,sBACL,GAAA,EAAK,SAAA;AAAA,sBACL,GAAG,MAAA,CAAO;AAAA;AACZ,mBACD,CAAA;AAAA,gBACH,CAAC,CAAA;AAAA,cACH;AAAA,YACF,CAAA,MAAO;AACL,cAAA,MAAM,WAAW,KAAA,EAAO,GAAA;AACxB,cAAA,MAAM,GAAA,GAAM,MAAA,CAAO,SAAA,CAAU,QAAQ,IAAK,QAAA,GAAsB,CAAA;AAChE,cAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,iBAAA,CAAkB,EAAA,EAAI;AAAA,kBAClD,EAAA;AAAA,kBACA,KAAA,EAAO,GAAA,GAAM,CAAA,GAAI,EAAE,GAAG,OAAO,GAAA,EAAK,GAAA,GAAM,CAAA,EAAE,GAAI;AAAC,iBAChD,CAAA;AAAA,cACH,CAAC,CAAA;AAAA,YACH;AAEA,YAAA,MAAM,QAAA,GACJ,iBAAA,EAAmB,QAAA,EAAU,WAAA,GAAcI,gCAAmB,CAAA;AAChE,YAAA,IAAI,MAAA,CAAO,OAAO,MAAA,EAAQ;AACxB,cAAA,IAAA,CAAK,OAAO,OAAA,CAAQ;AAAA,gBAClB,KAAA,EAAOC,8BAAA;AAAA,gBACP,YAAA,EAAc;AAAA,kBACZ,MAAA,EAAQ,SAAA;AAAA,kBACR,QAAA;AAAA,kBACA,QAAQ,MAAA,CAAO;AAAA;AACjB,eACD,CAAA;AAAA,YACH;AACA,YAAA,MAAM,eAAe,IAAA,CAAK,SAAA;AAAA,cACxB,OAAO,MAAA,CAAO,GAAA,CAAI,CAAA,CAAA,KAAKC,qBAAA,CAAe,CAAC,CAAC;AAAA,aAC1C;AAEA,YAAA,IAAI,WAAA,GAAc,IAAA,CAAK,UAAA,EAAW,CAAE,OAAO,YAAY,CAAA;AAEvD,YAAA,IAAI,OAAO,EAAA,EAAI;AACb,cAAA,MAAM,EAAE,UAAA,EAAY,OAAA,EAAQ,GAC1B,MAAM,KAAK,kBAAA,CAAmB,WAAA;AAAA,gBAAY,CAAA,EAAA,KACxC,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,EAAA,EAAI;AAAA,kBACtC,UAAA,EAAY;AAAA,oBACV,SAAA;AAAA,oBACA,GAAG,OAAO,gBAAA,CAAiB,GAAA;AAAA,sBAAI,CAAA,CAAA,KAC7BC,+BAAA,CAAmB,CAAA,CAAE,MAAM;AAAA;AAC7B;AACF,iBACD;AAAA,eACH;AAEF,cAAA,WAAA,GAAc,YACX,MAAA,CAAOP,gCAAA,CAAgB,EAAE,GAAG,MAAA,CAAO,iBAAiB,CAAC,EACrD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,gBAAgB,CAAC,CAAC,EACzD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,SAAS,CAAC,CAAC,EAClD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,WAAW,CAAC,CAAC,EACpD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,OAAO,CAAC,CAAC,CAAA;AAAA,YAC9C;AAEA,YAAA,MAAM,UAAA,GAAa,WAAA,CAAY,MAAA,CAAO,KAAK,CAAA;AAC3C,YAAA,IAAI,eAAe,kBAAA,EAAoB;AAIrC,cAAA,KAAA,CAAM,2BAAA,EAA4B;AAClC,cAAA;AAAA,YACF;AASA,YAAA,IAAI,CAAC,OAAO,EAAA,EAAI;AAEd,cAAA,OAAA,CAAQ,OAAA,CAAQ,MAAS,CAAA,CACtB,IAAA;AAAA,gBAAK,MACJ,KAAK,iBAAA,GAAoB;AAAA,kBACvB,iBAAA;AAAA,kBACA,QAAQ,MAAA,CAAO;AAAA,iBAChB;AAAA,eACH,CACC,MAAM,CAAA,KAAA,KAAS;AACd,gBAAA,IAAA,CAAK,MAAA,CAAO,KAAA;AAAA,kBACV,CAAA,8CAAA,EAAiDQ,qBAAA;AAAA,oBAC/C;AAAA,mBACD,CAAA;AAAA,iBACH;AAAA,cACF,CAAC,CAAA;AAEH,cAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,2BAAA,CAA4B,EAAA,EAAI;AAAA,kBAC5D,EAAA;AAAA,kBACA,MAAA,EAAQ,YAAA;AAAA,kBACR;AAAA,iBACD,CAAA;AAAA,cACH,CAAC,CAAA;AAED,cAAA,MAAM,IAAA,CAAK,SAAS,MAAA,CAAO;AAAA,gBACzB,UAAA,EAAY,CAACD,+BAAA,CAAmB,iBAAiB,CAAC;AAAA,eACnD,CAAA;AAED,cAAA,KAAA,CAAM,wBAAA,EAAyB;AAC/B,cAAA;AAAA,YACF;AAEA,YAAA,MAAA,CAAO,eAAA,CAAgB,SAAS,GAAA,GAAM,EAAA;AACtC,YAAA,IAAI,kBAAA;AACJ,YAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,cAAA,MAAM,EAAE,QAAA,EAAS,GACf,MAAM,IAAA,CAAK,kBAAA,CAAmB,sBAAsB,EAAA,EAAI;AAAA,gBACtD,EAAA;AAAA,gBACA,iBAAiB,MAAA,CAAO,eAAA;AAAA,gBACxB,UAAA;AAAA,gBACA,MAAA,EAAQ,YAAA;AAAA,gBACR,WAAW,MAAA,CAAO,SAAA;AAAA,gBAClB,kBAAkB,MAAA,CAAO,gBAAA;AAAA,gBACzB,WAAA;AAAA,gBACA,aAAa,MAAA,CAAO;AAAA,eACrB,CAAA;AACH,cAAA,kBAAA,GAAqB,IAAI,GAAA;AAAA,gBACvB,QAAA,CAAS,SAAA,CAAU,GAAA,CAAI,CAAA,CAAA,KAAK;AAAA,kBAC1B,CAAA,EAAG,EAAE,iBAAiB,CAAA,CAAA,EAAI,EAAE,IAAI,CAAA,EAAA,EAAK,EAAE,iBAAiB,CAAA,CAAA;AAAA,kBACxD,CAAA,CAAE;AAAA,iBACH;AAAA,eACH;AAAA,YACF,CAAC,CAAA;AAED,YAAA,MAAM,qBAAqB,IAAI,GAAA;AAAA,cAC7B,MAAA,CAAO,SAAA,CAAU,GAAA,CAAI,CAAA,QAAA,KAAY;AAC/B,gBAAA,MAAM,eAAA,GAAkBA,+BAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAC1D,gBAAA,MAAM,eAAA,GAAkBA,+BAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAC1D,gBAAA,OAAO;AAAA,kBACL,GAAG,eAAe,CAAA,CAAA,EAAI,QAAA,CAAS,IAAI,KAAK,eAAe,CAAA,CAAA;AAAA,kBACvD;AAAA,iBACF;AAAA,cACF,CAAC;AAAA,aACH;AAEA,YAAA,MAAM,mBAAA,uBAA0B,GAAA,CAAY;AAAA,cAC1CA,+BAAA,CAAmB,OAAO,eAAe;AAAA,aAC1C,CAAA;AACD,YAAA,kBAAA,CAAmB,OAAA,CAAQ,CAAC,eAAA,EAAiB,SAAA,KAAc;AACzD,cAAA,IAAI,CAAC,kBAAA,CAAmB,GAAA,CAAI,SAAS,CAAA,EAAG;AACtC,gBAAA,mBAAA,CAAoB,IAAI,eAAe,CAAA;AAAA,cACzC;AAAA,YACF,CAAC,CAAA;AACD,YAAA,kBAAA,CAAoB,OAAA,CAAQ,CAAC,eAAA,EAAiB,SAAA,KAAc;AAC1D,cAAA,IAAI,CAAC,kBAAA,CAAmB,GAAA,CAAI,SAAS,CAAA,EAAG;AACtC,gBAAA,mBAAA,CAAoB,IAAI,eAAe,CAAA;AAAA,cACzC;AAAA,YACF,CAAC,CAAA;AAED,YAAA,MAAM,IAAA,CAAK,SAAS,MAAA,CAAO;AAAA,cACzB,UAAA,EAAY;AAAA,aACb,CAAA;AAED,YAAA,KAAA,CAAM,yBAAA,EAA0B;AAAA,UAClC,SAAS,KAAA,EAAO;AACd,YAAAE,kBAAA,CAAY,KAAK,CAAA;AACjB,YAAA,KAAA,CAAM,WAAW,KAAK,CAAA;AAAA,UACxB;AAAA,QACF,CAAC,CAAA;AAAA,MACH;AAAA,KACD,CAAA;AAAA,EACH;AAAA,EAEQ,kBAAA,GAAiC;AACvC,IAAA,MAAM,cAAA,GACJ,IAAA,CAAK,MAAA,CAAO,iBAAA,CAAkB,wBAAwB,CAAA,IAAK,QAAA;AAC7D,IAAA,IAAI,mBAAmB,QAAA,EAAU;AAC/B,MAAA,OAAO,MAAM;AAAA,MAAC,CAAA;AAAA,IAChB;AAEA,IAAA,MAAM,iBAAA,GAAoBC,iCAAA,CAA4B,IAAA,CAAK,MAAM,CAAA;AAEjE,IAAA,MAAM,UAAU,YAAY;AAC1B,MAAA,IAAI;AACF,QAAA,MAAM,CAAA,GAAI,MAAMC,6CAAA,CAAuB;AAAA,UACrC,MAAM,IAAA,CAAK,IAAA;AAAA,UACX,QAAA,EAAU;AAAA,SACX,CAAA;AACD,QAAA,IAAI,IAAI,CAAA,EAAG;AACT,UAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,CAAA,QAAA,EAAW,CAAC,CAAA,kBAAA,CAAoB,CAAA;AAAA,QACnD;AAAA,MACF,SAAS,KAAA,EAAO;AACd,QAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,CAAA,kCAAA,CAAA,EAAsC,KAAK,CAAA;AAAA,MAC9D;AAAA,IACF,CAAA;AAEA,IAAA,MAAM,eAAA,GAAkB,IAAI,eAAA,EAAgB;AAC5C,IAAA,IAAA,CAAK,UAAU,YAAA,CAAa;AAAA,MAC1B,EAAA,EAAI,wBAAA;AAAA,MACJ,SAAA,EAAW,EAAE,YAAA,EAAc,IAAA,CAAK,uBAAA,EAAwB;AAAA,MACxD,OAAA,EAAS,EAAE,YAAA,EAAc,IAAA,CAAK,0BAA0B,GAAA,EAAI;AAAA,MAC5D,EAAA,EAAI,OAAA;AAAA,MACJ,QAAQ,eAAA,CAAgB;AAAA,KACzB,CAAA;AAED,IAAA,OAAO,MAAM;AACX,MAAA,eAAA,CAAgB,KAAA,EAAM;AAAA,IACxB,CAAA;AAAA,EACF;AACF;AAGA,SAAS,eAAA,GAAkB;AAEzB,EAAA,MAAM,wBAAwBC,2BAAA,CAAoB;AAAA,IAChD,IAAA,EAAM,kCAAA;AAAA,IACN,IAAA,EAAM,6EAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBC,2BAAA,CAAoB;AAAA,IACjD,IAAA,EAAM,qCAAA;AAAA,IACN,IAAA,EAAM,8FAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBA,2BAAA,CAAoB;AAAA,IACjD,IAAA,EAAM,qCAAA;AAAA,IACN,IAAA,EAAM,wFAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,2BAA2BA,2BAAA,CAAoB;AAAA,IACnD,IAAA,EAAM,wCAAA;AAAA,IACN,IAAA,EAAM;AAAA,GACP,CAAA;AAED,EAAA,MAAM,KAAA,GAAQC,WAAA,CAAQ,QAAA,CAAS,SAAS,CAAA;AACxC,EAAA,MAAM,oBAAoB,KAAA,CAAM,aAAA;AAAA,IAC9B,kCAAA;AAAA,IACA,EAAE,aAAa,8BAAA;AAA+B,GAChD;AAEA,EAAA,MAAM,qBAAqB,KAAA,CAAM,eAAA;AAAA,IAC/B,6BAAA;AAAA,IACA;AAAA,MACE,WAAA,EAAa,+CAAA;AAAA,MACb,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,MAAM,qBAAqB,KAAA,CAAM,eAAA;AAAA,IAC/B,6BAAA;AAAA,IACA;AAAA,MACE,WAAA,EAAa,yCAAA;AAAA,MACb,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,MAAM,uBAAuB,KAAA,CAAM,eAAA;AAAA,IACjC,gCAAA;AAAA,IACA;AAAA,MACE,WAAA,EACE,uGAAA;AAAA,MACF,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,SAAS,YAAA,CAAa,MAAwB,MAAA,EAAuB;AACnE,IAAA,MAAM,SAAA,GAAY,QAAQ,MAAA,EAAO;AACjC,IAAA,MAAM,eAAA,GAAkB,uBAAuB,UAAA,EAAW;AAC1D,IAAA,MAAM,kBAAA,GAAqB,uBAAuB,UAAA,EAAW;AAE7D,IAAA,MAAA,CAAO,KAAA,CAAM,CAAA,WAAA,EAAc,IAAA,CAAK,SAAS,CAAA,CAAE,CAAA;AAE3C,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,MAAM,UAAU,CAAC,IAAA,CAAK,aAAa,OAAA,EAAQ,CAAE,GAAG,SAAS,CAAA;AACzD,MAAA,wBAAA,CAAyB,QAAQ,OAAO,CAAA;AACxC,MAAA,oBAAA,CAAqB,OAAO,OAAO,CAAA;AAAA,IACrC;AAEA,IAAA,SAAS,OAAA,GAAU;AACjB,MAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,MAAA,CAAO,SAAS,CAAA;AACtC,MAAA,OAAO,KAAA,CAAM,CAAC,CAAA,GAAI,KAAA,CAAM,CAAC,CAAA,GAAI,GAAA;AAAA,IAC/B;AAEA,IAAA,SAAS,wBAAwB,MAAA,EAAgC;AAC/D,MAAA,kBAAA,CAAmB,EAAE,MAAA,EAAQ,MAAA,CAAO,EAAA,GAAK,IAAA,GAAO,UAAU,CAAA;AAC1D,MAAA,kBAAA,CAAmB,MAAA,CAAO,SAAQ,EAAG;AAAA,QACnC,MAAA,EAAQ,MAAA,CAAO,EAAA,GAAK,IAAA,GAAO;AAAA,OAC5B,CAAA;AAAA,IACH;AAEA,IAAA,SAAS,2BAAA,GAA8B;AACrC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,WAAA,EAAa,CAAA;AACvC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,WAAA,IAAe,CAAC,CAAA;AAEpD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAC5D,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAAA,IAClD;AAEA,IAAA,SAAS,wBAAA,GAA2B;AAClC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,QAAA,EAAU,CAAA;AACpC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,QAAA,IAAY,CAAC,CAAA;AAEjD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AACzD,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAAA,IAC/C;AAEA,IAAA,SAAS,yBAAA,GAA4B;AACnC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,SAAA,EAAW,CAAA;AACrC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,SAAA,IAAa,CAAC,CAAA;AAElD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAC1D,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAAA,IAChD;AAEA,IAAA,SAAS,WAAW,KAAA,EAAc;AAChC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,QAAA,IAAY,CAAC,CAAA;AACjD,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAC7C,MAAA,MAAA,CAAO,IAAA,CAAK,CAAA,cAAA,EAAiB,IAAA,CAAK,SAAS,WAAW,KAAK,CAAA;AAAA,IAC7D;AAEA,IAAA,OAAO;AAAA,MACL,uBAAA;AAAA,MACA,2BAAA;AAAA,MACA,wBAAA;AAAA,MACA,yBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF;AAEA,EAAA,OAAO,EAAE,YAAA,EAAa;AACxB;;;;"}

package/dist/processing/DefaultCatalogProcessingOrchestrator.cjs.js

@@ -190,9 +190,6 @@ class DefaultCatalogProcessingOrchestrator {
             );
             if (thisValid) {
               valid = true;
-              if (this.options.legacySingleProcessorValidation) {
-                break;
-              }
             }
           } catch (e) {
             throw new errors.InputError(

package/dist/processing/DefaultCatalogProcessingOrchestrator.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultCatalogProcessingOrchestrator.cjs.js","sources":["../../src/processing/DefaultCatalogProcessingOrchestrator.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Span, trace } from '@opentelemetry/api';\nimport {\n  Entity,\n  EntityPolicy,\n  LocationEntity,\n  parseLocationRef,\n  stringifyEntityRef,\n  stringifyLocationRef,\n} from '@backstage/catalog-model';\nimport {\n  assertError,\n  ConflictError,\n  InputError,\n  NotAllowedError,\n} from '@backstage/errors';\nimport { JsonValue } from '@backstage/types';\nimport { ScmIntegrationRegistry } from '@backstage/integration';\nimport path from 'path';\nimport { LocationSpec } from '@backstage/plugin-catalog-common';\nimport {\n  CatalogProcessor,\n  CatalogProcessorParser,\n  processingResult,\n} from '@backstage/plugin-catalog-node';\nimport {\n  CatalogProcessingOrchestrator,\n  EntityProcessingRequest,\n  EntityProcessingResult,\n} from './types';\nimport { ProcessorOutputCollector } from './ProcessorOutputCollector';\nimport {\n  getEntityLocationRef,\n  getEntityOriginLocationRef,\n  isLocationEntity,\n  isObject,\n  toAbsoluteUrl,\n  validateEntity,\n  validateEntityEnvelope,\n} from './util';\nimport { CatalogRulesEnforcer } from '../ingestion/CatalogRules';\nimport { ProcessorCacheManager } from './ProcessorCacheManager';\nimport {\n  addEntityAttributes,\n  TRACER_ID,\n  withActiveSpan,\n} from '../util/opentelemetry';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\nconst tracer = trace.getTracer(TRACER_ID);\n\ntype Context = {\n  entityRef: string;\n  location: LocationSpec;\n  originLocation: LocationSpec;\n  collector: ProcessorOutputCollector;\n  cache: ProcessorCacheManager;\n};\n\nfunction addProcessorAttributes(\n  span: Span,\n  stage: string,\n  processor: CatalogProcessor,\n) {\n  span.setAttribute('backstage.catalog.processor.stage', stage);\n  span.setAttribute(\n    'backstage.catalog.processor.name',\n    processor.getProcessorName(),\n  );\n}\n\nexport class DefaultCatalogProcessingOrchestrator\n  implements CatalogProcessingOrchestrator\n{\n  constructor(\n    private readonly options: {\n      processors: CatalogProcessor[];\n      integrations: ScmIntegrationRegistry;\n      logger: LoggerService;\n      parser: CatalogProcessorParser;\n      policy: EntityPolicy;\n      rulesEnforcer: CatalogRulesEnforcer;\n      legacySingleProcessorValidation: boolean;\n    },\n  ) {}\n\n  async process(\n    request: EntityProcessingRequest,\n  ): Promise<EntityProcessingResult> {\n    return this.processSingleEntity(request.entity, request.state);\n  }\n\n  private async processSingleEntity(\n    unprocessedEntity: Entity,\n    state: JsonValue | undefined,\n  ): Promise<EntityProcessingResult> {\n    const collector = new ProcessorOutputCollector(\n      this.options.logger,\n      unprocessedEntity,\n    );\n\n    // Cache that is scoped to the entity and processor\n    const cache = new ProcessorCacheManager(\n      isObject(state) && isObject(state.cache) ? state.cache : {},\n    );\n\n    try {\n      // This will be checked and mutated step by step below\n      let entity: Entity = unprocessedEntity;\n\n      // NOTE: At this early point, we can only rely on the envelope having to\n      // be valid; full entity + kind validation happens after the (potentially\n      // mutative) pre-steps. This means that the code below can't make a lot\n      // of assumptions about the data despite it using the Entity type.\n      try {\n        validateEntityEnvelope(entity);\n      } catch (e) {\n        throw new InputError(\n          `Entity envelope failed validation before processing`,\n          e,\n        );\n      }\n\n      // TODO: which one do we actually use for the location?\n      // source-location? - maybe probably doesn't exist yet?\n      const context: Context = {\n        entityRef: stringifyEntityRef(entity),\n        location: parseLocationRef(getEntityLocationRef(entity)),\n        originLocation: parseLocationRef(getEntityOriginLocationRef(entity)),\n        cache,\n        collector,\n      };\n\n      // Run the steps\n      entity = await this.runPreProcessStep(entity, context);\n      entity = await this.runPolicyStep(entity);\n      await this.runValidateStep(entity, context);\n      if (isLocationEntity(entity)) {\n        await this.runSpecialLocationStep(entity, context);\n      }\n      entity = await this.runPostProcessStep(entity, context);\n\n      // Check that any emitted entities are permitted to originate from that\n      // particular location according to the catalog rules\n      const collectorResults = context.collector.results();\n      for (const deferredEntity of collectorResults.deferredEntities) {\n        if (\n          !this.options.rulesEnforcer.isAllowed(\n            deferredEntity.entity,\n            context.originLocation,\n          )\n        ) {\n          throw new NotAllowedError(\n            `Entity ${stringifyEntityRef(\n              deferredEntity.entity,\n            )} at ${stringifyLocationRef(\n              context.location,\n            )}, originated at ${stringifyLocationRef(\n              context.originLocation,\n            )}, is not of an allowed kind for that location`,\n          );\n        }\n      }\n\n      return {\n        ...collectorResults,\n        completedEntity: entity,\n        state: { cache: cache.collect() },\n        ok: collectorResults.errors.length === 0,\n      };\n    } catch (error) {\n      assertError(error);\n      return {\n        ok: false,\n        errors: collector.results().errors.concat(error),\n      };\n    }\n  }\n\n  // Pre-process phase, used to populate entities with data that is required\n  // during the main processing step\n  private async runPreProcessStep(\n    entity: Entity,\n    context: Context,\n  ): Promise<Entity> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute('backstage.catalog.processor.stage', 'preProcess');\n      let res = entity;\n\n      for (const processor of this.options.processors) {\n        if (processor.preProcessEntity) {\n          let innerRes = res;\n          res = await withActiveSpan(tracer, 'ProcessingStep', async span => {\n            addEntityAttributes(span, entity);\n            addProcessorAttributes(span, 'preProcessEntity', processor);\n            try {\n              innerRes = await processor.preProcessEntity!(\n                innerRes,\n                context.location,\n                context.collector.forProcessor(processor),\n                context.originLocation,\n                context.cache.forProcessor(processor),\n              );\n            } catch (e) {\n              throw new InputError(\n                `Processor ${processor.constructor.name} threw an error while preprocessing`,\n                e,\n              );\n            }\n            return innerRes;\n          });\n        }\n      }\n\n      return res;\n    });\n  }\n\n  /**\n   * Enforce entity policies making sure that entities conform to a general schema\n   */\n  private async runPolicyStep(entity: Entity): Promise<Entity> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute(\n        'backstage.catalog.processor.stage',\n        'enforcePolicy',\n      );\n      let policyEnforcedEntity: Entity | undefined;\n\n      try {\n        policyEnforcedEntity = await this.options.policy.enforce(entity);\n      } catch (e) {\n        throw new InputError(\n          `Policy check failed for ${stringifyEntityRef(entity)}`,\n          e,\n        );\n      }\n\n      if (!policyEnforcedEntity) {\n        throw new Error(\n          `Policy unexpectedly returned no data for ${stringifyEntityRef(\n            entity,\n          )}`,\n        );\n      }\n\n      return policyEnforcedEntity;\n    });\n  }\n\n  /**\n   * Validate the given entity\n   */\n  private async runValidateStep(\n    entity: Entity,\n    context: Context,\n  ): Promise<void> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute('backstage.catalog.processor.stage', 'validate');\n      // Double check that none of the previous steps tried to change something\n      // related to the entity ref, which would break downstream\n      if (stringifyEntityRef(entity) !== context.entityRef) {\n        throw new ConflictError(\n          'Fatal: The entity kind, namespace, or name changed during processing',\n        );\n      }\n\n      // Validate that the end result is a valid Entity at all\n      try {\n        validateEntity(entity);\n      } catch (e) {\n        throw new ConflictError(\n          `Entity envelope for ${context.entityRef} failed validation after preprocessing`,\n          e,\n        );\n      }\n\n      let valid = false;\n\n      for (const processor of this.options.processors) {\n        if (processor.validateEntityKind) {\n          try {\n            const thisValid = await withActiveSpan(\n              tracer,\n              'ProcessingStep',\n              async span => {\n                addEntityAttributes(span, entity);\n                addProcessorAttributes(span, 'validateEntityKind', processor);\n                return await processor.validateEntityKind!(entity);\n              },\n            );\n            if (thisValid) {\n              valid = true;\n              if (this.options.legacySingleProcessorValidation) {\n                break;\n              }\n            }\n          } catch (e) {\n            throw new InputError(\n              `Processor ${processor.constructor.name} threw an error while validating the entity ${context.entityRef}`,\n              e,\n            );\n          }\n        }\n      }\n\n      if (!valid) {\n        throw new InputError(\n          `No processor recognized the entity ${context.entityRef} as valid, possibly caused by a foreign kind or apiVersion`,\n        );\n      }\n    });\n  }\n\n  /**\n   * Backwards compatible processing of location entities\n   */\n  private async runSpecialLocationStep(\n    entity: LocationEntity,\n    context: Context,\n  ): Promise<void> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute(\n        'backstage.catalog.processor.stage',\n        'readLocation',\n      );\n      const { type = context.location.type, presence = 'required' } =\n        entity.spec;\n      const targets = new Array<string>();\n      if (entity.spec.target) {\n        targets.push(entity.spec.target);\n      }\n      if (entity.spec.targets) {\n        targets.push(...entity.spec.targets);\n      }\n\n      for (const maybeRelativeTarget of targets) {\n        if (type === 'file' && maybeRelativeTarget.endsWith(path.sep)) {\n          context.collector.generic()(\n            processingResult.inputError(\n              context.location,\n              `LocationEntityProcessor cannot handle ${type} type location with target ${context.location.target} that ends with a path separator`,\n            ),\n          );\n          continue;\n        }\n        const target = toAbsoluteUrl(\n          this.options.integrations,\n          context.location,\n          type,\n          maybeRelativeTarget,\n        );\n\n        let didRead = false;\n        for (const processor of this.options.processors) {\n          if (processor.readLocation) {\n            try {\n              const read = await withActiveSpan(\n                tracer,\n                'ProcessingStep',\n                async span => {\n                  addEntityAttributes(span, entity);\n                  addProcessorAttributes(span, 'readLocation', processor);\n                  return await processor.readLocation!(\n                    {\n                      type,\n                      target,\n                      presence,\n                    },\n                    presence === 'optional',\n                    context.collector.forProcessor(processor),\n                    this.options.parser,\n                    context.cache.forProcessor(processor, target),\n                  );\n                },\n              );\n              if (read) {\n                didRead = true;\n                break;\n              }\n            } catch (e) {\n              throw new InputError(\n                `Processor ${processor.constructor.name} threw an error while reading ${type}:${target}`,\n                e,\n              );\n            }\n          }\n        }\n        if (!didRead) {\n          throw new InputError(\n            `No processor was able to handle reading of ${type}:${target}`,\n          );\n        }\n      }\n    });\n  }\n\n  /**\n   * Main processing step of the entity\n   */\n  private async runPostProcessStep(\n    entity: Entity,\n    context: Context,\n  ): Promise<Entity> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute(\n        'backstage.catalog.processor.stage',\n        'postProcessEntity',\n      );\n      let res = entity;\n\n      for (const processor of this.options.processors) {\n        if (processor.postProcessEntity) {\n          let innerRes = res;\n          res = await withActiveSpan(tracer, 'ProcessingStep', async span => {\n            addEntityAttributes(span, entity);\n            addProcessorAttributes(span, 'postProcessEntity', processor);\n            try {\n              innerRes = await processor.postProcessEntity!(\n                innerRes,\n                context.location,\n                context.collector.forProcessor(processor),\n                context.cache.forProcessor(processor),\n              );\n            } catch (e) {\n              throw new InputError(\n                `Processor ${processor.constructor.name} threw an error while postprocessing`,\n                e,\n              );\n            }\n            return innerRes;\n          });\n        }\n      }\n\n      return res;\n    });\n  }\n}\n"],"names":["trace","TRACER_ID","ProcessorOutputCollector","ProcessorCacheManager","isObject","validateEntityEnvelope","InputError","stringifyEntityRef","parseLocationRef","getEntityLocationRef","getEntityOriginLocationRef","isLocationEntity","NotAllowedError","stringifyLocationRef","assertError","withActiveSpan","addEntityAttributes","ConflictError","validateEntity","path","processingResult","toAbsoluteUrl"],"mappings":";;;;;;;;;;;;;;;;AAgEA,MAAM,MAAA,GAASA,SAAA,CAAM,SAAA,CAAUC,uBAAS,CAAA;AAUxC,SAAS,sBAAA,CACP,IAAA,EACA,KAAA,EACA,SAAA,EACA;AACA,EAAA,IAAA,CAAK,YAAA,CAAa,qCAAqC,KAAK,CAAA;AAC5D,EAAA,IAAA,CAAK,YAAA;AAAA,IACH,kCAAA;AAAA,IACA,UAAU,gBAAA;AAAiB,GAC7B;AACF;AAEO,MAAM,oCAAA,CAEb;AAAA,EACE,YACmB,OAAA,EASjB;AATiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA,EAShB;AAAA,EAEH,MAAM,QACJ,OAAA,EACiC;AACjC,IAAA,OAAO,IAAA,CAAK,mBAAA,CAAoB,OAAA,CAAQ,MAAA,EAAQ,QAAQ,KAAK,CAAA;AAAA,EAC/D;AAAA,EAEA,MAAc,mBAAA,CACZ,iBAAA,EACA,KAAA,EACiC;AACjC,IAAA,MAAM,YAAY,IAAIC,iDAAA;AAAA,MACpB,KAAK,OAAA,CAAQ,MAAA;AAAA,MACb;AAAA,KACF;AAGA,IAAA,MAAM,QAAQ,IAAIC,2CAAA;AAAA,MAChBC,aAAA,CAAS,KAAK,CAAA,IAAKA,aAAA,CAAS,MAAM,KAAK,CAAA,GAAI,KAAA,CAAM,KAAA,GAAQ;AAAC,KAC5D;AAEA,IAAA,IAAI;AAEF,MAAA,IAAI,MAAA,GAAiB,iBAAA;AAMrB,MAAA,IAAI;AACF,QAAAC,2BAAA,CAAuB,MAAM,CAAA;AAAA,MAC/B,SAAS,CAAA,EAAG;AACV,QAAA,MAAM,IAAIC,iBAAA;AAAA,UACR,CAAA,mDAAA,CAAA;AAAA,UACA;AAAA,SACF;AAAA,MACF;AAIA,MAAA,MAAM,OAAA,GAAmB;AAAA,QACvB,SAAA,EAAWC,gCAAmB,MAAM,CAAA;AAAA,QACpC,QAAA,EAAUC,6BAAA,CAAiBC,yBAAA,CAAqB,MAAM,CAAC,CAAA;AAAA,QACvD,cAAA,EAAgBD,6BAAA,CAAiBE,+BAAA,CAA2B,MAAM,CAAC,CAAA;AAAA,QACnE,KAAA;AAAA,QACA;AAAA,OACF;AAGA,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,iBAAA,CAAkB,MAAA,EAAQ,OAAO,CAAA;AACrD,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,aAAA,CAAc,MAAM,CAAA;AACxC,MAAA,MAAM,IAAA,CAAK,eAAA,CAAgB,MAAA,EAAQ,OAAO,CAAA;AAC1C,MAAA,IAAIC,qBAAA,CAAiB,MAAM,CAAA,EAAG;AAC5B,QAAA,MAAM,IAAA,CAAK,sBAAA,CAAuB,MAAA,EAAQ,OAAO,CAAA;AAAA,MACnD;AACA,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,kBAAA,CAAmB,MAAA,EAAQ,OAAO,CAAA;AAItD,MAAA,MAAM,gBAAA,GAAmB,OAAA,CAAQ,SAAA,CAAU,OAAA,EAAQ;AACnD,MAAA,KAAA,MAAW,cAAA,IAAkB,iBAAiB,gBAAA,EAAkB;AAC9D,QAAA,IACE,CAAC,IAAA,CAAK,OAAA,CAAQ,aAAA,CAAc,SAAA;AAAA,UAC1B,cAAA,CAAe,MAAA;AAAA,UACf,OAAA,CAAQ;AAAA,SACV,EACA;AACA,UAAA,MAAM,IAAIC,sBAAA;AAAA,YACR,CAAA,OAAA,EAAUL,+BAAA;AAAA,cACR,cAAA,CAAe;AAAA,aAChB,CAAA,IAAA,EAAOM,iCAAA;AAAA,cACN,OAAA,CAAQ;AAAA,aACT,CAAA,gBAAA,EAAmBA,iCAAA;AAAA,cAClB,OAAA,CAAQ;AAAA,aACT,CAAA,6CAAA;AAAA,WACH;AAAA,QACF;AAAA,MACF;AAEA,MAAA,OAAO;AAAA,QACL,GAAG,gBAAA;AAAA,QACH,eAAA,EAAiB,MAAA;AAAA,QACjB,KAAA,EAAO,EAAE,KAAA,EAAO,KAAA,CAAM,SAAQ,EAAE;AAAA,QAChC,EAAA,EAAI,gBAAA,CAAiB,MAAA,CAAO,MAAA,KAAW;AAAA,OACzC;AAAA,IACF,SAAS,KAAA,EAAO;AACd,MAAAC,kBAAA,CAAY,KAAK,CAAA;AACjB,MAAA,OAAO;AAAA,QACL,EAAA,EAAI,KAAA;AAAA,QACJ,QAAQ,SAAA,CAAU,OAAA,EAAQ,CAAE,MAAA,CAAO,OAAO,KAAK;AAAA,OACjD;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA,EAIA,MAAc,iBAAA,CACZ,MAAA,EACA,OAAA,EACiB;AACjB,IAAA,OAAO,MAAMC,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA,CAAa,qCAAqC,YAAY,CAAA;AACxE,MAAA,IAAI,GAAA,GAAM,MAAA;AAEV,MAAA,KAAA,MAAW,SAAA,IAAa,IAAA,CAAK,OAAA,CAAQ,UAAA,EAAY;AAC/C,QAAA,IAAI,UAAU,gBAAA,EAAkB;AAC9B,UAAA,IAAI,QAAA,GAAW,GAAA;AACf,UAAA,GAAA,GAAM,MAAMD,4BAAA,CAAe,MAAA,EAAQ,gBAAA,EAAkB,OAAM,IAAA,KAAQ;AACjE,YAAAC,iCAAA,CAAoB,MAAM,MAAM,CAAA;AAChC,YAAA,sBAAA,CAAuB,IAAA,EAAM,oBAAoB,SAAS,CAAA;AAC1D,YAAA,IAAI;AACF,cAAA,QAAA,GAAW,MAAM,SAAA,CAAU,gBAAA;AAAA,gBACzB,QAAA;AAAA,gBACA,OAAA,CAAQ,QAAA;AAAA,gBACR,OAAA,CAAQ,SAAA,CAAU,YAAA,CAAa,SAAS,CAAA;AAAA,gBACxC,OAAA,CAAQ,cAAA;AAAA,gBACR,OAAA,CAAQ,KAAA,CAAM,YAAA,CAAa,SAAS;AAAA,eACtC;AAAA,YACF,SAAS,CAAA,EAAG;AACV,cAAA,MAAM,IAAIV,iBAAA;AAAA,gBACR,CAAA,UAAA,EAAa,SAAA,CAAU,WAAA,CAAY,IAAI,CAAA,mCAAA,CAAA;AAAA,gBACvC;AAAA,eACF;AAAA,YACF;AACA,YAAA,OAAO,QAAA;AAAA,UACT,CAAC,CAAA;AAAA,QACH;AAAA,MACF;AAEA,MAAA,OAAO,GAAA;AAAA,IACT,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,cAAc,MAAA,EAAiC;AAC3D,IAAA,OAAO,MAAMS,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA;AAAA,QACR,mCAAA;AAAA,QACA;AAAA,OACF;AACA,MAAA,IAAI,oBAAA;AAEJ,MAAA,IAAI;AACF,QAAA,oBAAA,GAAuB,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,QAAQ,MAAM,CAAA;AAAA,MACjE,SAAS,CAAA,EAAG;AACV,QAAA,MAAM,IAAIV,iBAAA;AAAA,UACR,CAAA,wBAAA,EAA2BC,+BAAA,CAAmB,MAAM,CAAC,CAAA,CAAA;AAAA,UACrD;AAAA,SACF;AAAA,MACF;AAEA,MAAA,IAAI,CAAC,oBAAA,EAAsB;AACzB,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,CAAA,yCAAA,EAA4CA,+BAAA;AAAA,YAC1C;AAAA,WACD,CAAA;AAAA,SACH;AAAA,MACF;AAEA,MAAA,OAAO,oBAAA;AAAA,IACT,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,eAAA,CACZ,MAAA,EACA,OAAA,EACe;AACf,IAAA,OAAO,MAAMQ,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA,CAAa,qCAAqC,UAAU,CAAA;AAGtE,MAAA,IAAIT,+BAAA,CAAmB,MAAM,CAAA,KAAM,OAAA,CAAQ,SAAA,EAAW;AACpD,QAAA,MAAM,IAAIU,oBAAA;AAAA,UACR;AAAA,SACF;AAAA,MACF;AAGA,MAAA,IAAI;AACF,QAAAC,mBAAA,CAAe,MAAM,CAAA;AAAA,MACvB,SAAS,CAAA,EAAG;AACV,QAAA,MAAM,IAAID,oBAAA;AAAA,UACR,CAAA,oBAAA,EAAuB,QAAQ,SAAS,CAAA,sCAAA,CAAA;AAAA,UACxC;AAAA,SACF;AAAA,MACF;AAEA,MAAA,IAAI,KAAA,GAAQ,KAAA;AAEZ,MAAA,KAAA,MAAW,SAAA,IAAa,IAAA,CAAK,OAAA,CAAQ,UAAA,EAAY;AAC/C,QAAA,IAAI,UAAU,kBAAA,EAAoB;AAChC,UAAA,IAAI;AACF,YAAA,MAAM,YAAY,MAAMF,4BAAA;AAAA,cACtB,MAAA;AAAA,cACA,gBAAA;AAAA,cACA,OAAM,IAAA,KAAQ;AACZ,gBAAAC,iCAAA,CAAoB,MAAM,MAAM,CAAA;AAChC,gBAAA,sBAAA,CAAuB,IAAA,EAAM,sBAAsB,SAAS,CAAA;AAC5D,gBAAA,OAAO,MAAM,SAAA,CAAU,kBAAA,CAAoB,MAAM,CAAA;AAAA,cACnD;AAAA,aACF;AACA,YAAA,IAAI,SAAA,EAAW;AACb,cAAA,KAAA,GAAQ,IAAA;AACR,cAAA,IAAI,IAAA,CAAK,QAAQ,+BAAA,EAAiC;AAChD,gBAAA;AAAA,cACF;AAAA,YACF;AAAA,UACF,SAAS,CAAA,EAAG;AACV,YAAA,MAAM,IAAIV,iBAAA;AAAA,cACR,aAAa,SAAA,CAAU,WAAA,CAAY,IAAI,CAAA,4CAAA,EAA+C,QAAQ,SAAS,CAAA,CAAA;AAAA,cACvG;AAAA,aACF;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAEA,MAAA,IAAI,CAAC,KAAA,EAAO;AACV,QAAA,MAAM,IAAIA,iBAAA;AAAA,UACR,CAAA,mCAAA,EAAsC,QAAQ,SAAS,CAAA,0DAAA;AAAA,SACzD;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,sBAAA,CACZ,MAAA,EACA,OAAA,EACe;AACf,IAAA,OAAO,MAAMS,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA;AAAA,QACR,mCAAA;AAAA,QACA;AAAA,OACF;AACA,MAAA,MAAM,EAAE,OAAO,OAAA,CAAQ,QAAA,CAAS,MAAM,QAAA,GAAW,UAAA,KAC/C,MAAA,CAAO,IAAA;AACT,MAAA,MAAM,OAAA,GAAU,IAAI,KAAA,EAAc;AAClC,MAAA,IAAI,MAAA,CAAO,KAAK,MAAA,EAAQ;AACtB,QAAA,OAAA,CAAQ,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,MAAM,CAAA;AAAA,MACjC;AACA,MAAA,IAAI,MAAA,CAAO,KAAK,OAAA,EAAS;AACvB,QAAA,OAAA,CAAQ,IAAA,CAAK,GAAG,MAAA,CAAO,IAAA,CAAK,OAAO,CAAA;AAAA,MACrC;AAEA,MAAA,KAAA,MAAW,uBAAuB,OAAA,EAAS;AACzC,QAAA,IAAI,SAAS,MAAA,IAAU,mBAAA,CAAoB,QAAA,CAASG,qBAAA,CAAK,GAAG,CAAA,EAAG;AAC7D,UAAA,OAAA,CAAQ,UAAU,OAAA,EAAQ;AAAA,YACxBC,kCAAA,CAAiB,UAAA;AAAA,cACf,OAAA,CAAQ,QAAA;AAAA,cACR,CAAA,sCAAA,EAAyC,IAAI,CAAA,2BAAA,EAA8B,OAAA,CAAQ,SAAS,MAAM,CAAA,gCAAA;AAAA;AACpG,WACF;AACA,UAAA;AAAA,QACF;AACA,QAAA,MAAM,MAAA,GAASC,kBAAA;AAAA,UACb,KAAK,OAAA,CAAQ,YAAA;AAAA,UACb,OAAA,CAAQ,QAAA;AAAA,UACR,IAAA;AAAA,UACA;AAAA,SACF;AAEA,QAAA,IAAI,OAAA,GAAU,KAAA;AACd,QAAA,KAAA,MAAW,SAAA,IAAa,IAAA,CAAK,OAAA,CAAQ,UAAA,EAAY;AAC/C,UAAA,IAAI,UAAU,YAAA,EAAc;AAC1B,YAAA,IAAI;AACF,cAAA,MAAM,OAAO,MAAMN,4BAAA;AAAA,gBACjB,MAAA;AAAA,gBACA,gBAAA;AAAA,gBACA,OAAM,IAAA,KAAQ;AACZ,kBAAAC,iCAAA,CAAoB,MAAM,MAAM,CAAA;AAChC,kBAAA,sBAAA,CAAuB,IAAA,EAAM,gBAAgB,SAAS,CAAA;AACtD,kBAAA,OAAO,MAAM,SAAA,CAAU,YAAA;AAAA,oBACrB;AAAA,sBACE,IAAA;AAAA,sBACA,MAAA;AAAA,sBACA;AAAA,qBACF;AAAA,oBACA,QAAA,KAAa,UAAA;AAAA,oBACb,OAAA,CAAQ,SAAA,CAAU,YAAA,CAAa,SAAS,CAAA;AAAA,oBACxC,KAAK,OAAA,CAAQ,MAAA;AAAA,oBACb,OAAA,CAAQ,KAAA,CAAM,YAAA,CAAa,SAAA,EAAW,MAAM;AAAA,mBAC9C;AAAA,gBACF;AAAA,eACF;AACA,cAAA,IAAI,IAAA,EAAM;AACR,gBAAA,OAAA,GAAU,IAAA;AACV,gBAAA;AAAA,cACF;AAAA,YACF,SAAS,CAAA,EAAG;AACV,cAAA,MAAM,IAAIV,iBAAA;AAAA,gBACR,aAAa,SAAA,CAAU,WAAA,CAAY,IAAI,CAAA,8BAAA,EAAiC,IAAI,IAAI,MAAM,CAAA,CAAA;AAAA,gBACtF;AAAA,eACF;AAAA,YACF;AAAA,UACF;AAAA,QACF;AACA,QAAA,IAAI,CAAC,OAAA,EAAS;AACZ,UAAA,MAAM,IAAIA,iBAAA;AAAA,YACR,CAAA,2CAAA,EAA8C,IAAI,CAAA,CAAA,EAAI,MAAM,CAAA;AAAA,WAC9D;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,kBAAA,CACZ,MAAA,EACA,OAAA,EACiB;AACjB,IAAA,OAAO,MAAMS,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA;AAAA,QACR,mCAAA;AAAA,QACA;AAAA,OACF;AACA,MAAA,IAAI,GAAA,GAAM,MAAA;AAEV,MAAA,KAAA,MAAW,SAAA,IAAa,IAAA,CAAK,OAAA,CAAQ,UAAA,EAAY;AAC/C,QAAA,IAAI,UAAU,iBAAA,EAAmB;AAC/B,UAAA,IAAI,QAAA,GAAW,GAAA;AACf,UAAA,GAAA,GAAM,MAAMD,4BAAA,CAAe,MAAA,EAAQ,gBAAA,EAAkB,OAAM,IAAA,KAAQ;AACjE,YAAAC,iCAAA,CAAoB,MAAM,MAAM,CAAA;AAChC,YAAA,sBAAA,CAAuB,IAAA,EAAM,qBAAqB,SAAS,CAAA;AAC3D,YAAA,IAAI;AACF,cAAA,QAAA,GAAW,MAAM,SAAA,CAAU,iBAAA;AAAA,gBACzB,QAAA;AAAA,gBACA,OAAA,CAAQ,QAAA;AAAA,gBACR,OAAA,CAAQ,SAAA,CAAU,YAAA,CAAa,SAAS,CAAA;AAAA,gBACxC,OAAA,CAAQ,KAAA,CAAM,YAAA,CAAa,SAAS;AAAA,eACtC;AAAA,YACF,SAAS,CAAA,EAAG;AACV,cAAA,MAAM,IAAIV,iBAAA;AAAA,gBACR,CAAA,UAAA,EAAa,SAAA,CAAU,WAAA,CAAY,IAAI,CAAA,oCAAA,CAAA;AAAA,gBACvC;AAAA,eACF;AAAA,YACF;AACA,YAAA,OAAO,QAAA;AAAA,UACT,CAAC,CAAA;AAAA,QACH;AAAA,MACF;AAEA,MAAA,OAAO,GAAA;AAAA,IACT,CAAC,CAAA;AAAA,EACH;AACF;;;;"}
+{"version":3,"file":"DefaultCatalogProcessingOrchestrator.cjs.js","sources":["../../src/processing/DefaultCatalogProcessingOrchestrator.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Span, trace } from '@opentelemetry/api';\nimport {\n  Entity,\n  EntityPolicy,\n  LocationEntity,\n  parseLocationRef,\n  stringifyEntityRef,\n  stringifyLocationRef,\n} from '@backstage/catalog-model';\nimport {\n  assertError,\n  ConflictError,\n  InputError,\n  NotAllowedError,\n} from '@backstage/errors';\nimport { JsonValue } from '@backstage/types';\nimport { ScmIntegrationRegistry } from '@backstage/integration';\nimport path from 'path';\nimport { LocationSpec } from '@backstage/plugin-catalog-common';\nimport {\n  CatalogProcessor,\n  CatalogProcessorParser,\n  processingResult,\n} from '@backstage/plugin-catalog-node';\nimport {\n  CatalogProcessingOrchestrator,\n  EntityProcessingRequest,\n  EntityProcessingResult,\n} from './types';\nimport { ProcessorOutputCollector } from './ProcessorOutputCollector';\nimport {\n  getEntityLocationRef,\n  getEntityOriginLocationRef,\n  isLocationEntity,\n  isObject,\n  toAbsoluteUrl,\n  validateEntity,\n  validateEntityEnvelope,\n} from './util';\nimport { CatalogRulesEnforcer } from '../ingestion/CatalogRules';\nimport { ProcessorCacheManager } from './ProcessorCacheManager';\nimport {\n  addEntityAttributes,\n  TRACER_ID,\n  withActiveSpan,\n} from '../util/opentelemetry';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\nconst tracer = trace.getTracer(TRACER_ID);\n\ntype Context = {\n  entityRef: string;\n  location: LocationSpec;\n  originLocation: LocationSpec;\n  collector: ProcessorOutputCollector;\n  cache: ProcessorCacheManager;\n};\n\nfunction addProcessorAttributes(\n  span: Span,\n  stage: string,\n  processor: CatalogProcessor,\n) {\n  span.setAttribute('backstage.catalog.processor.stage', stage);\n  span.setAttribute(\n    'backstage.catalog.processor.name',\n    processor.getProcessorName(),\n  );\n}\n\nexport class DefaultCatalogProcessingOrchestrator\n  implements CatalogProcessingOrchestrator\n{\n  constructor(\n    private readonly options: {\n      processors: CatalogProcessor[];\n      integrations: ScmIntegrationRegistry;\n      logger: LoggerService;\n      parser: CatalogProcessorParser;\n      policy: EntityPolicy;\n      rulesEnforcer: CatalogRulesEnforcer;\n    },\n  ) {}\n\n  async process(\n    request: EntityProcessingRequest,\n  ): Promise<EntityProcessingResult> {\n    return this.processSingleEntity(request.entity, request.state);\n  }\n\n  private async processSingleEntity(\n    unprocessedEntity: Entity,\n    state: JsonValue | undefined,\n  ): Promise<EntityProcessingResult> {\n    const collector = new ProcessorOutputCollector(\n      this.options.logger,\n      unprocessedEntity,\n    );\n\n    // Cache that is scoped to the entity and processor\n    const cache = new ProcessorCacheManager(\n      isObject(state) && isObject(state.cache) ? state.cache : {},\n    );\n\n    try {\n      // This will be checked and mutated step by step below\n      let entity: Entity = unprocessedEntity;\n\n      // NOTE: At this early point, we can only rely on the envelope having to\n      // be valid; full entity + kind validation happens after the (potentially\n      // mutative) pre-steps. This means that the code below can't make a lot\n      // of assumptions about the data despite it using the Entity type.\n      try {\n        validateEntityEnvelope(entity);\n      } catch (e) {\n        throw new InputError(\n          `Entity envelope failed validation before processing`,\n          e,\n        );\n      }\n\n      // TODO: which one do we actually use for the location?\n      // source-location? - maybe probably doesn't exist yet?\n      const context: Context = {\n        entityRef: stringifyEntityRef(entity),\n        location: parseLocationRef(getEntityLocationRef(entity)),\n        originLocation: parseLocationRef(getEntityOriginLocationRef(entity)),\n        cache,\n        collector,\n      };\n\n      // Run the steps\n      entity = await this.runPreProcessStep(entity, context);\n      entity = await this.runPolicyStep(entity);\n      await this.runValidateStep(entity, context);\n      if (isLocationEntity(entity)) {\n        await this.runSpecialLocationStep(entity, context);\n      }\n      entity = await this.runPostProcessStep(entity, context);\n\n      // Check that any emitted entities are permitted to originate from that\n      // particular location according to the catalog rules\n      const collectorResults = context.collector.results();\n      for (const deferredEntity of collectorResults.deferredEntities) {\n        if (\n          !this.options.rulesEnforcer.isAllowed(\n            deferredEntity.entity,\n            context.originLocation,\n          )\n        ) {\n          throw new NotAllowedError(\n            `Entity ${stringifyEntityRef(\n              deferredEntity.entity,\n            )} at ${stringifyLocationRef(\n              context.location,\n            )}, originated at ${stringifyLocationRef(\n              context.originLocation,\n            )}, is not of an allowed kind for that location`,\n          );\n        }\n      }\n\n      return {\n        ...collectorResults,\n        completedEntity: entity,\n        state: { cache: cache.collect() },\n        ok: collectorResults.errors.length === 0,\n      };\n    } catch (error) {\n      assertError(error);\n      return {\n        ok: false,\n        errors: collector.results().errors.concat(error),\n      };\n    }\n  }\n\n  // Pre-process phase, used to populate entities with data that is required\n  // during the main processing step\n  private async runPreProcessStep(\n    entity: Entity,\n    context: Context,\n  ): Promise<Entity> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute('backstage.catalog.processor.stage', 'preProcess');\n      let res = entity;\n\n      for (const processor of this.options.processors) {\n        if (processor.preProcessEntity) {\n          let innerRes = res;\n          res = await withActiveSpan(tracer, 'ProcessingStep', async span => {\n            addEntityAttributes(span, entity);\n            addProcessorAttributes(span, 'preProcessEntity', processor);\n            try {\n              innerRes = await processor.preProcessEntity!(\n                innerRes,\n                context.location,\n                context.collector.forProcessor(processor),\n                context.originLocation,\n                context.cache.forProcessor(processor),\n              );\n            } catch (e) {\n              throw new InputError(\n                `Processor ${processor.constructor.name} threw an error while preprocessing`,\n                e,\n              );\n            }\n            return innerRes;\n          });\n        }\n      }\n\n      return res;\n    });\n  }\n\n  /**\n   * Enforce entity policies making sure that entities conform to a general schema\n   */\n  private async runPolicyStep(entity: Entity): Promise<Entity> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute(\n        'backstage.catalog.processor.stage',\n        'enforcePolicy',\n      );\n      let policyEnforcedEntity: Entity | undefined;\n\n      try {\n        policyEnforcedEntity = await this.options.policy.enforce(entity);\n      } catch (e) {\n        throw new InputError(\n          `Policy check failed for ${stringifyEntityRef(entity)}`,\n          e,\n        );\n      }\n\n      if (!policyEnforcedEntity) {\n        throw new Error(\n          `Policy unexpectedly returned no data for ${stringifyEntityRef(\n            entity,\n          )}`,\n        );\n      }\n\n      return policyEnforcedEntity;\n    });\n  }\n\n  /**\n   * Validate the given entity\n   */\n  private async runValidateStep(\n    entity: Entity,\n    context: Context,\n  ): Promise<void> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute('backstage.catalog.processor.stage', 'validate');\n      // Double check that none of the previous steps tried to change something\n      // related to the entity ref, which would break downstream\n      if (stringifyEntityRef(entity) !== context.entityRef) {\n        throw new ConflictError(\n          'Fatal: The entity kind, namespace, or name changed during processing',\n        );\n      }\n\n      // Validate that the end result is a valid Entity at all\n      try {\n        validateEntity(entity);\n      } catch (e) {\n        throw new ConflictError(\n          `Entity envelope for ${context.entityRef} failed validation after preprocessing`,\n          e,\n        );\n      }\n\n      let valid = false;\n\n      for (const processor of this.options.processors) {\n        if (processor.validateEntityKind) {\n          try {\n            const thisValid = await withActiveSpan(\n              tracer,\n              'ProcessingStep',\n              async span => {\n                addEntityAttributes(span, entity);\n                addProcessorAttributes(span, 'validateEntityKind', processor);\n                return await processor.validateEntityKind!(entity);\n              },\n            );\n            if (thisValid) {\n              valid = true;\n            }\n          } catch (e) {\n            throw new InputError(\n              `Processor ${processor.constructor.name} threw an error while validating the entity ${context.entityRef}`,\n              e,\n            );\n          }\n        }\n      }\n\n      if (!valid) {\n        throw new InputError(\n          `No processor recognized the entity ${context.entityRef} as valid, possibly caused by a foreign kind or apiVersion`,\n        );\n      }\n    });\n  }\n\n  /**\n   * Backwards compatible processing of location entities\n   */\n  private async runSpecialLocationStep(\n    entity: LocationEntity,\n    context: Context,\n  ): Promise<void> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute(\n        'backstage.catalog.processor.stage',\n        'readLocation',\n      );\n      const { type = context.location.type, presence = 'required' } =\n        entity.spec;\n      const targets = new Array<string>();\n      if (entity.spec.target) {\n        targets.push(entity.spec.target);\n      }\n      if (entity.spec.targets) {\n        targets.push(...entity.spec.targets);\n      }\n\n      for (const maybeRelativeTarget of targets) {\n        if (type === 'file' && maybeRelativeTarget.endsWith(path.sep)) {\n          context.collector.generic()(\n            processingResult.inputError(\n              context.location,\n              `LocationEntityProcessor cannot handle ${type} type location with target ${context.location.target} that ends with a path separator`,\n            ),\n          );\n          continue;\n        }\n        const target = toAbsoluteUrl(\n          this.options.integrations,\n          context.location,\n          type,\n          maybeRelativeTarget,\n        );\n\n        let didRead = false;\n        for (const processor of this.options.processors) {\n          if (processor.readLocation) {\n            try {\n              const read = await withActiveSpan(\n                tracer,\n                'ProcessingStep',\n                async span => {\n                  addEntityAttributes(span, entity);\n                  addProcessorAttributes(span, 'readLocation', processor);\n                  return await processor.readLocation!(\n                    {\n                      type,\n                      target,\n                      presence,\n                    },\n                    presence === 'optional',\n                    context.collector.forProcessor(processor),\n                    this.options.parser,\n                    context.cache.forProcessor(processor, target),\n                  );\n                },\n              );\n              if (read) {\n                didRead = true;\n                break;\n              }\n            } catch (e) {\n              throw new InputError(\n                `Processor ${processor.constructor.name} threw an error while reading ${type}:${target}`,\n                e,\n              );\n            }\n          }\n        }\n        if (!didRead) {\n          throw new InputError(\n            `No processor was able to handle reading of ${type}:${target}`,\n          );\n        }\n      }\n    });\n  }\n\n  /**\n   * Main processing step of the entity\n   */\n  private async runPostProcessStep(\n    entity: Entity,\n    context: Context,\n  ): Promise<Entity> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute(\n        'backstage.catalog.processor.stage',\n        'postProcessEntity',\n      );\n      let res = entity;\n\n      for (const processor of this.options.processors) {\n        if (processor.postProcessEntity) {\n          let innerRes = res;\n          res = await withActiveSpan(tracer, 'ProcessingStep', async span => {\n            addEntityAttributes(span, entity);\n            addProcessorAttributes(span, 'postProcessEntity', processor);\n            try {\n              innerRes = await processor.postProcessEntity!(\n                innerRes,\n                context.location,\n                context.collector.forProcessor(processor),\n                context.cache.forProcessor(processor),\n              );\n            } catch (e) {\n              throw new InputError(\n                `Processor ${processor.constructor.name} threw an error while postprocessing`,\n                e,\n              );\n            }\n            return innerRes;\n          });\n        }\n      }\n\n      return res;\n    });\n  }\n}\n"],"names":["trace","TRACER_ID","ProcessorOutputCollector","ProcessorCacheManager","isObject","validateEntityEnvelope","InputError","stringifyEntityRef","parseLocationRef","getEntityLocationRef","getEntityOriginLocationRef","isLocationEntity","NotAllowedError","stringifyLocationRef","assertError","withActiveSpan","addEntityAttributes","ConflictError","validateEntity","path","processingResult","toAbsoluteUrl"],"mappings":";;;;;;;;;;;;;;;;AAgEA,MAAM,MAAA,GAASA,SAAA,CAAM,SAAA,CAAUC,uBAAS,CAAA;AAUxC,SAAS,sBAAA,CACP,IAAA,EACA,KAAA,EACA,SAAA,EACA;AACA,EAAA,IAAA,CAAK,YAAA,CAAa,qCAAqC,KAAK,CAAA;AAC5D,EAAA,IAAA,CAAK,YAAA;AAAA,IACH,kCAAA;AAAA,IACA,UAAU,gBAAA;AAAiB,GAC7B;AACF;AAEO,MAAM,oCAAA,CAEb;AAAA,EACE,YACmB,OAAA,EAQjB;AARiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA,EAQhB;AAAA,EAEH,MAAM,QACJ,OAAA,EACiC;AACjC,IAAA,OAAO,IAAA,CAAK,mBAAA,CAAoB,OAAA,CAAQ,MAAA,EAAQ,QAAQ,KAAK,CAAA;AAAA,EAC/D;AAAA,EAEA,MAAc,mBAAA,CACZ,iBAAA,EACA,KAAA,EACiC;AACjC,IAAA,MAAM,YAAY,IAAIC,iDAAA;AAAA,MACpB,KAAK,OAAA,CAAQ,MAAA;AAAA,MACb;AAAA,KACF;AAGA,IAAA,MAAM,QAAQ,IAAIC,2CAAA;AAAA,MAChBC,aAAA,CAAS,KAAK,CAAA,IAAKA,aAAA,CAAS,MAAM,KAAK,CAAA,GAAI,KAAA,CAAM,KAAA,GAAQ;AAAC,KAC5D;AAEA,IAAA,IAAI;AAEF,MAAA,IAAI,MAAA,GAAiB,iBAAA;AAMrB,MAAA,IAAI;AACF,QAAAC,2BAAA,CAAuB,MAAM,CAAA;AAAA,MAC/B,SAAS,CAAA,EAAG;AACV,QAAA,MAAM,IAAIC,iBAAA;AAAA,UACR,CAAA,mDAAA,CAAA;AAAA,UACA;AAAA,SACF;AAAA,MACF;AAIA,MAAA,MAAM,OAAA,GAAmB;AAAA,QACvB,SAAA,EAAWC,gCAAmB,MAAM,CAAA;AAAA,QACpC,QAAA,EAAUC,6BAAA,CAAiBC,yBAAA,CAAqB,MAAM,CAAC,CAAA;AAAA,QACvD,cAAA,EAAgBD,6BAAA,CAAiBE,+BAAA,CAA2B,MAAM,CAAC,CAAA;AAAA,QACnE,KAAA;AAAA,QACA;AAAA,OACF;AAGA,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,iBAAA,CAAkB,MAAA,EAAQ,OAAO,CAAA;AACrD,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,aAAA,CAAc,MAAM,CAAA;AACxC,MAAA,MAAM,IAAA,CAAK,eAAA,CAAgB,MAAA,EAAQ,OAAO,CAAA;AAC1C,MAAA,IAAIC,qBAAA,CAAiB,MAAM,CAAA,EAAG;AAC5B,QAAA,MAAM,IAAA,CAAK,sBAAA,CAAuB,MAAA,EAAQ,OAAO,CAAA;AAAA,MACnD;AACA,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,kBAAA,CAAmB,MAAA,EAAQ,OAAO,CAAA;AAItD,MAAA,MAAM,gBAAA,GAAmB,OAAA,CAAQ,SAAA,CAAU,OAAA,EAAQ;AACnD,MAAA,KAAA,MAAW,cAAA,IAAkB,iBAAiB,gBAAA,EAAkB;AAC9D,QAAA,IACE,CAAC,IAAA,CAAK,OAAA,CAAQ,aAAA,CAAc,SAAA;AAAA,UAC1B,cAAA,CAAe,MAAA;AAAA,UACf,OAAA,CAAQ;AAAA,SACV,EACA;AACA,UAAA,MAAM,IAAIC,sBAAA;AAAA,YACR,CAAA,OAAA,EAAUL,+BAAA;AAAA,cACR,cAAA,CAAe;AAAA,aAChB,CAAA,IAAA,EAAOM,iCAAA;AAAA,cACN,OAAA,CAAQ;AAAA,aACT,CAAA,gBAAA,EAAmBA,iCAAA;AAAA,cAClB,OAAA,CAAQ;AAAA,aACT,CAAA,6CAAA;AAAA,WACH;AAAA,QACF;AAAA,MACF;AAEA,MAAA,OAAO;AAAA,QACL,GAAG,gBAAA;AAAA,QACH,eAAA,EAAiB,MAAA;AAAA,QACjB,KAAA,EAAO,EAAE,KAAA,EAAO,KAAA,CAAM,SAAQ,EAAE;AAAA,QAChC,EAAA,EAAI,gBAAA,CAAiB,MAAA,CAAO,MAAA,KAAW;AAAA,OACzC;AAAA,IACF,SAAS,KAAA,EAAO;AACd,MAAAC,kBAAA,CAAY,KAAK,CAAA;AACjB,MAAA,OAAO;AAAA,QACL,EAAA,EAAI,KAAA;AAAA,QACJ,QAAQ,SAAA,CAAU,OAAA,EAAQ,CAAE,MAAA,CAAO,OAAO,KAAK;AAAA,OACjD;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA,EAIA,MAAc,iBAAA,CACZ,MAAA,EACA,OAAA,EACiB;AACjB,IAAA,OAAO,MAAMC,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA,CAAa,qCAAqC,YAAY,CAAA;AACxE,MAAA,IAAI,GAAA,GAAM,MAAA;AAEV,MAAA,KAAA,MAAW,SAAA,IAAa,IAAA,CAAK,OAAA,CAAQ,UAAA,EAAY;AAC/C,QAAA,IAAI,UAAU,gBAAA,EAAkB;AAC9B,UAAA,IAAI,QAAA,GAAW,GAAA;AACf,UAAA,GAAA,GAAM,MAAMD,4BAAA,CAAe,MAAA,EAAQ,gBAAA,EAAkB,OAAM,IAAA,KAAQ;AACjE,YAAAC,iCAAA,CAAoB,MAAM,MAAM,CAAA;AAChC,YAAA,sBAAA,CAAuB,IAAA,EAAM,oBAAoB,SAAS,CAAA;AAC1D,YAAA,IAAI;AACF,cAAA,QAAA,GAAW,MAAM,SAAA,CAAU,gBAAA;AAAA,gBACzB,QAAA;AAAA,gBACA,OAAA,CAAQ,QAAA;AAAA,gBACR,OAAA,CAAQ,SAAA,CAAU,YAAA,CAAa,SAAS,CAAA;AAAA,gBACxC,OAAA,CAAQ,cAAA;AAAA,gBACR,OAAA,CAAQ,KAAA,CAAM,YAAA,CAAa,SAAS;AAAA,eACtC;AAAA,YACF,SAAS,CAAA,EAAG;AACV,cAAA,MAAM,IAAIV,iBAAA;AAAA,gBACR,CAAA,UAAA,EAAa,SAAA,CAAU,WAAA,CAAY,IAAI,CAAA,mCAAA,CAAA;AAAA,gBACvC;AAAA,eACF;AAAA,YACF;AACA,YAAA,OAAO,QAAA;AAAA,UACT,CAAC,CAAA;AAAA,QACH;AAAA,MACF;AAEA,MAAA,OAAO,GAAA;AAAA,IACT,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,cAAc,MAAA,EAAiC;AAC3D,IAAA,OAAO,MAAMS,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA;AAAA,QACR,mCAAA;AAAA,QACA;AAAA,OACF;AACA,MAAA,IAAI,oBAAA;AAEJ,MAAA,IAAI;AACF,QAAA,oBAAA,GAAuB,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,QAAQ,MAAM,CAAA;AAAA,MACjE,SAAS,CAAA,EAAG;AACV,QAAA,MAAM,IAAIV,iBAAA;AAAA,UACR,CAAA,wBAAA,EAA2BC,+BAAA,CAAmB,MAAM,CAAC,CAAA,CAAA;AAAA,UACrD;AAAA,SACF;AAAA,MACF;AAEA,MAAA,IAAI,CAAC,oBAAA,EAAsB;AACzB,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,CAAA,yCAAA,EAA4CA,+BAAA;AAAA,YAC1C;AAAA,WACD,CAAA;AAAA,SACH;AAAA,MACF;AAEA,MAAA,OAAO,oBAAA;AAAA,IACT,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,eAAA,CACZ,MAAA,EACA,OAAA,EACe;AACf,IAAA,OAAO,MAAMQ,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA,CAAa,qCAAqC,UAAU,CAAA;AAGtE,MAAA,IAAIT,+BAAA,CAAmB,MAAM,CAAA,KAAM,OAAA,CAAQ,SAAA,EAAW;AACpD,QAAA,MAAM,IAAIU,oBAAA;AAAA,UACR;AAAA,SACF;AAAA,MACF;AAGA,MAAA,IAAI;AACF,QAAAC,mBAAA,CAAe,MAAM,CAAA;AAAA,MACvB,SAAS,CAAA,EAAG;AACV,QAAA,MAAM,IAAID,oBAAA;AAAA,UACR,CAAA,oBAAA,EAAuB,QAAQ,SAAS,CAAA,sCAAA,CAAA;AAAA,UACxC;AAAA,SACF;AAAA,MACF;AAEA,MAAA,IAAI,KAAA,GAAQ,KAAA;AAEZ,MAAA,KAAA,MAAW,SAAA,IAAa,IAAA,CAAK,OAAA,CAAQ,UAAA,EAAY;AAC/C,QAAA,IAAI,UAAU,kBAAA,EAAoB;AAChC,UAAA,IAAI;AACF,YAAA,MAAM,YAAY,MAAMF,4BAAA;AAAA,cACtB,MAAA;AAAA,cACA,gBAAA;AAAA,cACA,OAAM,IAAA,KAAQ;AACZ,gBAAAC,iCAAA,CAAoB,MAAM,MAAM,CAAA;AAChC,gBAAA,sBAAA,CAAuB,IAAA,EAAM,sBAAsB,SAAS,CAAA;AAC5D,gBAAA,OAAO,MAAM,SAAA,CAAU,kBAAA,CAAoB,MAAM,CAAA;AAAA,cACnD;AAAA,aACF;AACA,YAAA,IAAI,SAAA,EAAW;AACb,cAAA,KAAA,GAAQ,IAAA;AAAA,YACV;AAAA,UACF,SAAS,CAAA,EAAG;AACV,YAAA,MAAM,IAAIV,iBAAA;AAAA,cACR,aAAa,SAAA,CAAU,WAAA,CAAY,IAAI,CAAA,4CAAA,EAA+C,QAAQ,SAAS,CAAA,CAAA;AAAA,cACvG;AAAA,aACF;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAEA,MAAA,IAAI,CAAC,KAAA,EAAO;AACV,QAAA,MAAM,IAAIA,iBAAA;AAAA,UACR,CAAA,mCAAA,EAAsC,QAAQ,SAAS,CAAA,0DAAA;AAAA,SACzD;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,sBAAA,CACZ,MAAA,EACA,OAAA,EACe;AACf,IAAA,OAAO,MAAMS,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA;AAAA,QACR,mCAAA;AAAA,QACA;AAAA,OACF;AACA,MAAA,MAAM,EAAE,OAAO,OAAA,CAAQ,QAAA,CAAS,MAAM,QAAA,GAAW,UAAA,KAC/C,MAAA,CAAO,IAAA;AACT,MAAA,MAAM,OAAA,GAAU,IAAI,KAAA,EAAc;AAClC,MAAA,IAAI,MAAA,CAAO,KAAK,MAAA,EAAQ;AACtB,QAAA,OAAA,CAAQ,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,MAAM,CAAA;AAAA,MACjC;AACA,MAAA,IAAI,MAAA,CAAO,KAAK,OAAA,EAAS;AACvB,QAAA,OAAA,CAAQ,IAAA,CAAK,GAAG,MAAA,CAAO,IAAA,CAAK,OAAO,CAAA;AAAA,MACrC;AAEA,MAAA,KAAA,MAAW,uBAAuB,OAAA,EAAS;AACzC,QAAA,IAAI,SAAS,MAAA,IAAU,mBAAA,CAAoB,QAAA,CAASG,qBAAA,CAAK,GAAG,CAAA,EAAG;AAC7D,UAAA,OAAA,CAAQ,UAAU,OAAA,EAAQ;AAAA,YACxBC,kCAAA,CAAiB,UAAA;AAAA,cACf,OAAA,CAAQ,QAAA;AAAA,cACR,CAAA,sCAAA,EAAyC,IAAI,CAAA,2BAAA,EAA8B,OAAA,CAAQ,SAAS,MAAM,CAAA,gCAAA;AAAA;AACpG,WACF;AACA,UAAA;AAAA,QACF;AACA,QAAA,MAAM,MAAA,GAASC,kBAAA;AAAA,UACb,KAAK,OAAA,CAAQ,YAAA;AAAA,UACb,OAAA,CAAQ,QAAA;AAAA,UACR,IAAA;AAAA,UACA;AAAA,SACF;AAEA,QAAA,IAAI,OAAA,GAAU,KAAA;AACd,QAAA,KAAA,MAAW,SAAA,IAAa,IAAA,CAAK,OAAA,CAAQ,UAAA,EAAY;AAC/C,UAAA,IAAI,UAAU,YAAA,EAAc;AAC1B,YAAA,IAAI;AACF,cAAA,MAAM,OAAO,MAAMN,4BAAA;AAAA,gBACjB,MAAA;AAAA,gBACA,gBAAA;AAAA,gBACA,OAAM,IAAA,KAAQ;AACZ,kBAAAC,iCAAA,CAAoB,MAAM,MAAM,CAAA;AAChC,kBAAA,sBAAA,CAAuB,IAAA,EAAM,gBAAgB,SAAS,CAAA;AACtD,kBAAA,OAAO,MAAM,SAAA,CAAU,YAAA;AAAA,oBACrB;AAAA,sBACE,IAAA;AAAA,sBACA,MAAA;AAAA,sBACA;AAAA,qBACF;AAAA,oBACA,QAAA,KAAa,UAAA;AAAA,oBACb,OAAA,CAAQ,SAAA,CAAU,YAAA,CAAa,SAAS,CAAA;AAAA,oBACxC,KAAK,OAAA,CAAQ,MAAA;AAAA,oBACb,OAAA,CAAQ,KAAA,CAAM,YAAA,CAAa,SAAA,EAAW,MAAM;AAAA,mBAC9C;AAAA,gBACF;AAAA,eACF;AACA,cAAA,IAAI,IAAA,EAAM;AACR,gBAAA,OAAA,GAAU,IAAA;AACV,gBAAA;AAAA,cACF;AAAA,YACF,SAAS,CAAA,EAAG;AACV,cAAA,MAAM,IAAIV,iBAAA;AAAA,gBACR,aAAa,SAAA,CAAU,WAAA,CAAY,IAAI,CAAA,8BAAA,EAAiC,IAAI,IAAI,MAAM,CAAA,CAAA;AAAA,gBACtF;AAAA,eACF;AAAA,YACF;AAAA,UACF;AAAA,QACF;AACA,QAAA,IAAI,CAAC,OAAA,EAAS;AACZ,UAAA,MAAM,IAAIA,iBAAA;AAAA,YACR,CAAA,2CAAA,EAA8C,IAAI,CAAA,CAAA,EAAI,MAAM,CAAA;AAAA,WAC9D;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,kBAAA,CACZ,MAAA,EACA,OAAA,EACiB;AACjB,IAAA,OAAO,MAAMS,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA;AAAA,QACR,mCAAA;AAAA,QACA;AAAA,OACF;AACA,MAAA,IAAI,GAAA,GAAM,MAAA;AAEV,MAAA,KAAA,MAAW,SAAA,IAAa,IAAA,CAAK,OAAA,CAAQ,UAAA,EAAY;AAC/C,QAAA,IAAI,UAAU,iBAAA,EAAmB;AAC/B,UAAA,IAAI,QAAA,GAAW,GAAA;AACf,UAAA,GAAA,GAAM,MAAMD,4BAAA,CAAe,MAAA,EAAQ,gBAAA,EAAkB,OAAM,IAAA,KAAQ;AACjE,YAAAC,iCAAA,CAAoB,MAAM,MAAM,CAAA;AAChC,YAAA,sBAAA,CAAuB,IAAA,EAAM,qBAAqB,SAAS,CAAA;AAC3D,YAAA,IAAI;AACF,cAAA,QAAA,GAAW,MAAM,SAAA,CAAU,iBAAA;AAAA,gBACzB,QAAA;AAAA,gBACA,OAAA,CAAQ,QAAA;AAAA,gBACR,OAAA,CAAQ,SAAA,CAAU,YAAA,CAAa,SAAS,CAAA;AAAA,gBACxC,OAAA,CAAQ,KAAA,CAAM,YAAA,CAAa,SAAS;AAAA,eACtC;AAAA,YACF,SAAS,CAAA,EAAG;AACV,cAAA,MAAM,IAAIV,iBAAA;AAAA,gBACR,CAAA,UAAA,EAAa,SAAA,CAAU,WAAA,CAAY,IAAI,CAAA,oCAAA,CAAA;AAAA,gBACvC;AAAA,eACF;AAAA,YACF;AACA,YAAA,OAAO,QAAA;AAAA,UACT,CAAC,CAAA;AAAA,QACH;AAAA,MACF;AAEA,MAAA,OAAO,GAAA;AAAA,IACT,CAAC,CAAA;AAAA,EACH;AACF;;;;"}