@backstage/plugin-catalog-backend 3.1.1-next.1 → 3.1.1

package/CHANGELOG.md

@@ -1,28 +1,10 @@
 # @backstage/plugin-catalog-backend
 
-## 3.1.1-next.1
+## 3.1.1
 
 ### Patch Changes
 
-- Updated dependencies
-  - @backstage/config@1.3.4-next.0
-  - @backstage/integration@1.18.1-next.1
-  - @backstage/backend-plugin-api@1.4.4-next.0
-  - @backstage/plugin-permission-common@0.9.2-next.0
-  - @backstage/plugin-permission-node@0.10.5-next.0
-  - @backstage/backend-openapi-utils@0.6.2-next.0
-  - @backstage/plugin-catalog-node@1.19.1-next.0
-  - @backstage/plugin-events-node@0.4.16-next.0
-  - @backstage/plugin-catalog-common@1.1.6-next.0
-  - @backstage/catalog-client@1.12.0
-
-## 3.1.1-next.0
-
-### Patch Changes
-
-- 9890488: Internal refactor to remove remnants of the old backend system
-- 2aaf01a: Fix for duplicate search results in entity-facets API call
-- e489661: Moved catalog processor and provider disabling and priorities under own config objects.
+- 27b6849: Moved catalog processor and provider disabling and priorities under own config objects.
 
   This is due to issue with some existing providers, such as GitHub, using array syntax for the provider configuration.
 
@@ -45,19 +27,7 @@
   ```
 
 - Updated dependencies
-  - @backstage/integration@1.18.1-next.0
-  - @backstage/plugin-permission-node@0.10.4
-  - @backstage/backend-openapi-utils@0.6.1
-  - @backstage/backend-plugin-api@1.4.3
-  - @backstage/catalog-client@1.12.0
-  - @backstage/catalog-model@1.7.5
-  - @backstage/config@1.3.3
-  - @backstage/errors@1.2.7
-  - @backstage/types@1.2.2
-  - @backstage/plugin-catalog-common@1.1.5
-  - @backstage/plugin-catalog-node@1.19.0
-  - @backstage/plugin-events-node@0.4.15
-  - @backstage/plugin-permission-common@0.9.1
+  - @backstage/config@1.3.4
 
 ## 3.1.0
 

package/dist/database/DefaultProcessingDatabase.cjs.js

@@ -222,7 +222,7 @@ class DefaultProcessingDatabase {
         this.options.logger.warn(
           `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`
         );
-        if (locationKey) {
+        if (this.options.eventBroker && locationKey) {
           const eventParams = {
             topic: constants.CATALOG_CONFLICTS_TOPIC,
             eventPayload: {
@@ -233,7 +233,7 @@ class DefaultProcessingDatabase {
               lastConflictAt: luxon.DateTime.now().toISO()
             }
           };
-          await this.options.events.publish(eventParams);
+          await this.options.eventBroker?.publish(eventParams);
         }
       }
     }

package/dist/database/DefaultProcessingDatabase.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultProcessingDatabase.cjs.js","sources":["../../src/database/DefaultProcessingDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError } from '@backstage/errors';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { ProcessingIntervalFunction } from '../processing/refresh';\nimport { rethrowError, timestampToDateTime } from './conversion';\nimport { initDatabaseMetrics } from './metrics';\nimport {\n  DbRefreshKeysRow,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n} from './tables';\nimport {\n  GetProcessableEntitiesResult,\n  ListParentsOptions,\n  ListParentsResult,\n  ProcessingDatabase,\n  RefreshStateItem,\n  Transaction,\n  UpdateEntityCacheOptions,\n  UpdateProcessedEntityOptions,\n} from './types';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { generateStableHash, generateTargetKey } from './util';\nimport { EventParams, EventsService } from '@backstage/plugin-events-node';\nimport { DateTime } from 'luxon';\nimport { CATALOG_CONFLICTS_TOPIC } from '../constants';\nimport { CatalogConflictEventPayload } from '../catalog/types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProcessingDatabase implements ProcessingDatabase {\n  constructor(\n    private readonly options: {\n      database: Knex;\n      logger: LoggerService;\n      refreshInterval: ProcessingIntervalFunction;\n      events: EventsService;\n    },\n  ) {\n    initDatabaseMetrics(options.database);\n  }\n\n  async updateProcessedEntity(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<{ previous: { relations: DbRelationsRow[] } }> {\n    const tx = txOpaque as Knex.Transaction;\n    const {\n      id,\n      processedEntity,\n      resultHash,\n      errors,\n      relations,\n      deferredEntities,\n      refreshKeys,\n      locationKey,\n    } = options;\n    const configClient = tx.client.config.client;\n    const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        processed_entity: JSON.stringify(processedEntity),\n        result_hash: resultHash,\n        errors,\n        location_key: locationKey,\n      })\n      .where('entity_id', id)\n      .andWhere(inner => {\n        if (!locationKey) {\n          return inner.whereNull('location_key');\n        }\n        return inner\n          .where('location_key', locationKey)\n          .orWhereNull('location_key');\n      });\n    if (refreshResult === 0) {\n      throw new ConflictError(\n        `Conflicting write of processing result for ${id} with location key '${locationKey}'`,\n      );\n    }\n    const sourceEntityRef = stringifyEntityRef(processedEntity);\n\n    // Schedule all deferred entities for future processing.\n    await this.addUnprocessedEntities(tx, {\n      entities: deferredEntities,\n      sourceEntityRef,\n    });\n\n    // Delete old relations\n    // NOTE(freben): knex implemented support for returning() on update queries for sqlite, but at the current time of writing (Sep 2022) not for delete() queries.\n    let previousRelationRows: DbRelationsRow[];\n    if (configClient.includes('sqlite3') || configClient.includes('mysql')) {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .select('*')\n        .where({ originating_entity_id: id });\n      await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete();\n    } else {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete()\n        .returning('*');\n    }\n\n    // Batch insert new relations\n    const relationRows: DbRelationsRow[] = relations.map(\n      ({ source, target, type }) => ({\n        originating_entity_id: id,\n        source_entity_ref: stringifyEntityRef(source),\n        target_entity_ref: stringifyEntityRef(target),\n        type,\n      }),\n    );\n\n    await tx.batchInsert(\n      'relations',\n      this.deduplicateRelations(relationRows),\n      BATCH_SIZE,\n    );\n\n    // Delete old refresh keys\n    await tx<DbRefreshKeysRow>('refresh_keys')\n      .where({ entity_id: id })\n      .delete();\n\n    // Insert the refresh keys for the processed entity\n    await tx.batchInsert(\n      'refresh_keys',\n      refreshKeys.map(k => ({\n        entity_id: id,\n        key: generateTargetKey(k.key),\n      })),\n      BATCH_SIZE,\n    );\n\n    return {\n      previous: {\n        relations: previousRelationRows,\n      },\n    };\n  }\n\n  async updateProcessedEntityErrors(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, errors, resultHash } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        errors,\n        result_hash: resultHash,\n      })\n      .where('entity_id', id);\n  }\n\n  async updateEntityCache(\n    txOpaque: Transaction,\n    options: UpdateEntityCacheOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, state } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({ cache: JSON.stringify(state ?? {}) })\n      .where('entity_id', id);\n  }\n\n  async getProcessableEntities(\n    maybeTx: Transaction | Knex,\n    request: { processBatchSize: number },\n  ): Promise<GetProcessableEntitiesResult> {\n    const knex = maybeTx as Knex.Transaction | Knex;\n\n    let itemsQuery = knex<DbRefreshStateRow>('refresh_state').select([\n      'entity_id',\n      'entity_ref',\n      'unprocessed_entity',\n      'result_hash',\n      'cache',\n      'errors',\n      'location_key',\n      'next_update_at',\n    ]);\n\n    // This avoids duplication of work because of race conditions and is\n    // also fast because locked rows are ignored rather than blocking.\n    // It's only available in MySQL and PostgreSQL\n    if (['mysql', 'mysql2', 'pg'].includes(knex.client.config.client)) {\n      itemsQuery = itemsQuery.forUpdate().skipLocked();\n    }\n\n    const items = await itemsQuery\n      .where('next_update_at', '<=', knex.fn.now())\n      .limit(request.processBatchSize)\n      .orderBy('next_update_at', 'asc');\n\n    const interval = this.options.refreshInterval();\n\n    const nextUpdateAt = (refreshInterval: number) => {\n      if (knex.client.config.client.includes('sqlite3')) {\n        return knex.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);\n      } else if (knex.client.config.client.includes('mysql')) {\n        return knex.raw(`now() + interval ${refreshInterval} second`);\n      }\n      return knex.raw(`now() + interval '${refreshInterval} seconds'`);\n    };\n\n    await knex<DbRefreshStateRow>('refresh_state')\n      .whereIn(\n        'entity_ref',\n        items.map(i => i.entity_ref),\n      )\n      .update({\n        next_update_at: nextUpdateAt(interval),\n      });\n\n    return {\n      items: items.map(\n        i =>\n          ({\n            id: i.entity_id,\n            entityRef: i.entity_ref,\n            unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity,\n            resultHash: i.result_hash || '',\n            nextUpdateAt: timestampToDateTime(i.next_update_at),\n            state: i.cache ? JSON.parse(i.cache) : undefined,\n            errors: i.errors,\n            locationKey: i.location_key,\n          } satisfies RefreshStateItem),\n      ),\n    };\n  }\n\n  async listParents(\n    txOpaque: Transaction,\n    options: ListParentsOptions,\n  ): Promise<ListParentsResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .whereIn('target_entity_ref', options.entityRefs)\n      .select();\n\n    const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);\n\n    return { entityRefs };\n  }\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] {\n    return lodash.uniqBy(\n      rows,\n      r => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`,\n    );\n  }\n\n  /**\n   * Add a set of deferred entities for processing.\n   * The entities will be added at the front of the processing queue.\n   */\n  private async addUnprocessedEntities(\n    txOpaque: Transaction,\n    options: {\n      sourceEntityRef: string;\n      entities: DeferredEntity[];\n    },\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n\n    // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards\n    const stateReferences = new Array<string>();\n\n    // Upsert all of the unprocessed entities into the refresh_state table, by\n    // their entity ref.\n    for (const { entity, locationKey } of options.entities) {\n      const entityRef = stringifyEntityRef(entity);\n      const hash = generateStableHash(entity);\n\n      const updated = await updateUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n      });\n      if (updated) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      const inserted = await insertUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n        logger: this.options.logger,\n      });\n      if (inserted) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      // If the row can't be inserted, we have a conflict, but it could be either\n      // because of a conflicting locationKey or a race with another instance, so check\n      // whether the conflicting entity has the same entityRef but a different locationKey\n      const conflictingKey = await checkLocationKeyConflict({\n        tx,\n        entityRef,\n        locationKey,\n      });\n      if (conflictingKey) {\n        this.options.logger.warn(\n          `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n        );\n        if (locationKey) {\n          const eventParams: EventParams<CatalogConflictEventPayload> = {\n            topic: CATALOG_CONFLICTS_TOPIC,\n            eventPayload: {\n              unprocessedEntity: entity,\n              entityRef,\n              newLocationKey: locationKey,\n              existingLocationKey: conflictingKey,\n              lastConflictAt: DateTime.now().toISO()!,\n            },\n          };\n          await this.options.events.publish(eventParams);\n        }\n      }\n    }\n\n    // Lastly, replace refresh state references for the originating entity and any successfully added entities\n    await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n      // Remove all existing references from the originating entity\n      .where({ source_entity_ref: options.sourceEntityRef })\n      // And remove any existing references to entities that we're inserting new references for\n      .orWhereIn('target_entity_ref', stateReferences)\n      .delete();\n    await tx.batchInsert(\n      'refresh_state_references',\n      stateReferences.map(entityRef => ({\n        source_entity_ref: options.sourceEntityRef,\n        target_entity_ref: entityRef,\n      })),\n      BATCH_SIZE,\n    );\n  }\n}\n"],"names":["initDatabaseMetrics","errors","ConflictError","stringifyEntityRef","generateTargetKey","timestampToDateTime","rethrowError","lodash","generateStableHash","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","CATALOG_CONFLICTS_TOPIC","DateTime"],"mappings":";;;;;;;;;;;;;;;;;;AAsDA,MAAM,UAAA,GAAa,EAAA;AAEZ,MAAM,yBAAA,CAAwD;AAAA,EACnE,YACmB,OAAA,EAMjB;AANiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAOjB,IAAAA,2BAAA,CAAoB,QAAQ,QAAQ,CAAA;AAAA,EACtC;AAAA,EAEA,MAAM,qBAAA,CACJ,QAAA,EACA,OAAA,EACwD;AACxD,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM;AAAA,MACJ,EAAA;AAAA,MACA,eAAA;AAAA,MACA,UAAA;AAAA,cACAC,QAAA;AAAA,MACA,SAAA;AAAA,MACA,gBAAA;AAAA,MACA,WAAA;AAAA,MACA;AAAA,KACF,GAAI,OAAA;AACJ,IAAA,MAAM,YAAA,GAAe,EAAA,CAAG,MAAA,CAAO,MAAA,CAAO,MAAA;AACtC,IAAA,MAAM,aAAA,GAAgB,MAAM,EAAA,CAAsB,eAAe,EAC9D,MAAA,CAAO;AAAA,MACN,gBAAA,EAAkB,IAAA,CAAK,SAAA,CAAU,eAAe,CAAA;AAAA,MAChD,WAAA,EAAa,UAAA;AAAA,cACbA,QAAA;AAAA,MACA,YAAA,EAAc;AAAA,KACf,CAAA,CACA,KAAA,CAAM,aAAa,EAAE,CAAA,CACrB,SAAS,CAAA,KAAA,KAAS;AACjB,MAAA,IAAI,CAAC,WAAA,EAAa;AAChB,QAAA,OAAO,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA,MACvC;AACA,MAAA,OAAO,MACJ,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,IAC/B,CAAC,CAAA;AACH,IAAA,IAAI,kBAAkB,CAAA,EAAG;AACvB,MAAA,MAAM,IAAIC,oBAAA;AAAA,QACR,CAAA,2CAAA,EAA8C,EAAE,CAAA,oBAAA,EAAuB,WAAW,CAAA,CAAA;AAAA,OACpF;AAAA,IACF;AACA,IAAA,MAAM,eAAA,GAAkBC,gCAAmB,eAAe,CAAA;AAG1D,IAAA,MAAM,IAAA,CAAK,uBAAuB,EAAA,EAAI;AAAA,MACpC,QAAA,EAAU,gBAAA;AAAA,MACV;AAAA,KACD,CAAA;AAID,IAAA,IAAI,oBAAA;AACJ,IAAA,IAAI,aAAa,QAAA,CAAS,SAAS,KAAK,YAAA,CAAa,QAAA,CAAS,OAAO,CAAA,EAAG;AACtE,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,MAAA,CAAO,GAAG,CAAA,CACV,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA;AACtC,MAAA,MAAM,EAAA,CAAmB,WAAW,CAAA,CACjC,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO;AAAA,IACZ,CAAA,MAAO;AACL,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO,CACP,UAAU,GAAG,CAAA;AAAA,IAClB;AAGA,IAAA,MAAM,eAAiC,SAAA,CAAU,GAAA;AAAA,MAC/C,CAAC,EAAE,MAAA,EAAQ,MAAA,EAAQ,MAAK,MAAO;AAAA,QAC7B,qBAAA,EAAuB,EAAA;AAAA,QACvB,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C;AAAA,OACF;AAAA,KACF;AAEA,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,WAAA;AAAA,MACA,IAAA,CAAK,qBAAqB,YAAY,CAAA;AAAA,MACtC;AAAA,KACF;AAGA,IAAA,MAAM,EAAA,CAAqB,cAAc,CAAA,CACtC,KAAA,CAAM,EAAE,SAAA,EAAW,EAAA,EAAI,CAAA,CACvB,MAAA,EAAO;AAGV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,cAAA;AAAA,MACA,WAAA,CAAY,IAAI,CAAA,CAAA,MAAM;AAAA,QACpB,SAAA,EAAW,EAAA;AAAA,QACX,GAAA,EAAKC,sBAAA,CAAkB,CAAA,CAAE,GAAG;AAAA,OAC9B,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAEA,IAAA,OAAO;AAAA,MACL,QAAA,EAAU;AAAA,QACR,SAAA,EAAW;AAAA;AACb,KACF;AAAA,EACF;AAAA,EAEA,MAAM,2BAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,MAAA,EAAQ,UAAA,EAAW,GAAI,OAAA;AAEnC,IAAA,MAAM,EAAA,CAAsB,eAAe,CAAA,CACxC,MAAA,CAAO;AAAA,MACN,MAAA;AAAA,MACA,WAAA,EAAa;AAAA,KACd,CAAA,CACA,KAAA,CAAM,WAAA,EAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,iBAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,KAAA,EAAM,GAAI,OAAA;AAEtB,IAAA,MAAM,GAAsB,eAAe,CAAA,CACxC,MAAA,CAAO,EAAE,OAAO,IAAA,CAAK,SAAA,CAAU,KAAA,IAAS,EAAE,CAAA,EAAG,CAAA,CAC7C,KAAA,CAAM,aAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,sBAAA,CACJ,OAAA,EACA,OAAA,EACuC;AACvC,IAAA,MAAM,IAAA,GAAO,OAAA;AAEb,IAAA,IAAI,UAAA,GAAa,IAAA,CAAwB,eAAe,CAAA,CAAE,MAAA,CAAO;AAAA,MAC/D,WAAA;AAAA,MACA,YAAA;AAAA,MACA,oBAAA;AAAA,MACA,aAAA;AAAA,MACA,OAAA;AAAA,MACA,QAAA;AAAA,MACA,cAAA;AAAA,MACA;AAAA,KACD,CAAA;AAKD,IAAA,IAAI,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,SAAS,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,MAAM,CAAA,EAAG;AACjE,MAAA,UAAA,GAAa,UAAA,CAAW,SAAA,EAAU,CAAE,UAAA,EAAW;AAAA,IACjD;AAEA,IAAA,MAAM,QAAQ,MAAM,UAAA,CACjB,KAAA,CAAM,gBAAA,EAAkB,MAAM,IAAA,CAAK,EAAA,CAAG,GAAA,EAAK,EAC3C,KAAA,CAAM,OAAA,CAAQ,gBAAgB,CAAA,CAC9B,OAAA,CAAQ,kBAAkB,KAAK,CAAA;AAElC,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,OAAA,CAAQ,eAAA,EAAgB;AAE9C,IAAA,MAAM,YAAA,GAAe,CAAC,eAAA,KAA4B;AAChD,MAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AACjD,QAAA,OAAO,KAAK,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,eAAe,UAAU,CAAC,CAAA;AAAA,MACtE,WAAW,IAAA,CAAK,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACtD,QAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,iBAAA,EAAoB,eAAe,CAAA,OAAA,CAAS,CAAA;AAAA,MAC9D;AACA,MAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,kBAAA,EAAqB,eAAe,CAAA,SAAA,CAAW,CAAA;AAAA,IACjE,CAAA;AAEA,IAAA,MAAM,IAAA,CAAwB,eAAe,CAAA,CAC1C,OAAA;AAAA,MACC,YAAA;AAAA,MACA,KAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,MAE5B,MAAA,CAAO;AAAA,MACN,cAAA,EAAgB,aAAa,QAAQ;AAAA,KACtC,CAAA;AAEH,IAAA,OAAO;AAAA,MACL,OAAO,KAAA,CAAM,GAAA;AAAA,QACX,CAAA,CAAA,MACG;AAAA,UACC,IAAI,CAAA,CAAE,SAAA;AAAA,UACN,WAAW,CAAA,CAAE,UAAA;AAAA,UACb,iBAAA,EAAmB,IAAA,CAAK,KAAA,CAAM,CAAA,CAAE,kBAAkB,CAAA;AAAA,UAClD,UAAA,EAAY,EAAE,WAAA,IAAe,EAAA;AAAA,UAC7B,YAAA,EAAcC,8BAAA,CAAoB,CAAA,CAAE,cAAc,CAAA;AAAA,UAClD,OAAO,CAAA,CAAE,KAAA,GAAQ,KAAK,KAAA,CAAM,CAAA,CAAE,KAAK,CAAA,GAAI,MAAA;AAAA,UACvC,QAAQ,CAAA,CAAE,MAAA;AAAA,UACV,aAAa,CAAA,CAAE;AAAA,SACjB;AAAA;AACJ,KACF;AAAA,EACF;AAAA,EAEA,MAAM,WAAA,CACJ,QAAA,EACA,OAAA,EAC4B;AAC5B,IAAA,MAAM,EAAA,GAAK,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,MAEC,OAAA,CAAQ,mBAAA,EAAqB,OAAA,CAAQ,UAAU,EAC/C,MAAA,EAAO;AAEV,IAAA,MAAM,UAAA,GAAa,KAAK,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,iBAAkB,CAAA,CAAE,OAAO,OAAO,CAAA;AAErE,IAAA,OAAO,EAAE,UAAA,EAAW;AAAA,EACtB;AAAA,EAEA,MAAM,YAAe,EAAA,EAAiD;AACpE,IAAA,IAAI;AACF,MAAA,IAAI,MAAA,GAAwB,KAAA,CAAA;AAE5B,MAAA,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,WAAA;AAAA,QAC1B,OAAM,EAAA,KAAM;AAGV,UAAA,MAAA,GAAS,MAAM,GAAG,EAAE,CAAA;AAAA,QACtB,CAAA;AAAA,QACA;AAAA;AAAA,UAEE,qBAAA,EAAuB;AAAA;AACzB,OACF;AAEA,MAAA,OAAO,MAAA;AAAA,IACT,SAAS,CAAA,EAAG;AACV,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,0BAAA,EAA6B,CAAC,CAAA,CAAE,CAAA;AAC1D,MAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA,IACtB;AAAA,EACF;AAAA,EAEQ,qBAAqB,IAAA,EAA0C;AACrE,IAAA,OAAOC,uBAAA,CAAO,MAAA;AAAA,MACZ,IAAA;AAAA,MACA,CAAA,CAAA,KAAK,GAAG,CAAA,CAAE,iBAAiB,IAAI,CAAA,CAAE,iBAAiB,CAAA,CAAA,EAAI,CAAA,CAAE,IAAI,CAAA;AAAA,KAC9D;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,sBAAA,CACZ,QAAA,EACA,OAAA,EAIe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AAGX,IAAA,MAAM,eAAA,GAAkB,IAAI,KAAA,EAAc;AAI1C,IAAA,KAAA,MAAW,EAAE,MAAA,EAAQ,WAAA,EAAY,IAAK,QAAQ,QAAA,EAAU;AACtD,MAAA,MAAM,SAAA,GAAYJ,gCAAmB,MAAM,CAAA;AAC3C,MAAA,MAAM,IAAA,GAAOK,wBAAmB,MAAM,CAAA;AAEtC,MAAA,MAAM,OAAA,GAAU,MAAMC,+CAAA,CAAwB;AAAA,QAC5C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,OAAA,EAAS;AACX,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,QAAA,GAAW,MAAMC,+CAAA,CAAwB;AAAA,QAC7C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA,WAAA;AAAA,QACA,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAKA,MAAA,MAAM,cAAA,GAAiB,MAAMC,iDAAA,CAAyB;AAAA,QACpD,EAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,cAAA,EAAgB;AAClB,QAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,UAClB,CAAA,+BAAA,EAAkC,SAAS,CAAA,uBAAA,EAA0B,cAAc,iBAAiB,WAAW,CAAA;AAAA,SACjH;AACA,QAAA,IAAI,WAAA,EAAa;AACf,UAAA,MAAM,WAAA,GAAwD;AAAA,YAC5D,KAAA,EAAOC,iCAAA;AAAA,YACP,YAAA,EAAc;AAAA,cACZ,iBAAA,EAAmB,MAAA;AAAA,cACnB,SAAA;AAAA,cACA,cAAA,EAAgB,WAAA;AAAA,cAChB,mBAAA,EAAqB,cAAA;AAAA,cACrB,cAAA,EAAgBC,cAAA,CAAS,GAAA,EAAI,CAAE,KAAA;AAAM;AACvC,WACF;AACA,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,OAAA,CAAQ,WAAW,CAAA;AAAA,QAC/C;AAAA,MACF;AAAA,IACF;AAGA,IAAA,MAAM,EAAA,CAAgC,0BAA0B,CAAA,CAE7D,KAAA,CAAM,EAAE,iBAAA,EAAmB,OAAA,CAAQ,eAAA,EAAiB,CAAA,CAEpD,SAAA,CAAU,mBAAA,EAAqB,eAAe,EAC9C,MAAA,EAAO;AACV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,0BAAA;AAAA,MACA,eAAA,CAAgB,IAAI,CAAA,SAAA,MAAc;AAAA,QAChC,mBAAmB,OAAA,CAAQ,eAAA;AAAA,QAC3B,iBAAA,EAAmB;AAAA,OACrB,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAAA,EACF;AACF;;;;"}
+{"version":3,"file":"DefaultProcessingDatabase.cjs.js","sources":["../../src/database/DefaultProcessingDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError } from '@backstage/errors';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { ProcessingIntervalFunction } from '../processing/refresh';\nimport { rethrowError, timestampToDateTime } from './conversion';\nimport { initDatabaseMetrics } from './metrics';\nimport {\n  DbRefreshKeysRow,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n} from './tables';\nimport {\n  GetProcessableEntitiesResult,\n  ListParentsOptions,\n  ListParentsResult,\n  ProcessingDatabase,\n  RefreshStateItem,\n  Transaction,\n  UpdateEntityCacheOptions,\n  UpdateProcessedEntityOptions,\n} from './types';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { generateStableHash, generateTargetKey } from './util';\nimport {\n  EventBroker,\n  EventParams,\n  EventsService,\n} from '@backstage/plugin-events-node';\nimport { DateTime } from 'luxon';\nimport { CATALOG_CONFLICTS_TOPIC } from '../constants';\nimport { CatalogConflictEventPayload } from '../catalog/types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProcessingDatabase implements ProcessingDatabase {\n  constructor(\n    private readonly options: {\n      database: Knex;\n      logger: LoggerService;\n      refreshInterval: ProcessingIntervalFunction;\n      eventBroker?: EventBroker | EventsService;\n    },\n  ) {\n    initDatabaseMetrics(options.database);\n  }\n\n  async updateProcessedEntity(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<{ previous: { relations: DbRelationsRow[] } }> {\n    const tx = txOpaque as Knex.Transaction;\n    const {\n      id,\n      processedEntity,\n      resultHash,\n      errors,\n      relations,\n      deferredEntities,\n      refreshKeys,\n      locationKey,\n    } = options;\n    const configClient = tx.client.config.client;\n    const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        processed_entity: JSON.stringify(processedEntity),\n        result_hash: resultHash,\n        errors,\n        location_key: locationKey,\n      })\n      .where('entity_id', id)\n      .andWhere(inner => {\n        if (!locationKey) {\n          return inner.whereNull('location_key');\n        }\n        return inner\n          .where('location_key', locationKey)\n          .orWhereNull('location_key');\n      });\n    if (refreshResult === 0) {\n      throw new ConflictError(\n        `Conflicting write of processing result for ${id} with location key '${locationKey}'`,\n      );\n    }\n    const sourceEntityRef = stringifyEntityRef(processedEntity);\n\n    // Schedule all deferred entities for future processing.\n    await this.addUnprocessedEntities(tx, {\n      entities: deferredEntities,\n      sourceEntityRef,\n    });\n\n    // Delete old relations\n    // NOTE(freben): knex implemented support for returning() on update queries for sqlite, but at the current time of writing (Sep 2022) not for delete() queries.\n    let previousRelationRows: DbRelationsRow[];\n    if (configClient.includes('sqlite3') || configClient.includes('mysql')) {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .select('*')\n        .where({ originating_entity_id: id });\n      await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete();\n    } else {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete()\n        .returning('*');\n    }\n\n    // Batch insert new relations\n    const relationRows: DbRelationsRow[] = relations.map(\n      ({ source, target, type }) => ({\n        originating_entity_id: id,\n        source_entity_ref: stringifyEntityRef(source),\n        target_entity_ref: stringifyEntityRef(target),\n        type,\n      }),\n    );\n\n    await tx.batchInsert(\n      'relations',\n      this.deduplicateRelations(relationRows),\n      BATCH_SIZE,\n    );\n\n    // Delete old refresh keys\n    await tx<DbRefreshKeysRow>('refresh_keys')\n      .where({ entity_id: id })\n      .delete();\n\n    // Insert the refresh keys for the processed entity\n    await tx.batchInsert(\n      'refresh_keys',\n      refreshKeys.map(k => ({\n        entity_id: id,\n        key: generateTargetKey(k.key),\n      })),\n      BATCH_SIZE,\n    );\n\n    return {\n      previous: {\n        relations: previousRelationRows,\n      },\n    };\n  }\n\n  async updateProcessedEntityErrors(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, errors, resultHash } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        errors,\n        result_hash: resultHash,\n      })\n      .where('entity_id', id);\n  }\n\n  async updateEntityCache(\n    txOpaque: Transaction,\n    options: UpdateEntityCacheOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, state } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({ cache: JSON.stringify(state ?? {}) })\n      .where('entity_id', id);\n  }\n\n  async getProcessableEntities(\n    maybeTx: Transaction | Knex,\n    request: { processBatchSize: number },\n  ): Promise<GetProcessableEntitiesResult> {\n    const knex = maybeTx as Knex.Transaction | Knex;\n\n    let itemsQuery = knex<DbRefreshStateRow>('refresh_state').select([\n      'entity_id',\n      'entity_ref',\n      'unprocessed_entity',\n      'result_hash',\n      'cache',\n      'errors',\n      'location_key',\n      'next_update_at',\n    ]);\n\n    // This avoids duplication of work because of race conditions and is\n    // also fast because locked rows are ignored rather than blocking.\n    // It's only available in MySQL and PostgreSQL\n    if (['mysql', 'mysql2', 'pg'].includes(knex.client.config.client)) {\n      itemsQuery = itemsQuery.forUpdate().skipLocked();\n    }\n\n    const items = await itemsQuery\n      .where('next_update_at', '<=', knex.fn.now())\n      .limit(request.processBatchSize)\n      .orderBy('next_update_at', 'asc');\n\n    const interval = this.options.refreshInterval();\n\n    const nextUpdateAt = (refreshInterval: number) => {\n      if (knex.client.config.client.includes('sqlite3')) {\n        return knex.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);\n      } else if (knex.client.config.client.includes('mysql')) {\n        return knex.raw(`now() + interval ${refreshInterval} second`);\n      }\n      return knex.raw(`now() + interval '${refreshInterval} seconds'`);\n    };\n\n    await knex<DbRefreshStateRow>('refresh_state')\n      .whereIn(\n        'entity_ref',\n        items.map(i => i.entity_ref),\n      )\n      .update({\n        next_update_at: nextUpdateAt(interval),\n      });\n\n    return {\n      items: items.map(\n        i =>\n          ({\n            id: i.entity_id,\n            entityRef: i.entity_ref,\n            unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity,\n            resultHash: i.result_hash || '',\n            nextUpdateAt: timestampToDateTime(i.next_update_at),\n            state: i.cache ? JSON.parse(i.cache) : undefined,\n            errors: i.errors,\n            locationKey: i.location_key,\n          } satisfies RefreshStateItem),\n      ),\n    };\n  }\n\n  async listParents(\n    txOpaque: Transaction,\n    options: ListParentsOptions,\n  ): Promise<ListParentsResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .whereIn('target_entity_ref', options.entityRefs)\n      .select();\n\n    const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);\n\n    return { entityRefs };\n  }\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] {\n    return lodash.uniqBy(\n      rows,\n      r => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`,\n    );\n  }\n\n  /**\n   * Add a set of deferred entities for processing.\n   * The entities will be added at the front of the processing queue.\n   */\n  private async addUnprocessedEntities(\n    txOpaque: Transaction,\n    options: {\n      sourceEntityRef: string;\n      entities: DeferredEntity[];\n    },\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n\n    // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards\n    const stateReferences = new Array<string>();\n\n    // Upsert all of the unprocessed entities into the refresh_state table, by\n    // their entity ref.\n    for (const { entity, locationKey } of options.entities) {\n      const entityRef = stringifyEntityRef(entity);\n      const hash = generateStableHash(entity);\n\n      const updated = await updateUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n      });\n      if (updated) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      const inserted = await insertUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n        logger: this.options.logger,\n      });\n      if (inserted) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      // If the row can't be inserted, we have a conflict, but it could be either\n      // because of a conflicting locationKey or a race with another instance, so check\n      // whether the conflicting entity has the same entityRef but a different locationKey\n      const conflictingKey = await checkLocationKeyConflict({\n        tx,\n        entityRef,\n        locationKey,\n      });\n      if (conflictingKey) {\n        this.options.logger.warn(\n          `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n        );\n        if (this.options.eventBroker && locationKey) {\n          const eventParams: EventParams<CatalogConflictEventPayload> = {\n            topic: CATALOG_CONFLICTS_TOPIC,\n            eventPayload: {\n              unprocessedEntity: entity,\n              entityRef,\n              newLocationKey: locationKey,\n              existingLocationKey: conflictingKey,\n              lastConflictAt: DateTime.now().toISO()!,\n            },\n          };\n          await this.options.eventBroker?.publish(eventParams);\n        }\n      }\n    }\n\n    // Lastly, replace refresh state references for the originating entity and any successfully added entities\n    await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n      // Remove all existing references from the originating entity\n      .where({ source_entity_ref: options.sourceEntityRef })\n      // And remove any existing references to entities that we're inserting new references for\n      .orWhereIn('target_entity_ref', stateReferences)\n      .delete();\n    await tx.batchInsert(\n      'refresh_state_references',\n      stateReferences.map(entityRef => ({\n        source_entity_ref: options.sourceEntityRef,\n        target_entity_ref: entityRef,\n      })),\n      BATCH_SIZE,\n    );\n  }\n}\n"],"names":["initDatabaseMetrics","errors","ConflictError","stringifyEntityRef","generateTargetKey","timestampToDateTime","rethrowError","lodash","generateStableHash","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","CATALOG_CONFLICTS_TOPIC","DateTime"],"mappings":";;;;;;;;;;;;;;;;;;AA0DA,MAAM,UAAA,GAAa,EAAA;AAEZ,MAAM,yBAAA,CAAwD;AAAA,EACnE,YACmB,OAAA,EAMjB;AANiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAOjB,IAAAA,2BAAA,CAAoB,QAAQ,QAAQ,CAAA;AAAA,EACtC;AAAA,EAEA,MAAM,qBAAA,CACJ,QAAA,EACA,OAAA,EACwD;AACxD,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM;AAAA,MACJ,EAAA;AAAA,MACA,eAAA;AAAA,MACA,UAAA;AAAA,cACAC,QAAA;AAAA,MACA,SAAA;AAAA,MACA,gBAAA;AAAA,MACA,WAAA;AAAA,MACA;AAAA,KACF,GAAI,OAAA;AACJ,IAAA,MAAM,YAAA,GAAe,EAAA,CAAG,MAAA,CAAO,MAAA,CAAO,MAAA;AACtC,IAAA,MAAM,aAAA,GAAgB,MAAM,EAAA,CAAsB,eAAe,EAC9D,MAAA,CAAO;AAAA,MACN,gBAAA,EAAkB,IAAA,CAAK,SAAA,CAAU,eAAe,CAAA;AAAA,MAChD,WAAA,EAAa,UAAA;AAAA,cACbA,QAAA;AAAA,MACA,YAAA,EAAc;AAAA,KACf,CAAA,CACA,KAAA,CAAM,aAAa,EAAE,CAAA,CACrB,SAAS,CAAA,KAAA,KAAS;AACjB,MAAA,IAAI,CAAC,WAAA,EAAa;AAChB,QAAA,OAAO,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA,MACvC;AACA,MAAA,OAAO,MACJ,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,IAC/B,CAAC,CAAA;AACH,IAAA,IAAI,kBAAkB,CAAA,EAAG;AACvB,MAAA,MAAM,IAAIC,oBAAA;AAAA,QACR,CAAA,2CAAA,EAA8C,EAAE,CAAA,oBAAA,EAAuB,WAAW,CAAA,CAAA;AAAA,OACpF;AAAA,IACF;AACA,IAAA,MAAM,eAAA,GAAkBC,gCAAmB,eAAe,CAAA;AAG1D,IAAA,MAAM,IAAA,CAAK,uBAAuB,EAAA,EAAI;AAAA,MACpC,QAAA,EAAU,gBAAA;AAAA,MACV;AAAA,KACD,CAAA;AAID,IAAA,IAAI,oBAAA;AACJ,IAAA,IAAI,aAAa,QAAA,CAAS,SAAS,KAAK,YAAA,CAAa,QAAA,CAAS,OAAO,CAAA,EAAG;AACtE,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,MAAA,CAAO,GAAG,CAAA,CACV,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA;AACtC,MAAA,MAAM,EAAA,CAAmB,WAAW,CAAA,CACjC,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO;AAAA,IACZ,CAAA,MAAO;AACL,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO,CACP,UAAU,GAAG,CAAA;AAAA,IAClB;AAGA,IAAA,MAAM,eAAiC,SAAA,CAAU,GAAA;AAAA,MAC/C,CAAC,EAAE,MAAA,EAAQ,MAAA,EAAQ,MAAK,MAAO;AAAA,QAC7B,qBAAA,EAAuB,EAAA;AAAA,QACvB,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C;AAAA,OACF;AAAA,KACF;AAEA,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,WAAA;AAAA,MACA,IAAA,CAAK,qBAAqB,YAAY,CAAA;AAAA,MACtC;AAAA,KACF;AAGA,IAAA,MAAM,EAAA,CAAqB,cAAc,CAAA,CACtC,KAAA,CAAM,EAAE,SAAA,EAAW,EAAA,EAAI,CAAA,CACvB,MAAA,EAAO;AAGV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,cAAA;AAAA,MACA,WAAA,CAAY,IAAI,CAAA,CAAA,MAAM;AAAA,QACpB,SAAA,EAAW,EAAA;AAAA,QACX,GAAA,EAAKC,sBAAA,CAAkB,CAAA,CAAE,GAAG;AAAA,OAC9B,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAEA,IAAA,OAAO;AAAA,MACL,QAAA,EAAU;AAAA,QACR,SAAA,EAAW;AAAA;AACb,KACF;AAAA,EACF;AAAA,EAEA,MAAM,2BAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,MAAA,EAAQ,UAAA,EAAW,GAAI,OAAA;AAEnC,IAAA,MAAM,EAAA,CAAsB,eAAe,CAAA,CACxC,MAAA,CAAO;AAAA,MACN,MAAA;AAAA,MACA,WAAA,EAAa;AAAA,KACd,CAAA,CACA,KAAA,CAAM,WAAA,EAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,iBAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,KAAA,EAAM,GAAI,OAAA;AAEtB,IAAA,MAAM,GAAsB,eAAe,CAAA,CACxC,MAAA,CAAO,EAAE,OAAO,IAAA,CAAK,SAAA,CAAU,KAAA,IAAS,EAAE,CAAA,EAAG,CAAA,CAC7C,KAAA,CAAM,aAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,sBAAA,CACJ,OAAA,EACA,OAAA,EACuC;AACvC,IAAA,MAAM,IAAA,GAAO,OAAA;AAEb,IAAA,IAAI,UAAA,GAAa,IAAA,CAAwB,eAAe,CAAA,CAAE,MAAA,CAAO;AAAA,MAC/D,WAAA;AAAA,MACA,YAAA;AAAA,MACA,oBAAA;AAAA,MACA,aAAA;AAAA,MACA,OAAA;AAAA,MACA,QAAA;AAAA,MACA,cAAA;AAAA,MACA;AAAA,KACD,CAAA;AAKD,IAAA,IAAI,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,SAAS,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,MAAM,CAAA,EAAG;AACjE,MAAA,UAAA,GAAa,UAAA,CAAW,SAAA,EAAU,CAAE,UAAA,EAAW;AAAA,IACjD;AAEA,IAAA,MAAM,QAAQ,MAAM,UAAA,CACjB,KAAA,CAAM,gBAAA,EAAkB,MAAM,IAAA,CAAK,EAAA,CAAG,GAAA,EAAK,EAC3C,KAAA,CAAM,OAAA,CAAQ,gBAAgB,CAAA,CAC9B,OAAA,CAAQ,kBAAkB,KAAK,CAAA;AAElC,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,OAAA,CAAQ,eAAA,EAAgB;AAE9C,IAAA,MAAM,YAAA,GAAe,CAAC,eAAA,KAA4B;AAChD,MAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AACjD,QAAA,OAAO,KAAK,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,eAAe,UAAU,CAAC,CAAA;AAAA,MACtE,WAAW,IAAA,CAAK,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACtD,QAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,iBAAA,EAAoB,eAAe,CAAA,OAAA,CAAS,CAAA;AAAA,MAC9D;AACA,MAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,kBAAA,EAAqB,eAAe,CAAA,SAAA,CAAW,CAAA;AAAA,IACjE,CAAA;AAEA,IAAA,MAAM,IAAA,CAAwB,eAAe,CAAA,CAC1C,OAAA;AAAA,MACC,YAAA;AAAA,MACA,KAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,MAE5B,MAAA,CAAO;AAAA,MACN,cAAA,EAAgB,aAAa,QAAQ;AAAA,KACtC,CAAA;AAEH,IAAA,OAAO;AAAA,MACL,OAAO,KAAA,CAAM,GAAA;AAAA,QACX,CAAA,CAAA,MACG;AAAA,UACC,IAAI,CAAA,CAAE,SAAA;AAAA,UACN,WAAW,CAAA,CAAE,UAAA;AAAA,UACb,iBAAA,EAAmB,IAAA,CAAK,KAAA,CAAM,CAAA,CAAE,kBAAkB,CAAA;AAAA,UAClD,UAAA,EAAY,EAAE,WAAA,IAAe,EAAA;AAAA,UAC7B,YAAA,EAAcC,8BAAA,CAAoB,CAAA,CAAE,cAAc,CAAA;AAAA,UAClD,OAAO,CAAA,CAAE,KAAA,GAAQ,KAAK,KAAA,CAAM,CAAA,CAAE,KAAK,CAAA,GAAI,MAAA;AAAA,UACvC,QAAQ,CAAA,CAAE,MAAA;AAAA,UACV,aAAa,CAAA,CAAE;AAAA,SACjB;AAAA;AACJ,KACF;AAAA,EACF;AAAA,EAEA,MAAM,WAAA,CACJ,QAAA,EACA,OAAA,EAC4B;AAC5B,IAAA,MAAM,EAAA,GAAK,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,MAEC,OAAA,CAAQ,mBAAA,EAAqB,OAAA,CAAQ,UAAU,EAC/C,MAAA,EAAO;AAEV,IAAA,MAAM,UAAA,GAAa,KAAK,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,iBAAkB,CAAA,CAAE,OAAO,OAAO,CAAA;AAErE,IAAA,OAAO,EAAE,UAAA,EAAW;AAAA,EACtB;AAAA,EAEA,MAAM,YAAe,EAAA,EAAiD;AACpE,IAAA,IAAI;AACF,MAAA,IAAI,MAAA,GAAwB,KAAA,CAAA;AAE5B,MAAA,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,WAAA;AAAA,QAC1B,OAAM,EAAA,KAAM;AAGV,UAAA,MAAA,GAAS,MAAM,GAAG,EAAE,CAAA;AAAA,QACtB,CAAA;AAAA,QACA;AAAA;AAAA,UAEE,qBAAA,EAAuB;AAAA;AACzB,OACF;AAEA,MAAA,OAAO,MAAA;AAAA,IACT,SAAS,CAAA,EAAG;AACV,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,0BAAA,EAA6B,CAAC,CAAA,CAAE,CAAA;AAC1D,MAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA,IACtB;AAAA,EACF;AAAA,EAEQ,qBAAqB,IAAA,EAA0C;AACrE,IAAA,OAAOC,uBAAA,CAAO,MAAA;AAAA,MACZ,IAAA;AAAA,MACA,CAAA,CAAA,KAAK,GAAG,CAAA,CAAE,iBAAiB,IAAI,CAAA,CAAE,iBAAiB,CAAA,CAAA,EAAI,CAAA,CAAE,IAAI,CAAA;AAAA,KAC9D;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,sBAAA,CACZ,QAAA,EACA,OAAA,EAIe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AAGX,IAAA,MAAM,eAAA,GAAkB,IAAI,KAAA,EAAc;AAI1C,IAAA,KAAA,MAAW,EAAE,MAAA,EAAQ,WAAA,EAAY,IAAK,QAAQ,QAAA,EAAU;AACtD,MAAA,MAAM,SAAA,GAAYJ,gCAAmB,MAAM,CAAA;AAC3C,MAAA,MAAM,IAAA,GAAOK,wBAAmB,MAAM,CAAA;AAEtC,MAAA,MAAM,OAAA,GAAU,MAAMC,+CAAA,CAAwB;AAAA,QAC5C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,OAAA,EAAS;AACX,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,QAAA,GAAW,MAAMC,+CAAA,CAAwB;AAAA,QAC7C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA,WAAA;AAAA,QACA,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAKA,MAAA,MAAM,cAAA,GAAiB,MAAMC,iDAAA,CAAyB;AAAA,QACpD,EAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,cAAA,EAAgB;AAClB,QAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,UAClB,CAAA,+BAAA,EAAkC,SAAS,CAAA,uBAAA,EAA0B,cAAc,iBAAiB,WAAW,CAAA;AAAA,SACjH;AACA,QAAA,IAAI,IAAA,CAAK,OAAA,CAAQ,WAAA,IAAe,WAAA,EAAa;AAC3C,UAAA,MAAM,WAAA,GAAwD;AAAA,YAC5D,KAAA,EAAOC,iCAAA;AAAA,YACP,YAAA,EAAc;AAAA,cACZ,iBAAA,EAAmB,MAAA;AAAA,cACnB,SAAA;AAAA,cACA,cAAA,EAAgB,WAAA;AAAA,cAChB,mBAAA,EAAqB,cAAA;AAAA,cACrB,cAAA,EAAgBC,cAAA,CAAS,GAAA,EAAI,CAAE,KAAA;AAAM;AACvC,WACF;AACA,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,WAAA,EAAa,OAAA,CAAQ,WAAW,CAAA;AAAA,QACrD;AAAA,MACF;AAAA,IACF;AAGA,IAAA,MAAM,EAAA,CAAgC,0BAA0B,CAAA,CAE7D,KAAA,CAAM,EAAE,iBAAA,EAAmB,OAAA,CAAQ,eAAA,EAAiB,CAAA,CAEpD,SAAA,CAAU,mBAAA,EAAqB,eAAe,EAC9C,MAAA,EAAO;AACV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,0BAAA;AAAA,MACA,eAAA,CAAgB,IAAI,CAAA,SAAA,MAAc;AAAA,QAChC,mBAAmB,OAAA,CAAQ,eAAA;AAAA,QAC3B,iBAAA,EAAmB;AAAA,OACrB,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAAA,EACF;AACF;;;;"}

package/dist/processing/DefaultCatalogProcessingEngine.cjs.js

@@ -34,7 +34,7 @@ class DefaultCatalogProcessingEngine {
   orphanCleanupIntervalMs;
   onProcessingError;
   tracker;
-  events;
+  eventBroker;
   stopFunc;
   constructor(options) {
     this.config = options.config;
@@ -49,7 +49,7 @@ class DefaultCatalogProcessingEngine {
     this.orphanCleanupIntervalMs = options.orphanCleanupIntervalMs ?? 3e4;
     this.onProcessingError = options.onProcessingError;
     this.tracker = options.tracker ?? progressTracker();
-    this.events = options.events;
+    this.eventBroker = options.eventBroker;
     this.stopFunc = void 0;
   }
   async start() {
@@ -128,7 +128,7 @@ class DefaultCatalogProcessingEngine {
             }
             const location = unprocessedEntity?.metadata?.annotations?.[catalogModel.ANNOTATION_LOCATION];
             if (result.errors.length) {
-              this.events.publish({
+              this.eventBroker?.publish({
                 topic: constants.CATALOG_ERRORS_TOPIC,
                 eventPayload: {
                   entity: entityRef,
@@ -260,16 +260,22 @@ class DefaultCatalogProcessingEngine {
         this.logger.warn(`Failed to delete orphaned entities`, error);
       }
     };
-    const abortController = new AbortController();
-    this.scheduler.scheduleTask({
-      id: "catalog_orphan_cleanup",
-      frequency: { milliseconds: this.orphanCleanupIntervalMs },
-      timeout: { milliseconds: this.orphanCleanupIntervalMs * 0.8 },
-      fn: runOnce,
-      signal: abortController.signal
-    });
+    if (this.scheduler) {
+      const abortController = new AbortController();
+      this.scheduler.scheduleTask({
+        id: "catalog_orphan_cleanup",
+        frequency: { milliseconds: this.orphanCleanupIntervalMs },
+        timeout: { milliseconds: this.orphanCleanupIntervalMs * 0.8 },
+        fn: runOnce,
+        signal: abortController.signal
+      });
+      return () => {
+        abortController.abort();
+      };
+    }
+    const intervalKey = setInterval(runOnce, this.orphanCleanupIntervalMs);
     return () => {
-      abortController.abort();
+      clearInterval(intervalKey);
     };
   }
 }

package/dist/processing/DefaultCatalogProcessingEngine.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultCatalogProcessingEngine.cjs.js","sources":["../../src/processing/DefaultCatalogProcessingEngine.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  ANNOTATION_LOCATION,\n  Entity,\n  stringifyEntityRef,\n} from '@backstage/catalog-model';\nimport { assertError, serializeError, stringifyError } from '@backstage/errors';\nimport { Hash } from 'crypto';\nimport stableStringify from 'fast-json-stable-stringify';\nimport { Knex } from 'knex';\nimport { metrics, trace } from '@opentelemetry/api';\nimport { ProcessingDatabase, RefreshStateItem } from '../database/types';\nimport { createCounterMetric, createSummaryMetric } from '../util/metrics';\nimport { CatalogProcessingOrchestrator, EntityProcessingResult } from './types';\nimport { Stitcher, stitchingStrategyFromConfig } from '../stitching/types';\nimport { startTaskPipeline } from './TaskPipeline';\nimport { Config } from '@backstage/config';\nimport {\n  addEntityAttributes,\n  TRACER_ID,\n  withActiveSpan,\n} from '../util/opentelemetry';\nimport { deleteOrphanedEntities } from '../database/operations/util/deleteOrphanedEntities';\nimport { EventsService } from '@backstage/plugin-events-node';\nimport { CATALOG_ERRORS_TOPIC } from '../constants';\nimport { LoggerService, SchedulerService } from '@backstage/backend-plugin-api';\n\nconst CACHE_TTL = 5;\n\nconst tracer = trace.getTracer(TRACER_ID);\n\nexport type ProgressTracker = ReturnType<typeof progressTracker>;\n\nconst stableStringifyArray = (arr: any[]) => {\n  const sorted = arr.map(stableStringify).sort();\n  return `[${sorted.join(',')}]`;\n};\n\n// NOTE(freben): Perhaps surprisingly, this class does not implement the\n// CatalogProcessingEngine type. That type is externally visible and its name is\n// the way it is for historic reasons. This class has no particular reason to\n// implement that precise interface; nowadays there are several different\n// engines \"hiding\" behind the CatalogProcessingEngine interface, of which this\n// is just one.\nexport class DefaultCatalogProcessingEngine {\n  private readonly config: Config;\n  private readonly scheduler: SchedulerService;\n  private readonly logger: LoggerService;\n  private readonly knex: Knex;\n  private readonly processingDatabase: ProcessingDatabase;\n  private readonly orchestrator: CatalogProcessingOrchestrator;\n  private readonly stitcher: Stitcher;\n  private readonly createHash: () => Hash;\n  private readonly pollingIntervalMs: number;\n  private readonly orphanCleanupIntervalMs: number;\n  private readonly onProcessingError?: (event: {\n    unprocessedEntity: Entity;\n    errors: Error[];\n  }) => Promise<void> | void;\n  private readonly tracker: ProgressTracker;\n  private readonly events: EventsService;\n\n  private stopFunc?: () => void;\n\n  constructor(options: {\n    config: Config;\n    scheduler: SchedulerService;\n    logger: LoggerService;\n    knex: Knex;\n    processingDatabase: ProcessingDatabase;\n    orchestrator: CatalogProcessingOrchestrator;\n    stitcher: Stitcher;\n    createHash: () => Hash;\n    pollingIntervalMs?: number;\n    orphanCleanupIntervalMs?: number;\n    onProcessingError?: (event: {\n      unprocessedEntity: Entity;\n      errors: Error[];\n    }) => Promise<void> | void;\n    tracker?: ProgressTracker;\n    events: EventsService;\n  }) {\n    this.config = options.config;\n    this.scheduler = options.scheduler;\n    this.logger = options.logger;\n    this.knex = options.knex;\n    this.processingDatabase = options.processingDatabase;\n    this.orchestrator = options.orchestrator;\n    this.stitcher = options.stitcher;\n    this.createHash = options.createHash;\n    this.pollingIntervalMs = options.pollingIntervalMs ?? 1_000;\n    this.orphanCleanupIntervalMs = options.orphanCleanupIntervalMs ?? 30_000;\n    this.onProcessingError = options.onProcessingError;\n    this.tracker = options.tracker ?? progressTracker();\n    this.events = options.events;\n\n    this.stopFunc = undefined;\n  }\n\n  async start() {\n    if (this.stopFunc) {\n      throw new Error('Processing engine is already started');\n    }\n\n    const stopPipeline = this.startPipeline();\n    const stopCleanup = this.startOrphanCleanup();\n\n    this.stopFunc = () => {\n      stopPipeline();\n      stopCleanup();\n    };\n  }\n\n  async stop() {\n    if (this.stopFunc) {\n      this.stopFunc();\n      this.stopFunc = undefined;\n    }\n  }\n\n  private startPipeline(): () => void {\n    return startTaskPipeline<RefreshStateItem>({\n      lowWatermark: 5,\n      highWatermark: 10,\n      pollingIntervalMs: this.pollingIntervalMs,\n      loadTasks: async count => {\n        try {\n          const { items } =\n            await this.processingDatabase.getProcessableEntities(this.knex, {\n              processBatchSize: count,\n            });\n          return items;\n        } catch (error) {\n          this.logger.warn('Failed to load processing items', error);\n          return [];\n        }\n      },\n      processTask: async item => {\n        await withActiveSpan(tracer, 'ProcessingRun', async span => {\n          const track = this.tracker.processStart(item, this.logger);\n          addEntityAttributes(span, item.unprocessedEntity);\n\n          try {\n            const {\n              id,\n              state,\n              unprocessedEntity,\n              entityRef,\n              locationKey,\n              resultHash: previousResultHash,\n            } = item;\n            const result = await this.orchestrator.process({\n              entity: unprocessedEntity,\n              state,\n            });\n\n            track.markProcessorsCompleted(result);\n\n            if (result.ok) {\n              const { ttl: _, ...stateWithoutTtl } = state ?? {};\n              if (\n                stableStringify(stateWithoutTtl) !==\n                stableStringify(result.state)\n              ) {\n                await this.processingDatabase.transaction(async tx => {\n                  await this.processingDatabase.updateEntityCache(tx, {\n                    id,\n                    state: {\n                      ttl: CACHE_TTL,\n                      ...result.state,\n                    },\n                  });\n                });\n              }\n            } else {\n              const maybeTtl = state?.ttl;\n              const ttl = Number.isInteger(maybeTtl) ? (maybeTtl as number) : 0;\n              await this.processingDatabase.transaction(async tx => {\n                await this.processingDatabase.updateEntityCache(tx, {\n                  id,\n                  state: ttl > 0 ? { ...state, ttl: ttl - 1 } : {},\n                });\n              });\n            }\n\n            const location =\n              unprocessedEntity?.metadata?.annotations?.[ANNOTATION_LOCATION];\n            if (result.errors.length) {\n              this.events.publish({\n                topic: CATALOG_ERRORS_TOPIC,\n                eventPayload: {\n                  entity: entityRef,\n                  location,\n                  errors: result.errors,\n                },\n              });\n            }\n            const errorsString = JSON.stringify(\n              result.errors.map(e => serializeError(e)),\n            );\n\n            let hashBuilder = this.createHash().update(errorsString);\n\n            if (result.ok) {\n              const { entityRefs: parents } =\n                await this.processingDatabase.transaction(tx =>\n                  this.processingDatabase.listParents(tx, {\n                    entityRefs: [\n                      entityRef,\n                      ...result.deferredEntities.map(e =>\n                        stringifyEntityRef(e.entity),\n                      ),\n                    ],\n                  }),\n                );\n\n              hashBuilder = hashBuilder\n                .update(stableStringify({ ...result.completedEntity }))\n                .update(stableStringifyArray([...result.deferredEntities]))\n                .update(stableStringifyArray([...result.relations]))\n                .update(stableStringifyArray([...result.refreshKeys]))\n                .update(stableStringifyArray([...parents]));\n            }\n\n            const resultHash = hashBuilder.digest('hex');\n            if (resultHash === previousResultHash) {\n              // If nothing changed in our produced outputs, we cannot have any\n              // significant effect on our surroundings; therefore, we just abort\n              // without any updates / stitching.\n              track.markSuccessfulWithNoChanges();\n              return;\n            }\n\n            // If the result was marked as not OK, it signals that some part of the\n            // processing pipeline threw an exception. This can happen both as part of\n            // non-catastrophic things such as due to validation errors, as well as if\n            // something fatal happens inside the processing for other reasons. In any\n            // case, this means we can't trust that anything in the output is okay. So\n            // just store the errors and trigger a stich so that they become visible to\n            // the outside.\n            if (!result.ok) {\n              // notify the error listener if the entity can not be processed.\n              Promise.resolve(undefined)\n                .then(() =>\n                  this.onProcessingError?.({\n                    unprocessedEntity,\n                    errors: result.errors,\n                  }),\n                )\n                .catch(error => {\n                  this.logger.debug(\n                    `Processing error listener threw an exception, ${stringifyError(\n                      error,\n                    )}`,\n                  );\n                });\n\n              await this.processingDatabase.transaction(async tx => {\n                await this.processingDatabase.updateProcessedEntityErrors(tx, {\n                  id,\n                  errors: errorsString,\n                  resultHash,\n                });\n              });\n\n              await this.stitcher.stitch({\n                entityRefs: [stringifyEntityRef(unprocessedEntity)],\n              });\n\n              track.markSuccessfulWithErrors();\n              return;\n            }\n\n            result.completedEntity.metadata.uid = id;\n            let oldRelationSources: Map<string, string>;\n            await this.processingDatabase.transaction(async tx => {\n              const { previous } =\n                await this.processingDatabase.updateProcessedEntity(tx, {\n                  id,\n                  processedEntity: result.completedEntity,\n                  resultHash,\n                  errors: errorsString,\n                  relations: result.relations,\n                  deferredEntities: result.deferredEntities,\n                  locationKey,\n                  refreshKeys: result.refreshKeys,\n                });\n              oldRelationSources = new Map(\n                previous.relations.map(r => [\n                  `${r.source_entity_ref}:${r.type}->${r.target_entity_ref}`,\n                  r.source_entity_ref,\n                ]),\n              );\n            });\n\n            const newRelationSources = new Map<string, string>(\n              result.relations.map(relation => {\n                const sourceEntityRef = stringifyEntityRef(relation.source);\n                const targetEntityRef = stringifyEntityRef(relation.target);\n                return [\n                  `${sourceEntityRef}:${relation.type}->${targetEntityRef}`,\n                  sourceEntityRef,\n                ];\n              }),\n            );\n\n            const setOfThingsToStitch = new Set<string>([\n              stringifyEntityRef(result.completedEntity),\n            ]);\n            newRelationSources.forEach((sourceEntityRef, uniqueKey) => {\n              if (!oldRelationSources.has(uniqueKey)) {\n                setOfThingsToStitch.add(sourceEntityRef);\n              }\n            });\n            oldRelationSources!.forEach((sourceEntityRef, uniqueKey) => {\n              if (!newRelationSources.has(uniqueKey)) {\n                setOfThingsToStitch.add(sourceEntityRef);\n              }\n            });\n\n            await this.stitcher.stitch({\n              entityRefs: setOfThingsToStitch,\n            });\n\n            track.markSuccessfulWithChanges();\n          } catch (error) {\n            assertError(error);\n            track.markFailed(error);\n          }\n        });\n      },\n    });\n  }\n\n  private startOrphanCleanup(): () => void {\n    const orphanStrategy =\n      this.config.getOptionalString('catalog.orphanStrategy') ?? 'delete';\n    if (orphanStrategy !== 'delete') {\n      return () => {};\n    }\n\n    const stitchingStrategy = stitchingStrategyFromConfig(this.config);\n\n    const runOnce = async () => {\n      try {\n        const n = await deleteOrphanedEntities({\n          knex: this.knex,\n          strategy: stitchingStrategy,\n        });\n        if (n > 0) {\n          this.logger.info(`Deleted ${n} orphaned entities`);\n        }\n      } catch (error) {\n        this.logger.warn(`Failed to delete orphaned entities`, error);\n      }\n    };\n\n    const abortController = new AbortController();\n    this.scheduler.scheduleTask({\n      id: 'catalog_orphan_cleanup',\n      frequency: { milliseconds: this.orphanCleanupIntervalMs },\n      timeout: { milliseconds: this.orphanCleanupIntervalMs * 0.8 },\n      fn: runOnce,\n      signal: abortController.signal,\n    });\n\n    return () => {\n      abortController.abort();\n    };\n  }\n}\n\n// Helps wrap the timing and logging behaviors\nfunction progressTracker() {\n  // prom-client metrics are deprecated in favour of OpenTelemetry metrics.\n  const promProcessedEntities = createCounterMetric({\n    name: 'catalog_processed_entities_count',\n    help: 'Amount of entities processed, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingDuration = createSummaryMetric({\n    name: 'catalog_processing_duration_seconds',\n    help: 'Time spent executing the full processing flow, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessorsDuration = createSummaryMetric({\n    name: 'catalog_processors_duration_seconds',\n    help: 'Time spent executing catalog processors, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingQueueDelay = createSummaryMetric({\n    name: 'catalog_processing_queue_delay_seconds',\n    help: 'The amount of delay between being scheduled for processing, and the start of actually being processed, DEPRECATED, use OpenTelemetry metrics instead',\n  });\n\n  const meter = metrics.getMeter('default');\n  const processedEntities = meter.createCounter(\n    'catalog.processed.entities.count',\n    { description: 'Amount of entities processed' },\n  );\n\n  const processingDuration = meter.createHistogram(\n    'catalog.processing.duration',\n    {\n      description: 'Time spent executing the full processing flow',\n      unit: 'seconds',\n    },\n  );\n\n  const processorsDuration = meter.createHistogram(\n    'catalog.processors.duration',\n    {\n      description: 'Time spent executing catalog processors',\n      unit: 'seconds',\n    },\n  );\n\n  const processingQueueDelay = meter.createHistogram(\n    'catalog.processing.queue.delay',\n    {\n      description:\n        'The amount of delay between being scheduled for processing, and the start of actually being processed',\n      unit: 'seconds',\n    },\n  );\n\n  function processStart(item: RefreshStateItem, logger: LoggerService) {\n    const startTime = process.hrtime();\n    const endOverallTimer = promProcessingDuration.startTimer();\n    const endProcessorsTimer = promProcessorsDuration.startTimer();\n\n    logger.debug(`Processing ${item.entityRef}`);\n\n    if (item.nextUpdateAt) {\n      const seconds = -item.nextUpdateAt.diffNow().as('seconds');\n      promProcessingQueueDelay.observe(seconds);\n      processingQueueDelay.record(seconds);\n    }\n\n    function endTime() {\n      const delta = process.hrtime(startTime);\n      return delta[0] + delta[1] / 1e9;\n    }\n\n    function markProcessorsCompleted(result: EntityProcessingResult) {\n      endProcessorsTimer({ result: result.ok ? 'ok' : 'failed' });\n      processorsDuration.record(endTime(), {\n        result: result.ok ? 'ok' : 'failed',\n      });\n    }\n\n    function markSuccessfulWithNoChanges() {\n      endOverallTimer({ result: 'unchanged' });\n      promProcessedEntities.inc({ result: 'unchanged' }, 1);\n\n      processingDuration.record(endTime(), { result: 'unchanged' });\n      processedEntities.add(1, { result: 'unchanged' });\n    }\n\n    function markSuccessfulWithErrors() {\n      endOverallTimer({ result: 'errors' });\n      promProcessedEntities.inc({ result: 'errors' }, 1);\n\n      processingDuration.record(endTime(), { result: 'errors' });\n      processedEntities.add(1, { result: 'errors' });\n    }\n\n    function markSuccessfulWithChanges() {\n      endOverallTimer({ result: 'changed' });\n      promProcessedEntities.inc({ result: 'changed' }, 1);\n\n      processingDuration.record(endTime(), { result: 'changed' });\n      processedEntities.add(1, { result: 'changed' });\n    }\n\n    function markFailed(error: Error) {\n      promProcessedEntities.inc({ result: 'failed' }, 1);\n      processedEntities.add(1, { result: 'failed' });\n      logger.warn(`Processing of ${item.entityRef} failed`, error);\n    }\n\n    return {\n      markProcessorsCompleted,\n      markSuccessfulWithNoChanges,\n      markSuccessfulWithErrors,\n      markSuccessfulWithChanges,\n      markFailed,\n    };\n  }\n\n  return { processStart };\n}\n"],"names":["trace","TRACER_ID","stableStringify","startTaskPipeline","withActiveSpan","addEntityAttributes","ANNOTATION_LOCATION","CATALOG_ERRORS_TOPIC","serializeError","stringifyEntityRef","stringifyError","assertError","stitchingStrategyFromConfig","deleteOrphanedEntities","createCounterMetric","createSummaryMetric","metrics"],"mappings":";;;;;;;;;;;;;;;;;AA0CA,MAAM,SAAA,GAAY,CAAA;AAElB,MAAM,MAAA,GAASA,SAAA,CAAM,SAAA,CAAUC,uBAAS,CAAA;AAIxC,MAAM,oBAAA,GAAuB,CAAC,GAAA,KAAe;AAC3C,EAAA,MAAM,MAAA,GAAS,GAAA,CAAI,GAAA,CAAIC,gCAAe,EAAE,IAAA,EAAK;AAC7C,EAAA,OAAO,CAAA,CAAA,EAAI,MAAA,CAAO,IAAA,CAAK,GAAG,CAAC,CAAA,CAAA,CAAA;AAC7B,CAAA;AAQO,MAAM,8BAAA,CAA+B;AAAA,EACzB,MAAA;AAAA,EACA,SAAA;AAAA,EACA,MAAA;AAAA,EACA,IAAA;AAAA,EACA,kBAAA;AAAA,EACA,YAAA;AAAA,EACA,QAAA;AAAA,EACA,UAAA;AAAA,EACA,iBAAA;AAAA,EACA,uBAAA;AAAA,EACA,iBAAA;AAAA,EAIA,OAAA;AAAA,EACA,MAAA;AAAA,EAET,QAAA;AAAA,EAER,YAAY,OAAA,EAiBT;AACD,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,YAAY,OAAA,CAAQ,SAAA;AACzB,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,OAAO,OAAA,CAAQ,IAAA;AACpB,IAAA,IAAA,CAAK,qBAAqB,OAAA,CAAQ,kBAAA;AAClC,IAAA,IAAA,CAAK,eAAe,OAAA,CAAQ,YAAA;AAC5B,IAAA,IAAA,CAAK,WAAW,OAAA,CAAQ,QAAA;AACxB,IAAA,IAAA,CAAK,aAAa,OAAA,CAAQ,UAAA;AAC1B,IAAA,IAAA,CAAK,iBAAA,GAAoB,QAAQ,iBAAA,IAAqB,GAAA;AACtD,IAAA,IAAA,CAAK,uBAAA,GAA0B,QAAQ,uBAAA,IAA2B,GAAA;AAClE,IAAA,IAAA,CAAK,oBAAoB,OAAA,CAAQ,iBAAA;AACjC,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA,CAAQ,OAAA,IAAW,eAAA,EAAgB;AAClD,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AAEtB,IAAA,IAAA,CAAK,QAAA,GAAW,MAAA;AAAA,EAClB;AAAA,EAEA,MAAM,KAAA,GAAQ;AACZ,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,MAAM,IAAI,MAAM,sCAAsC,CAAA;AAAA,IACxD;AAEA,IAAA,MAAM,YAAA,GAAe,KAAK,aAAA,EAAc;AACxC,IAAA,MAAM,WAAA,GAAc,KAAK,kBAAA,EAAmB;AAE5C,IAAA,IAAA,CAAK,WAAW,MAAM;AACpB,MAAA,YAAA,EAAa;AACb,MAAA,WAAA,EAAY;AAAA,IACd,CAAA;AAAA,EACF;AAAA,EAEA,MAAM,IAAA,GAAO;AACX,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,IAAA,CAAK,QAAA,EAAS;AACd,MAAA,IAAA,CAAK,QAAA,GAAW,MAAA;AAAA,IAClB;AAAA,EACF;AAAA,EAEQ,aAAA,GAA4B;AAClC,IAAA,OAAOC,8BAAA,CAAoC;AAAA,MACzC,YAAA,EAAc,CAAA;AAAA,MACd,aAAA,EAAe,EAAA;AAAA,MACf,mBAAmB,IAAA,CAAK,iBAAA;AAAA,MACxB,SAAA,EAAW,OAAM,KAAA,KAAS;AACxB,QAAA,IAAI;AACF,UAAA,MAAM,EAAE,OAAM,GACZ,MAAM,KAAK,kBAAA,CAAmB,sBAAA,CAAuB,KAAK,IAAA,EAAM;AAAA,YAC9D,gBAAA,EAAkB;AAAA,WACnB,CAAA;AACH,UAAA,OAAO,KAAA;AAAA,QACT,SAAS,KAAA,EAAO;AACd,UAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,iCAAA,EAAmC,KAAK,CAAA;AACzD,UAAA,OAAO,EAAC;AAAA,QACV;AAAA,MACF,CAAA;AAAA,MACA,WAAA,EAAa,OAAM,IAAA,KAAQ;AACzB,QAAA,MAAMC,4BAAA,CAAe,MAAA,EAAQ,eAAA,EAAiB,OAAM,IAAA,KAAQ;AAC1D,UAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,YAAA,CAAa,IAAA,EAAM,KAAK,MAAM,CAAA;AACzD,UAAAC,iCAAA,CAAoB,IAAA,EAAM,KAAK,iBAAiB,CAAA;AAEhD,UAAA,IAAI;AACF,YAAA,MAAM;AAAA,cACJ,EAAA;AAAA,cACA,KAAA;AAAA,cACA,iBAAA;AAAA,cACA,SAAA;AAAA,cACA,WAAA;AAAA,cACA,UAAA,EAAY;AAAA,aACd,GAAI,IAAA;AACJ,YAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,YAAA,CAAa,OAAA,CAAQ;AAAA,cAC7C,MAAA,EAAQ,iBAAA;AAAA,cACR;AAAA,aACD,CAAA;AAED,YAAA,KAAA,CAAM,wBAAwB,MAAM,CAAA;AAEpC,YAAA,IAAI,OAAO,EAAA,EAAI;AACb,cAAA,MAAM,EAAE,GAAA,EAAK,CAAA,EAAG,GAAG,eAAA,EAAgB,GAAI,SAAS,EAAC;AACjD,cAAA,IACEH,iCAAgB,eAAe,CAAA,KAC/BA,gCAAA,CAAgB,MAAA,CAAO,KAAK,CAAA,EAC5B;AACA,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,kBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,iBAAA,CAAkB,EAAA,EAAI;AAAA,oBAClD,EAAA;AAAA,oBACA,KAAA,EAAO;AAAA,sBACL,GAAA,EAAK,SAAA;AAAA,sBACL,GAAG,MAAA,CAAO;AAAA;AACZ,mBACD,CAAA;AAAA,gBACH,CAAC,CAAA;AAAA,cACH;AAAA,YACF,CAAA,MAAO;AACL,cAAA,MAAM,WAAW,KAAA,EAAO,GAAA;AACxB,cAAA,MAAM,GAAA,GAAM,MAAA,CAAO,SAAA,CAAU,QAAQ,IAAK,QAAA,GAAsB,CAAA;AAChE,cAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,iBAAA,CAAkB,EAAA,EAAI;AAAA,kBAClD,EAAA;AAAA,kBACA,KAAA,EAAO,GAAA,GAAM,CAAA,GAAI,EAAE,GAAG,OAAO,GAAA,EAAK,GAAA,GAAM,CAAA,EAAE,GAAI;AAAC,iBAChD,CAAA;AAAA,cACH,CAAC,CAAA;AAAA,YACH;AAEA,YAAA,MAAM,QAAA,GACJ,iBAAA,EAAmB,QAAA,EAAU,WAAA,GAAcI,gCAAmB,CAAA;AAChE,YAAA,IAAI,MAAA,CAAO,OAAO,MAAA,EAAQ;AACxB,cAAA,IAAA,CAAK,OAAO,OAAA,CAAQ;AAAA,gBAClB,KAAA,EAAOC,8BAAA;AAAA,gBACP,YAAA,EAAc;AAAA,kBACZ,MAAA,EAAQ,SAAA;AAAA,kBACR,QAAA;AAAA,kBACA,QAAQ,MAAA,CAAO;AAAA;AACjB,eACD,CAAA;AAAA,YACH;AACA,YAAA,MAAM,eAAe,IAAA,CAAK,SAAA;AAAA,cACxB,OAAO,MAAA,CAAO,GAAA,CAAI,CAAA,CAAA,KAAKC,qBAAA,CAAe,CAAC,CAAC;AAAA,aAC1C;AAEA,YAAA,IAAI,WAAA,GAAc,IAAA,CAAK,UAAA,EAAW,CAAE,OAAO,YAAY,CAAA;AAEvD,YAAA,IAAI,OAAO,EAAA,EAAI;AACb,cAAA,MAAM,EAAE,UAAA,EAAY,OAAA,EAAQ,GAC1B,MAAM,KAAK,kBAAA,CAAmB,WAAA;AAAA,gBAAY,CAAA,EAAA,KACxC,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,EAAA,EAAI;AAAA,kBACtC,UAAA,EAAY;AAAA,oBACV,SAAA;AAAA,oBACA,GAAG,OAAO,gBAAA,CAAiB,GAAA;AAAA,sBAAI,CAAA,CAAA,KAC7BC,+BAAA,CAAmB,CAAA,CAAE,MAAM;AAAA;AAC7B;AACF,iBACD;AAAA,eACH;AAEF,cAAA,WAAA,GAAc,YACX,MAAA,CAAOP,gCAAA,CAAgB,EAAE,GAAG,MAAA,CAAO,iBAAiB,CAAC,EACrD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,gBAAgB,CAAC,CAAC,EACzD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,SAAS,CAAC,CAAC,EAClD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,WAAW,CAAC,CAAC,EACpD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,OAAO,CAAC,CAAC,CAAA;AAAA,YAC9C;AAEA,YAAA,MAAM,UAAA,GAAa,WAAA,CAAY,MAAA,CAAO,KAAK,CAAA;AAC3C,YAAA,IAAI,eAAe,kBAAA,EAAoB;AAIrC,cAAA,KAAA,CAAM,2BAAA,EAA4B;AAClC,cAAA;AAAA,YACF;AASA,YAAA,IAAI,CAAC,OAAO,EAAA,EAAI;AAEd,cAAA,OAAA,CAAQ,OAAA,CAAQ,MAAS,CAAA,CACtB,IAAA;AAAA,gBAAK,MACJ,KAAK,iBAAA,GAAoB;AAAA,kBACvB,iBAAA;AAAA,kBACA,QAAQ,MAAA,CAAO;AAAA,iBAChB;AAAA,eACH,CACC,MAAM,CAAA,KAAA,KAAS;AACd,gBAAA,IAAA,CAAK,MAAA,CAAO,KAAA;AAAA,kBACV,CAAA,8CAAA,EAAiDQ,qBAAA;AAAA,oBAC/C;AAAA,mBACD,CAAA;AAAA,iBACH;AAAA,cACF,CAAC,CAAA;AAEH,cAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,2BAAA,CAA4B,EAAA,EAAI;AAAA,kBAC5D,EAAA;AAAA,kBACA,MAAA,EAAQ,YAAA;AAAA,kBACR;AAAA,iBACD,CAAA;AAAA,cACH,CAAC,CAAA;AAED,cAAA,MAAM,IAAA,CAAK,SAAS,MAAA,CAAO;AAAA,gBACzB,UAAA,EAAY,CAACD,+BAAA,CAAmB,iBAAiB,CAAC;AAAA,eACnD,CAAA;AAED,cAAA,KAAA,CAAM,wBAAA,EAAyB;AAC/B,cAAA;AAAA,YACF;AAEA,YAAA,MAAA,CAAO,eAAA,CAAgB,SAAS,GAAA,GAAM,EAAA;AACtC,YAAA,IAAI,kBAAA;AACJ,YAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,cAAA,MAAM,EAAE,QAAA,EAAS,GACf,MAAM,IAAA,CAAK,kBAAA,CAAmB,sBAAsB,EAAA,EAAI;AAAA,gBACtD,EAAA;AAAA,gBACA,iBAAiB,MAAA,CAAO,eAAA;AAAA,gBACxB,UAAA;AAAA,gBACA,MAAA,EAAQ,YAAA;AAAA,gBACR,WAAW,MAAA,CAAO,SAAA;AAAA,gBAClB,kBAAkB,MAAA,CAAO,gBAAA;AAAA,gBACzB,WAAA;AAAA,gBACA,aAAa,MAAA,CAAO;AAAA,eACrB,CAAA;AACH,cAAA,kBAAA,GAAqB,IAAI,GAAA;AAAA,gBACvB,QAAA,CAAS,SAAA,CAAU,GAAA,CAAI,CAAA,CAAA,KAAK;AAAA,kBAC1B,CAAA,EAAG,EAAE,iBAAiB,CAAA,CAAA,EAAI,EAAE,IAAI,CAAA,EAAA,EAAK,EAAE,iBAAiB,CAAA,CAAA;AAAA,kBACxD,CAAA,CAAE;AAAA,iBACH;AAAA,eACH;AAAA,YACF,CAAC,CAAA;AAED,YAAA,MAAM,qBAAqB,IAAI,GAAA;AAAA,cAC7B,MAAA,CAAO,SAAA,CAAU,GAAA,CAAI,CAAA,QAAA,KAAY;AAC/B,gBAAA,MAAM,eAAA,GAAkBA,+BAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAC1D,gBAAA,MAAM,eAAA,GAAkBA,+BAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAC1D,gBAAA,OAAO;AAAA,kBACL,GAAG,eAAe,CAAA,CAAA,EAAI,QAAA,CAAS,IAAI,KAAK,eAAe,CAAA,CAAA;AAAA,kBACvD;AAAA,iBACF;AAAA,cACF,CAAC;AAAA,aACH;AAEA,YAAA,MAAM,mBAAA,uBAA0B,GAAA,CAAY;AAAA,cAC1CA,+BAAA,CAAmB,OAAO,eAAe;AAAA,aAC1C,CAAA;AACD,YAAA,kBAAA,CAAmB,OAAA,CAAQ,CAAC,eAAA,EAAiB,SAAA,KAAc;AACzD,cAAA,IAAI,CAAC,kBAAA,CAAmB,GAAA,CAAI,SAAS,CAAA,EAAG;AACtC,gBAAA,mBAAA,CAAoB,IAAI,eAAe,CAAA;AAAA,cACzC;AAAA,YACF,CAAC,CAAA;AACD,YAAA,kBAAA,CAAoB,OAAA,CAAQ,CAAC,eAAA,EAAiB,SAAA,KAAc;AAC1D,cAAA,IAAI,CAAC,kBAAA,CAAmB,GAAA,CAAI,SAAS,CAAA,EAAG;AACtC,gBAAA,mBAAA,CAAoB,IAAI,eAAe,CAAA;AAAA,cACzC;AAAA,YACF,CAAC,CAAA;AAED,YAAA,MAAM,IAAA,CAAK,SAAS,MAAA,CAAO;AAAA,cACzB,UAAA,EAAY;AAAA,aACb,CAAA;AAED,YAAA,KAAA,CAAM,yBAAA,EAA0B;AAAA,UAClC,SAAS,KAAA,EAAO;AACd,YAAAE,kBAAA,CAAY,KAAK,CAAA;AACjB,YAAA,KAAA,CAAM,WAAW,KAAK,CAAA;AAAA,UACxB;AAAA,QACF,CAAC,CAAA;AAAA,MACH;AAAA,KACD,CAAA;AAAA,EACH;AAAA,EAEQ,kBAAA,GAAiC;AACvC,IAAA,MAAM,cAAA,GACJ,IAAA,CAAK,MAAA,CAAO,iBAAA,CAAkB,wBAAwB,CAAA,IAAK,QAAA;AAC7D,IAAA,IAAI,mBAAmB,QAAA,EAAU;AAC/B,MAAA,OAAO,MAAM;AAAA,MAAC,CAAA;AAAA,IAChB;AAEA,IAAA,MAAM,iBAAA,GAAoBC,iCAAA,CAA4B,IAAA,CAAK,MAAM,CAAA;AAEjE,IAAA,MAAM,UAAU,YAAY;AAC1B,MAAA,IAAI;AACF,QAAA,MAAM,CAAA,GAAI,MAAMC,6CAAA,CAAuB;AAAA,UACrC,MAAM,IAAA,CAAK,IAAA;AAAA,UACX,QAAA,EAAU;AAAA,SACX,CAAA;AACD,QAAA,IAAI,IAAI,CAAA,EAAG;AACT,UAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,CAAA,QAAA,EAAW,CAAC,CAAA,kBAAA,CAAoB,CAAA;AAAA,QACnD;AAAA,MACF,SAAS,KAAA,EAAO;AACd,QAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,CAAA,kCAAA,CAAA,EAAsC,KAAK,CAAA;AAAA,MAC9D;AAAA,IACF,CAAA;AAEA,IAAA,MAAM,eAAA,GAAkB,IAAI,eAAA,EAAgB;AAC5C,IAAA,IAAA,CAAK,UAAU,YAAA,CAAa;AAAA,MAC1B,EAAA,EAAI,wBAAA;AAAA,MACJ,SAAA,EAAW,EAAE,YAAA,EAAc,IAAA,CAAK,uBAAA,EAAwB;AAAA,MACxD,OAAA,EAAS,EAAE,YAAA,EAAc,IAAA,CAAK,0BAA0B,GAAA,EAAI;AAAA,MAC5D,EAAA,EAAI,OAAA;AAAA,MACJ,QAAQ,eAAA,CAAgB;AAAA,KACzB,CAAA;AAED,IAAA,OAAO,MAAM;AACX,MAAA,eAAA,CAAgB,KAAA,EAAM;AAAA,IACxB,CAAA;AAAA,EACF;AACF;AAGA,SAAS,eAAA,GAAkB;AAEzB,EAAA,MAAM,wBAAwBC,2BAAA,CAAoB;AAAA,IAChD,IAAA,EAAM,kCAAA;AAAA,IACN,IAAA,EAAM,6EAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBC,2BAAA,CAAoB;AAAA,IACjD,IAAA,EAAM,qCAAA;AAAA,IACN,IAAA,EAAM,8FAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBA,2BAAA,CAAoB;AAAA,IACjD,IAAA,EAAM,qCAAA;AAAA,IACN,IAAA,EAAM,wFAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,2BAA2BA,2BAAA,CAAoB;AAAA,IACnD,IAAA,EAAM,wCAAA;AAAA,IACN,IAAA,EAAM;AAAA,GACP,CAAA;AAED,EAAA,MAAM,KAAA,GAAQC,WAAA,CAAQ,QAAA,CAAS,SAAS,CAAA;AACxC,EAAA,MAAM,oBAAoB,KAAA,CAAM,aAAA;AAAA,IAC9B,kCAAA;AAAA,IACA,EAAE,aAAa,8BAAA;AAA+B,GAChD;AAEA,EAAA,MAAM,qBAAqB,KAAA,CAAM,eAAA;AAAA,IAC/B,6BAAA;AAAA,IACA;AAAA,MACE,WAAA,EAAa,+CAAA;AAAA,MACb,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,MAAM,qBAAqB,KAAA,CAAM,eAAA;AAAA,IAC/B,6BAAA;AAAA,IACA;AAAA,MACE,WAAA,EAAa,yCAAA;AAAA,MACb,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,MAAM,uBAAuB,KAAA,CAAM,eAAA;AAAA,IACjC,gCAAA;AAAA,IACA;AAAA,MACE,WAAA,EACE,uGAAA;AAAA,MACF,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,SAAS,YAAA,CAAa,MAAwB,MAAA,EAAuB;AACnE,IAAA,MAAM,SAAA,GAAY,QAAQ,MAAA,EAAO;AACjC,IAAA,MAAM,eAAA,GAAkB,uBAAuB,UAAA,EAAW;AAC1D,IAAA,MAAM,kBAAA,GAAqB,uBAAuB,UAAA,EAAW;AAE7D,IAAA,MAAA,CAAO,KAAA,CAAM,CAAA,WAAA,EAAc,IAAA,CAAK,SAAS,CAAA,CAAE,CAAA;AAE3C,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,MAAM,UAAU,CAAC,IAAA,CAAK,aAAa,OAAA,EAAQ,CAAE,GAAG,SAAS,CAAA;AACzD,MAAA,wBAAA,CAAyB,QAAQ,OAAO,CAAA;AACxC,MAAA,oBAAA,CAAqB,OAAO,OAAO,CAAA;AAAA,IACrC;AAEA,IAAA,SAAS,OAAA,GAAU;AACjB,MAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,MAAA,CAAO,SAAS,CAAA;AACtC,MAAA,OAAO,KAAA,CAAM,CAAC,CAAA,GAAI,KAAA,CAAM,CAAC,CAAA,GAAI,GAAA;AAAA,IAC/B;AAEA,IAAA,SAAS,wBAAwB,MAAA,EAAgC;AAC/D,MAAA,kBAAA,CAAmB,EAAE,MAAA,EAAQ,MAAA,CAAO,EAAA,GAAK,IAAA,GAAO,UAAU,CAAA;AAC1D,MAAA,kBAAA,CAAmB,MAAA,CAAO,SAAQ,EAAG;AAAA,QACnC,MAAA,EAAQ,MAAA,CAAO,EAAA,GAAK,IAAA,GAAO;AAAA,OAC5B,CAAA;AAAA,IACH;AAEA,IAAA,SAAS,2BAAA,GAA8B;AACrC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,WAAA,EAAa,CAAA;AACvC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,WAAA,IAAe,CAAC,CAAA;AAEpD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAC5D,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAAA,IAClD;AAEA,IAAA,SAAS,wBAAA,GAA2B;AAClC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,QAAA,EAAU,CAAA;AACpC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,QAAA,IAAY,CAAC,CAAA;AAEjD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AACzD,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAAA,IAC/C;AAEA,IAAA,SAAS,yBAAA,GAA4B;AACnC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,SAAA,EAAW,CAAA;AACrC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,SAAA,IAAa,CAAC,CAAA;AAElD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAC1D,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAAA,IAChD;AAEA,IAAA,SAAS,WAAW,KAAA,EAAc;AAChC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,QAAA,IAAY,CAAC,CAAA;AACjD,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAC7C,MAAA,MAAA,CAAO,IAAA,CAAK,CAAA,cAAA,EAAiB,IAAA,CAAK,SAAS,WAAW,KAAK,CAAA;AAAA,IAC7D;AAEA,IAAA,OAAO;AAAA,MACL,uBAAA;AAAA,MACA,2BAAA;AAAA,MACA,wBAAA;AAAA,MACA,yBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF;AAEA,EAAA,OAAO,EAAE,YAAA,EAAa;AACxB;;;;"}
+{"version":3,"file":"DefaultCatalogProcessingEngine.cjs.js","sources":["../../src/processing/DefaultCatalogProcessingEngine.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  ANNOTATION_LOCATION,\n  Entity,\n  stringifyEntityRef,\n} from '@backstage/catalog-model';\nimport { assertError, serializeError, stringifyError } from '@backstage/errors';\nimport { Hash } from 'crypto';\nimport stableStringify from 'fast-json-stable-stringify';\nimport { Knex } from 'knex';\nimport { metrics, trace } from '@opentelemetry/api';\nimport { ProcessingDatabase, RefreshStateItem } from '../database/types';\nimport { createCounterMetric, createSummaryMetric } from '../util/metrics';\nimport { CatalogProcessingOrchestrator, EntityProcessingResult } from './types';\nimport { Stitcher, stitchingStrategyFromConfig } from '../stitching/types';\nimport { startTaskPipeline } from './TaskPipeline';\nimport { Config } from '@backstage/config';\nimport {\n  addEntityAttributes,\n  TRACER_ID,\n  withActiveSpan,\n} from '../util/opentelemetry';\nimport { deleteOrphanedEntities } from '../database/operations/util/deleteOrphanedEntities';\nimport { EventBroker, EventsService } from '@backstage/plugin-events-node';\nimport { CATALOG_ERRORS_TOPIC } from '../constants';\nimport { LoggerService, SchedulerService } from '@backstage/backend-plugin-api';\n\nconst CACHE_TTL = 5;\n\nconst tracer = trace.getTracer(TRACER_ID);\n\nexport type ProgressTracker = ReturnType<typeof progressTracker>;\n\nconst stableStringifyArray = (arr: any[]) => {\n  const sorted = arr.map(stableStringify).sort();\n  return `[${sorted.join(',')}]`;\n};\n\n// NOTE(freben): Perhaps surprisingly, this class does not implement the\n// CatalogProcessingEngine type. That type is externally visible and its name is\n// the way it is for historic reasons. This class has no particular reason to\n// implement that precise interface; nowadays there are several different\n// engines \"hiding\" behind the CatalogProcessingEngine interface, of which this\n// is just one.\nexport class DefaultCatalogProcessingEngine {\n  private readonly config: Config;\n  private readonly scheduler?: SchedulerService;\n  private readonly logger: LoggerService;\n  private readonly knex: Knex;\n  private readonly processingDatabase: ProcessingDatabase;\n  private readonly orchestrator: CatalogProcessingOrchestrator;\n  private readonly stitcher: Stitcher;\n  private readonly createHash: () => Hash;\n  private readonly pollingIntervalMs: number;\n  private readonly orphanCleanupIntervalMs: number;\n  private readonly onProcessingError?: (event: {\n    unprocessedEntity: Entity;\n    errors: Error[];\n  }) => Promise<void> | void;\n  private readonly tracker: ProgressTracker;\n  private readonly eventBroker?: EventBroker | EventsService;\n\n  private stopFunc?: () => void;\n\n  constructor(options: {\n    config: Config;\n    scheduler?: SchedulerService;\n    logger: LoggerService;\n    knex: Knex;\n    processingDatabase: ProcessingDatabase;\n    orchestrator: CatalogProcessingOrchestrator;\n    stitcher: Stitcher;\n    createHash: () => Hash;\n    pollingIntervalMs?: number;\n    orphanCleanupIntervalMs?: number;\n    onProcessingError?: (event: {\n      unprocessedEntity: Entity;\n      errors: Error[];\n    }) => Promise<void> | void;\n    tracker?: ProgressTracker;\n    eventBroker?: EventBroker | EventsService;\n  }) {\n    this.config = options.config;\n    this.scheduler = options.scheduler;\n    this.logger = options.logger;\n    this.knex = options.knex;\n    this.processingDatabase = options.processingDatabase;\n    this.orchestrator = options.orchestrator;\n    this.stitcher = options.stitcher;\n    this.createHash = options.createHash;\n    this.pollingIntervalMs = options.pollingIntervalMs ?? 1_000;\n    this.orphanCleanupIntervalMs = options.orphanCleanupIntervalMs ?? 30_000;\n    this.onProcessingError = options.onProcessingError;\n    this.tracker = options.tracker ?? progressTracker();\n    this.eventBroker = options.eventBroker;\n\n    this.stopFunc = undefined;\n  }\n\n  async start() {\n    if (this.stopFunc) {\n      throw new Error('Processing engine is already started');\n    }\n\n    const stopPipeline = this.startPipeline();\n    const stopCleanup = this.startOrphanCleanup();\n\n    this.stopFunc = () => {\n      stopPipeline();\n      stopCleanup();\n    };\n  }\n\n  async stop() {\n    if (this.stopFunc) {\n      this.stopFunc();\n      this.stopFunc = undefined;\n    }\n  }\n\n  private startPipeline(): () => void {\n    return startTaskPipeline<RefreshStateItem>({\n      lowWatermark: 5,\n      highWatermark: 10,\n      pollingIntervalMs: this.pollingIntervalMs,\n      loadTasks: async count => {\n        try {\n          const { items } =\n            await this.processingDatabase.getProcessableEntities(this.knex, {\n              processBatchSize: count,\n            });\n          return items;\n        } catch (error) {\n          this.logger.warn('Failed to load processing items', error);\n          return [];\n        }\n      },\n      processTask: async item => {\n        await withActiveSpan(tracer, 'ProcessingRun', async span => {\n          const track = this.tracker.processStart(item, this.logger);\n          addEntityAttributes(span, item.unprocessedEntity);\n\n          try {\n            const {\n              id,\n              state,\n              unprocessedEntity,\n              entityRef,\n              locationKey,\n              resultHash: previousResultHash,\n            } = item;\n            const result = await this.orchestrator.process({\n              entity: unprocessedEntity,\n              state,\n            });\n\n            track.markProcessorsCompleted(result);\n\n            if (result.ok) {\n              const { ttl: _, ...stateWithoutTtl } = state ?? {};\n              if (\n                stableStringify(stateWithoutTtl) !==\n                stableStringify(result.state)\n              ) {\n                await this.processingDatabase.transaction(async tx => {\n                  await this.processingDatabase.updateEntityCache(tx, {\n                    id,\n                    state: {\n                      ttl: CACHE_TTL,\n                      ...result.state,\n                    },\n                  });\n                });\n              }\n            } else {\n              const maybeTtl = state?.ttl;\n              const ttl = Number.isInteger(maybeTtl) ? (maybeTtl as number) : 0;\n              await this.processingDatabase.transaction(async tx => {\n                await this.processingDatabase.updateEntityCache(tx, {\n                  id,\n                  state: ttl > 0 ? { ...state, ttl: ttl - 1 } : {},\n                });\n              });\n            }\n\n            const location =\n              unprocessedEntity?.metadata?.annotations?.[ANNOTATION_LOCATION];\n            if (result.errors.length) {\n              this.eventBroker?.publish({\n                topic: CATALOG_ERRORS_TOPIC,\n                eventPayload: {\n                  entity: entityRef,\n                  location,\n                  errors: result.errors,\n                },\n              });\n            }\n            const errorsString = JSON.stringify(\n              result.errors.map(e => serializeError(e)),\n            );\n\n            let hashBuilder = this.createHash().update(errorsString);\n\n            if (result.ok) {\n              const { entityRefs: parents } =\n                await this.processingDatabase.transaction(tx =>\n                  this.processingDatabase.listParents(tx, {\n                    entityRefs: [\n                      entityRef,\n                      ...result.deferredEntities.map(e =>\n                        stringifyEntityRef(e.entity),\n                      ),\n                    ],\n                  }),\n                );\n\n              hashBuilder = hashBuilder\n                .update(stableStringify({ ...result.completedEntity }))\n                .update(stableStringifyArray([...result.deferredEntities]))\n                .update(stableStringifyArray([...result.relations]))\n                .update(stableStringifyArray([...result.refreshKeys]))\n                .update(stableStringifyArray([...parents]));\n            }\n\n            const resultHash = hashBuilder.digest('hex');\n            if (resultHash === previousResultHash) {\n              // If nothing changed in our produced outputs, we cannot have any\n              // significant effect on our surroundings; therefore, we just abort\n              // without any updates / stitching.\n              track.markSuccessfulWithNoChanges();\n              return;\n            }\n\n            // If the result was marked as not OK, it signals that some part of the\n            // processing pipeline threw an exception. This can happen both as part of\n            // non-catastrophic things such as due to validation errors, as well as if\n            // something fatal happens inside the processing for other reasons. In any\n            // case, this means we can't trust that anything in the output is okay. So\n            // just store the errors and trigger a stich so that they become visible to\n            // the outside.\n            if (!result.ok) {\n              // notify the error listener if the entity can not be processed.\n              Promise.resolve(undefined)\n                .then(() =>\n                  this.onProcessingError?.({\n                    unprocessedEntity,\n                    errors: result.errors,\n                  }),\n                )\n                .catch(error => {\n                  this.logger.debug(\n                    `Processing error listener threw an exception, ${stringifyError(\n                      error,\n                    )}`,\n                  );\n                });\n\n              await this.processingDatabase.transaction(async tx => {\n                await this.processingDatabase.updateProcessedEntityErrors(tx, {\n                  id,\n                  errors: errorsString,\n                  resultHash,\n                });\n              });\n\n              await this.stitcher.stitch({\n                entityRefs: [stringifyEntityRef(unprocessedEntity)],\n              });\n\n              track.markSuccessfulWithErrors();\n              return;\n            }\n\n            result.completedEntity.metadata.uid = id;\n            let oldRelationSources: Map<string, string>;\n            await this.processingDatabase.transaction(async tx => {\n              const { previous } =\n                await this.processingDatabase.updateProcessedEntity(tx, {\n                  id,\n                  processedEntity: result.completedEntity,\n                  resultHash,\n                  errors: errorsString,\n                  relations: result.relations,\n                  deferredEntities: result.deferredEntities,\n                  locationKey,\n                  refreshKeys: result.refreshKeys,\n                });\n              oldRelationSources = new Map(\n                previous.relations.map(r => [\n                  `${r.source_entity_ref}:${r.type}->${r.target_entity_ref}`,\n                  r.source_entity_ref,\n                ]),\n              );\n            });\n\n            const newRelationSources = new Map<string, string>(\n              result.relations.map(relation => {\n                const sourceEntityRef = stringifyEntityRef(relation.source);\n                const targetEntityRef = stringifyEntityRef(relation.target);\n                return [\n                  `${sourceEntityRef}:${relation.type}->${targetEntityRef}`,\n                  sourceEntityRef,\n                ];\n              }),\n            );\n\n            const setOfThingsToStitch = new Set<string>([\n              stringifyEntityRef(result.completedEntity),\n            ]);\n            newRelationSources.forEach((sourceEntityRef, uniqueKey) => {\n              if (!oldRelationSources.has(uniqueKey)) {\n                setOfThingsToStitch.add(sourceEntityRef);\n              }\n            });\n            oldRelationSources!.forEach((sourceEntityRef, uniqueKey) => {\n              if (!newRelationSources.has(uniqueKey)) {\n                setOfThingsToStitch.add(sourceEntityRef);\n              }\n            });\n\n            await this.stitcher.stitch({\n              entityRefs: setOfThingsToStitch,\n            });\n\n            track.markSuccessfulWithChanges();\n          } catch (error) {\n            assertError(error);\n            track.markFailed(error);\n          }\n        });\n      },\n    });\n  }\n\n  private startOrphanCleanup(): () => void {\n    const orphanStrategy =\n      this.config.getOptionalString('catalog.orphanStrategy') ?? 'delete';\n    if (orphanStrategy !== 'delete') {\n      return () => {};\n    }\n\n    const stitchingStrategy = stitchingStrategyFromConfig(this.config);\n\n    const runOnce = async () => {\n      try {\n        const n = await deleteOrphanedEntities({\n          knex: this.knex,\n          strategy: stitchingStrategy,\n        });\n        if (n > 0) {\n          this.logger.info(`Deleted ${n} orphaned entities`);\n        }\n      } catch (error) {\n        this.logger.warn(`Failed to delete orphaned entities`, error);\n      }\n    };\n\n    if (this.scheduler) {\n      const abortController = new AbortController();\n\n      this.scheduler.scheduleTask({\n        id: 'catalog_orphan_cleanup',\n        frequency: { milliseconds: this.orphanCleanupIntervalMs },\n        timeout: { milliseconds: this.orphanCleanupIntervalMs * 0.8 },\n        fn: runOnce,\n        signal: abortController.signal,\n      });\n\n      return () => {\n        abortController.abort();\n      };\n    }\n\n    const intervalKey = setInterval(runOnce, this.orphanCleanupIntervalMs);\n    return () => {\n      clearInterval(intervalKey);\n    };\n  }\n}\n\n// Helps wrap the timing and logging behaviors\nfunction progressTracker() {\n  // prom-client metrics are deprecated in favour of OpenTelemetry metrics.\n  const promProcessedEntities = createCounterMetric({\n    name: 'catalog_processed_entities_count',\n    help: 'Amount of entities processed, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingDuration = createSummaryMetric({\n    name: 'catalog_processing_duration_seconds',\n    help: 'Time spent executing the full processing flow, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessorsDuration = createSummaryMetric({\n    name: 'catalog_processors_duration_seconds',\n    help: 'Time spent executing catalog processors, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingQueueDelay = createSummaryMetric({\n    name: 'catalog_processing_queue_delay_seconds',\n    help: 'The amount of delay between being scheduled for processing, and the start of actually being processed, DEPRECATED, use OpenTelemetry metrics instead',\n  });\n\n  const meter = metrics.getMeter('default');\n  const processedEntities = meter.createCounter(\n    'catalog.processed.entities.count',\n    { description: 'Amount of entities processed' },\n  );\n\n  const processingDuration = meter.createHistogram(\n    'catalog.processing.duration',\n    {\n      description: 'Time spent executing the full processing flow',\n      unit: 'seconds',\n    },\n  );\n\n  const processorsDuration = meter.createHistogram(\n    'catalog.processors.duration',\n    {\n      description: 'Time spent executing catalog processors',\n      unit: 'seconds',\n    },\n  );\n\n  const processingQueueDelay = meter.createHistogram(\n    'catalog.processing.queue.delay',\n    {\n      description:\n        'The amount of delay between being scheduled for processing, and the start of actually being processed',\n      unit: 'seconds',\n    },\n  );\n\n  function processStart(item: RefreshStateItem, logger: LoggerService) {\n    const startTime = process.hrtime();\n    const endOverallTimer = promProcessingDuration.startTimer();\n    const endProcessorsTimer = promProcessorsDuration.startTimer();\n\n    logger.debug(`Processing ${item.entityRef}`);\n\n    if (item.nextUpdateAt) {\n      const seconds = -item.nextUpdateAt.diffNow().as('seconds');\n      promProcessingQueueDelay.observe(seconds);\n      processingQueueDelay.record(seconds);\n    }\n\n    function endTime() {\n      const delta = process.hrtime(startTime);\n      return delta[0] + delta[1] / 1e9;\n    }\n\n    function markProcessorsCompleted(result: EntityProcessingResult) {\n      endProcessorsTimer({ result: result.ok ? 'ok' : 'failed' });\n      processorsDuration.record(endTime(), {\n        result: result.ok ? 'ok' : 'failed',\n      });\n    }\n\n    function markSuccessfulWithNoChanges() {\n      endOverallTimer({ result: 'unchanged' });\n      promProcessedEntities.inc({ result: 'unchanged' }, 1);\n\n      processingDuration.record(endTime(), { result: 'unchanged' });\n      processedEntities.add(1, { result: 'unchanged' });\n    }\n\n    function markSuccessfulWithErrors() {\n      endOverallTimer({ result: 'errors' });\n      promProcessedEntities.inc({ result: 'errors' }, 1);\n\n      processingDuration.record(endTime(), { result: 'errors' });\n      processedEntities.add(1, { result: 'errors' });\n    }\n\n    function markSuccessfulWithChanges() {\n      endOverallTimer({ result: 'changed' });\n      promProcessedEntities.inc({ result: 'changed' }, 1);\n\n      processingDuration.record(endTime(), { result: 'changed' });\n      processedEntities.add(1, { result: 'changed' });\n    }\n\n    function markFailed(error: Error) {\n      promProcessedEntities.inc({ result: 'failed' }, 1);\n      processedEntities.add(1, { result: 'failed' });\n      logger.warn(`Processing of ${item.entityRef} failed`, error);\n    }\n\n    return {\n      markProcessorsCompleted,\n      markSuccessfulWithNoChanges,\n      markSuccessfulWithErrors,\n      markSuccessfulWithChanges,\n      markFailed,\n    };\n  }\n\n  return { processStart };\n}\n"],"names":["trace","TRACER_ID","stableStringify","startTaskPipeline","withActiveSpan","addEntityAttributes","ANNOTATION_LOCATION","CATALOG_ERRORS_TOPIC","serializeError","stringifyEntityRef","stringifyError","assertError","stitchingStrategyFromConfig","deleteOrphanedEntities","createCounterMetric","createSummaryMetric","metrics"],"mappings":";;;;;;;;;;;;;;;;;AA0CA,MAAM,SAAA,GAAY,CAAA;AAElB,MAAM,MAAA,GAASA,SAAA,CAAM,SAAA,CAAUC,uBAAS,CAAA;AAIxC,MAAM,oBAAA,GAAuB,CAAC,GAAA,KAAe;AAC3C,EAAA,MAAM,MAAA,GAAS,GAAA,CAAI,GAAA,CAAIC,gCAAe,EAAE,IAAA,EAAK;AAC7C,EAAA,OAAO,CAAA,CAAA,EAAI,MAAA,CAAO,IAAA,CAAK,GAAG,CAAC,CAAA,CAAA,CAAA;AAC7B,CAAA;AAQO,MAAM,8BAAA,CAA+B;AAAA,EACzB,MAAA;AAAA,EACA,SAAA;AAAA,EACA,MAAA;AAAA,EACA,IAAA;AAAA,EACA,kBAAA;AAAA,EACA,YAAA;AAAA,EACA,QAAA;AAAA,EACA,UAAA;AAAA,EACA,iBAAA;AAAA,EACA,uBAAA;AAAA,EACA,iBAAA;AAAA,EAIA,OAAA;AAAA,EACA,WAAA;AAAA,EAET,QAAA;AAAA,EAER,YAAY,OAAA,EAiBT;AACD,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,YAAY,OAAA,CAAQ,SAAA;AACzB,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,OAAO,OAAA,CAAQ,IAAA;AACpB,IAAA,IAAA,CAAK,qBAAqB,OAAA,CAAQ,kBAAA;AAClC,IAAA,IAAA,CAAK,eAAe,OAAA,CAAQ,YAAA;AAC5B,IAAA,IAAA,CAAK,WAAW,OAAA,CAAQ,QAAA;AACxB,IAAA,IAAA,CAAK,aAAa,OAAA,CAAQ,UAAA;AAC1B,IAAA,IAAA,CAAK,iBAAA,GAAoB,QAAQ,iBAAA,IAAqB,GAAA;AACtD,IAAA,IAAA,CAAK,uBAAA,GAA0B,QAAQ,uBAAA,IAA2B,GAAA;AAClE,IAAA,IAAA,CAAK,oBAAoB,OAAA,CAAQ,iBAAA;AACjC,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA,CAAQ,OAAA,IAAW,eAAA,EAAgB;AAClD,IAAA,IAAA,CAAK,cAAc,OAAA,CAAQ,WAAA;AAE3B,IAAA,IAAA,CAAK,QAAA,GAAW,MAAA;AAAA,EAClB;AAAA,EAEA,MAAM,KAAA,GAAQ;AACZ,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,MAAM,IAAI,MAAM,sCAAsC,CAAA;AAAA,IACxD;AAEA,IAAA,MAAM,YAAA,GAAe,KAAK,aAAA,EAAc;AACxC,IAAA,MAAM,WAAA,GAAc,KAAK,kBAAA,EAAmB;AAE5C,IAAA,IAAA,CAAK,WAAW,MAAM;AACpB,MAAA,YAAA,EAAa;AACb,MAAA,WAAA,EAAY;AAAA,IACd,CAAA;AAAA,EACF;AAAA,EAEA,MAAM,IAAA,GAAO;AACX,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,IAAA,CAAK,QAAA,EAAS;AACd,MAAA,IAAA,CAAK,QAAA,GAAW,MAAA;AAAA,IAClB;AAAA,EACF;AAAA,EAEQ,aAAA,GAA4B;AAClC,IAAA,OAAOC,8BAAA,CAAoC;AAAA,MACzC,YAAA,EAAc,CAAA;AAAA,MACd,aAAA,EAAe,EAAA;AAAA,MACf,mBAAmB,IAAA,CAAK,iBAAA;AAAA,MACxB,SAAA,EAAW,OAAM,KAAA,KAAS;AACxB,QAAA,IAAI;AACF,UAAA,MAAM,EAAE,OAAM,GACZ,MAAM,KAAK,kBAAA,CAAmB,sBAAA,CAAuB,KAAK,IAAA,EAAM;AAAA,YAC9D,gBAAA,EAAkB;AAAA,WACnB,CAAA;AACH,UAAA,OAAO,KAAA;AAAA,QACT,SAAS,KAAA,EAAO;AACd,UAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,iCAAA,EAAmC,KAAK,CAAA;AACzD,UAAA,OAAO,EAAC;AAAA,QACV;AAAA,MACF,CAAA;AAAA,MACA,WAAA,EAAa,OAAM,IAAA,KAAQ;AACzB,QAAA,MAAMC,4BAAA,CAAe,MAAA,EAAQ,eAAA,EAAiB,OAAM,IAAA,KAAQ;AAC1D,UAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,YAAA,CAAa,IAAA,EAAM,KAAK,MAAM,CAAA;AACzD,UAAAC,iCAAA,CAAoB,IAAA,EAAM,KAAK,iBAAiB,CAAA;AAEhD,UAAA,IAAI;AACF,YAAA,MAAM;AAAA,cACJ,EAAA;AAAA,cACA,KAAA;AAAA,cACA,iBAAA;AAAA,cACA,SAAA;AAAA,cACA,WAAA;AAAA,cACA,UAAA,EAAY;AAAA,aACd,GAAI,IAAA;AACJ,YAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,YAAA,CAAa,OAAA,CAAQ;AAAA,cAC7C,MAAA,EAAQ,iBAAA;AAAA,cACR;AAAA,aACD,CAAA;AAED,YAAA,KAAA,CAAM,wBAAwB,MAAM,CAAA;AAEpC,YAAA,IAAI,OAAO,EAAA,EAAI;AACb,cAAA,MAAM,EAAE,GAAA,EAAK,CAAA,EAAG,GAAG,eAAA,EAAgB,GAAI,SAAS,EAAC;AACjD,cAAA,IACEH,iCAAgB,eAAe,CAAA,KAC/BA,gCAAA,CAAgB,MAAA,CAAO,KAAK,CAAA,EAC5B;AACA,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,kBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,iBAAA,CAAkB,EAAA,EAAI;AAAA,oBAClD,EAAA;AAAA,oBACA,KAAA,EAAO;AAAA,sBACL,GAAA,EAAK,SAAA;AAAA,sBACL,GAAG,MAAA,CAAO;AAAA;AACZ,mBACD,CAAA;AAAA,gBACH,CAAC,CAAA;AAAA,cACH;AAAA,YACF,CAAA,MAAO;AACL,cAAA,MAAM,WAAW,KAAA,EAAO,GAAA;AACxB,cAAA,MAAM,GAAA,GAAM,MAAA,CAAO,SAAA,CAAU,QAAQ,IAAK,QAAA,GAAsB,CAAA;AAChE,cAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,iBAAA,CAAkB,EAAA,EAAI;AAAA,kBAClD,EAAA;AAAA,kBACA,KAAA,EAAO,GAAA,GAAM,CAAA,GAAI,EAAE,GAAG,OAAO,GAAA,EAAK,GAAA,GAAM,CAAA,EAAE,GAAI;AAAC,iBAChD,CAAA;AAAA,cACH,CAAC,CAAA;AAAA,YACH;AAEA,YAAA,MAAM,QAAA,GACJ,iBAAA,EAAmB,QAAA,EAAU,WAAA,GAAcI,gCAAmB,CAAA;AAChE,YAAA,IAAI,MAAA,CAAO,OAAO,MAAA,EAAQ;AACxB,cAAA,IAAA,CAAK,aAAa,OAAA,CAAQ;AAAA,gBACxB,KAAA,EAAOC,8BAAA;AAAA,gBACP,YAAA,EAAc;AAAA,kBACZ,MAAA,EAAQ,SAAA;AAAA,kBACR,QAAA;AAAA,kBACA,QAAQ,MAAA,CAAO;AAAA;AACjB,eACD,CAAA;AAAA,YACH;AACA,YAAA,MAAM,eAAe,IAAA,CAAK,SAAA;AAAA,cACxB,OAAO,MAAA,CAAO,GAAA,CAAI,CAAA,CAAA,KAAKC,qBAAA,CAAe,CAAC,CAAC;AAAA,aAC1C;AAEA,YAAA,IAAI,WAAA,GAAc,IAAA,CAAK,UAAA,EAAW,CAAE,OAAO,YAAY,CAAA;AAEvD,YAAA,IAAI,OAAO,EAAA,EAAI;AACb,cAAA,MAAM,EAAE,UAAA,EAAY,OAAA,EAAQ,GAC1B,MAAM,KAAK,kBAAA,CAAmB,WAAA;AAAA,gBAAY,CAAA,EAAA,KACxC,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,EAAA,EAAI;AAAA,kBACtC,UAAA,EAAY;AAAA,oBACV,SAAA;AAAA,oBACA,GAAG,OAAO,gBAAA,CAAiB,GAAA;AAAA,sBAAI,CAAA,CAAA,KAC7BC,+BAAA,CAAmB,CAAA,CAAE,MAAM;AAAA;AAC7B;AACF,iBACD;AAAA,eACH;AAEF,cAAA,WAAA,GAAc,YACX,MAAA,CAAOP,gCAAA,CAAgB,EAAE,GAAG,MAAA,CAAO,iBAAiB,CAAC,EACrD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,gBAAgB,CAAC,CAAC,EACzD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,SAAS,CAAC,CAAC,EAClD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,WAAW,CAAC,CAAC,EACpD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,OAAO,CAAC,CAAC,CAAA;AAAA,YAC9C;AAEA,YAAA,MAAM,UAAA,GAAa,WAAA,CAAY,MAAA,CAAO,KAAK,CAAA;AAC3C,YAAA,IAAI,eAAe,kBAAA,EAAoB;AAIrC,cAAA,KAAA,CAAM,2BAAA,EAA4B;AAClC,cAAA;AAAA,YACF;AASA,YAAA,IAAI,CAAC,OAAO,EAAA,EAAI;AAEd,cAAA,OAAA,CAAQ,OAAA,CAAQ,MAAS,CAAA,CACtB,IAAA;AAAA,gBAAK,MACJ,KAAK,iBAAA,GAAoB;AAAA,kBACvB,iBAAA;AAAA,kBACA,QAAQ,MAAA,CAAO;AAAA,iBAChB;AAAA,eACH,CACC,MAAM,CAAA,KAAA,KAAS;AACd,gBAAA,IAAA,CAAK,MAAA,CAAO,KAAA;AAAA,kBACV,CAAA,8CAAA,EAAiDQ,qBAAA;AAAA,oBAC/C;AAAA,mBACD,CAAA;AAAA,iBACH;AAAA,cACF,CAAC,CAAA;AAEH,cAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,2BAAA,CAA4B,EAAA,EAAI;AAAA,kBAC5D,EAAA;AAAA,kBACA,MAAA,EAAQ,YAAA;AAAA,kBACR;AAAA,iBACD,CAAA;AAAA,cACH,CAAC,CAAA;AAED,cAAA,MAAM,IAAA,CAAK,SAAS,MAAA,CAAO;AAAA,gBACzB,UAAA,EAAY,CAACD,+BAAA,CAAmB,iBAAiB,CAAC;AAAA,eACnD,CAAA;AAED,cAAA,KAAA,CAAM,wBAAA,EAAyB;AAC/B,cAAA;AAAA,YACF;AAEA,YAAA,MAAA,CAAO,eAAA,CAAgB,SAAS,GAAA,GAAM,EAAA;AACtC,YAAA,IAAI,kBAAA;AACJ,YAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,cAAA,MAAM,EAAE,QAAA,EAAS,GACf,MAAM,IAAA,CAAK,kBAAA,CAAmB,sBAAsB,EAAA,EAAI;AAAA,gBACtD,EAAA;AAAA,gBACA,iBAAiB,MAAA,CAAO,eAAA;AAAA,gBACxB,UAAA;AAAA,gBACA,MAAA,EAAQ,YAAA;AAAA,gBACR,WAAW,MAAA,CAAO,SAAA;AAAA,gBAClB,kBAAkB,MAAA,CAAO,gBAAA;AAAA,gBACzB,WAAA;AAAA,gBACA,aAAa,MAAA,CAAO;AAAA,eACrB,CAAA;AACH,cAAA,kBAAA,GAAqB,IAAI,GAAA;AAAA,gBACvB,QAAA,CAAS,SAAA,CAAU,GAAA,CAAI,CAAA,CAAA,KAAK;AAAA,kBAC1B,CAAA,EAAG,EAAE,iBAAiB,CAAA,CAAA,EAAI,EAAE,IAAI,CAAA,EAAA,EAAK,EAAE,iBAAiB,CAAA,CAAA;AAAA,kBACxD,CAAA,CAAE;AAAA,iBACH;AAAA,eACH;AAAA,YACF,CAAC,CAAA;AAED,YAAA,MAAM,qBAAqB,IAAI,GAAA;AAAA,cAC7B,MAAA,CAAO,SAAA,CAAU,GAAA,CAAI,CAAA,QAAA,KAAY;AAC/B,gBAAA,MAAM,eAAA,GAAkBA,+BAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAC1D,gBAAA,MAAM,eAAA,GAAkBA,+BAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAC1D,gBAAA,OAAO;AAAA,kBACL,GAAG,eAAe,CAAA,CAAA,EAAI,QAAA,CAAS,IAAI,KAAK,eAAe,CAAA,CAAA;AAAA,kBACvD;AAAA,iBACF;AAAA,cACF,CAAC;AAAA,aACH;AAEA,YAAA,MAAM,mBAAA,uBAA0B,GAAA,CAAY;AAAA,cAC1CA,+BAAA,CAAmB,OAAO,eAAe;AAAA,aAC1C,CAAA;AACD,YAAA,kBAAA,CAAmB,OAAA,CAAQ,CAAC,eAAA,EAAiB,SAAA,KAAc;AACzD,cAAA,IAAI,CAAC,kBAAA,CAAmB,GAAA,CAAI,SAAS,CAAA,EAAG;AACtC,gBAAA,mBAAA,CAAoB,IAAI,eAAe,CAAA;AAAA,cACzC;AAAA,YACF,CAAC,CAAA;AACD,YAAA,kBAAA,CAAoB,OAAA,CAAQ,CAAC,eAAA,EAAiB,SAAA,KAAc;AAC1D,cAAA,IAAI,CAAC,kBAAA,CAAmB,GAAA,CAAI,SAAS,CAAA,EAAG;AACtC,gBAAA,mBAAA,CAAoB,IAAI,eAAe,CAAA;AAAA,cACzC;AAAA,YACF,CAAC,CAAA;AAED,YAAA,MAAM,IAAA,CAAK,SAAS,MAAA,CAAO;AAAA,cACzB,UAAA,EAAY;AAAA,aACb,CAAA;AAED,YAAA,KAAA,CAAM,yBAAA,EAA0B;AAAA,UAClC,SAAS,KAAA,EAAO;AACd,YAAAE,kBAAA,CAAY,KAAK,CAAA;AACjB,YAAA,KAAA,CAAM,WAAW,KAAK,CAAA;AAAA,UACxB;AAAA,QACF,CAAC,CAAA;AAAA,MACH;AAAA,KACD,CAAA;AAAA,EACH;AAAA,EAEQ,kBAAA,GAAiC;AACvC,IAAA,MAAM,cAAA,GACJ,IAAA,CAAK,MAAA,CAAO,iBAAA,CAAkB,wBAAwB,CAAA,IAAK,QAAA;AAC7D,IAAA,IAAI,mBAAmB,QAAA,EAAU;AAC/B,MAAA,OAAO,MAAM;AAAA,MAAC,CAAA;AAAA,IAChB;AAEA,IAAA,MAAM,iBAAA,GAAoBC,iCAAA,CAA4B,IAAA,CAAK,MAAM,CAAA;AAEjE,IAAA,MAAM,UAAU,YAAY;AAC1B,MAAA,IAAI;AACF,QAAA,MAAM,CAAA,GAAI,MAAMC,6CAAA,CAAuB;AAAA,UACrC,MAAM,IAAA,CAAK,IAAA;AAAA,UACX,QAAA,EAAU;AAAA,SACX,CAAA;AACD,QAAA,IAAI,IAAI,CAAA,EAAG;AACT,UAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,CAAA,QAAA,EAAW,CAAC,CAAA,kBAAA,CAAoB,CAAA;AAAA,QACnD;AAAA,MACF,SAAS,KAAA,EAAO;AACd,QAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,CAAA,kCAAA,CAAA,EAAsC,KAAK,CAAA;AAAA,MAC9D;AAAA,IACF,CAAA;AAEA,IAAA,IAAI,KAAK,SAAA,EAAW;AAClB,MAAA,MAAM,eAAA,GAAkB,IAAI,eAAA,EAAgB;AAE5C,MAAA,IAAA,CAAK,UAAU,YAAA,CAAa;AAAA,QAC1B,EAAA,EAAI,wBAAA;AAAA,QACJ,SAAA,EAAW,EAAE,YAAA,EAAc,IAAA,CAAK,uBAAA,EAAwB;AAAA,QACxD,OAAA,EAAS,EAAE,YAAA,EAAc,IAAA,CAAK,0BAA0B,GAAA,EAAI;AAAA,QAC5D,EAAA,EAAI,OAAA;AAAA,QACJ,QAAQ,eAAA,CAAgB;AAAA,OACzB,CAAA;AAED,MAAA,OAAO,MAAM;AACX,QAAA,eAAA,CAAgB,KAAA,EAAM;AAAA,MACxB,CAAA;AAAA,IACF;AAEA,IAAA,MAAM,WAAA,GAAc,WAAA,CAAY,OAAA,EAAS,IAAA,CAAK,uBAAuB,CAAA;AACrE,IAAA,OAAO,MAAM;AACX,MAAA,aAAA,CAAc,WAAW,CAAA;AAAA,IAC3B,CAAA;AAAA,EACF;AACF;AAGA,SAAS,eAAA,GAAkB;AAEzB,EAAA,MAAM,wBAAwBC,2BAAA,CAAoB;AAAA,IAChD,IAAA,EAAM,kCAAA;AAAA,IACN,IAAA,EAAM,6EAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBC,2BAAA,CAAoB;AAAA,IACjD,IAAA,EAAM,qCAAA;AAAA,IACN,IAAA,EAAM,8FAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBA,2BAAA,CAAoB;AAAA,IACjD,IAAA,EAAM,qCAAA;AAAA,IACN,IAAA,EAAM,wFAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,2BAA2BA,2BAAA,CAAoB;AAAA,IACnD,IAAA,EAAM,wCAAA;AAAA,IACN,IAAA,EAAM;AAAA,GACP,CAAA;AAED,EAAA,MAAM,KAAA,GAAQC,WAAA,CAAQ,QAAA,CAAS,SAAS,CAAA;AACxC,EAAA,MAAM,oBAAoB,KAAA,CAAM,aAAA;AAAA,IAC9B,kCAAA;AAAA,IACA,EAAE,aAAa,8BAAA;AAA+B,GAChD;AAEA,EAAA,MAAM,qBAAqB,KAAA,CAAM,eAAA;AAAA,IAC/B,6BAAA;AAAA,IACA;AAAA,MACE,WAAA,EAAa,+CAAA;AAAA,MACb,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,MAAM,qBAAqB,KAAA,CAAM,eAAA;AAAA,IAC/B,6BAAA;AAAA,IACA;AAAA,MACE,WAAA,EAAa,yCAAA;AAAA,MACb,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,MAAM,uBAAuB,KAAA,CAAM,eAAA;AAAA,IACjC,gCAAA;AAAA,IACA;AAAA,MACE,WAAA,EACE,uGAAA;AAAA,MACF,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,SAAS,YAAA,CAAa,MAAwB,MAAA,EAAuB;AACnE,IAAA,MAAM,SAAA,GAAY,QAAQ,MAAA,EAAO;AACjC,IAAA,MAAM,eAAA,GAAkB,uBAAuB,UAAA,EAAW;AAC1D,IAAA,MAAM,kBAAA,GAAqB,uBAAuB,UAAA,EAAW;AAE7D,IAAA,MAAA,CAAO,KAAA,CAAM,CAAA,WAAA,EAAc,IAAA,CAAK,SAAS,CAAA,CAAE,CAAA;AAE3C,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,MAAM,UAAU,CAAC,IAAA,CAAK,aAAa,OAAA,EAAQ,CAAE,GAAG,SAAS,CAAA;AACzD,MAAA,wBAAA,CAAyB,QAAQ,OAAO,CAAA;AACxC,MAAA,oBAAA,CAAqB,OAAO,OAAO,CAAA;AAAA,IACrC;AAEA,IAAA,SAAS,OAAA,GAAU;AACjB,MAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,MAAA,CAAO,SAAS,CAAA;AACtC,MAAA,OAAO,KAAA,CAAM,CAAC,CAAA,GAAI,KAAA,CAAM,CAAC,CAAA,GAAI,GAAA;AAAA,IAC/B;AAEA,IAAA,SAAS,wBAAwB,MAAA,EAAgC;AAC/D,MAAA,kBAAA,CAAmB,EAAE,MAAA,EAAQ,MAAA,CAAO,EAAA,GAAK,IAAA,GAAO,UAAU,CAAA;AAC1D,MAAA,kBAAA,CAAmB,MAAA,CAAO,SAAQ,EAAG;AAAA,QACnC,MAAA,EAAQ,MAAA,CAAO,EAAA,GAAK,IAAA,GAAO;AAAA,OAC5B,CAAA;AAAA,IACH;AAEA,IAAA,SAAS,2BAAA,GAA8B;AACrC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,WAAA,EAAa,CAAA;AACvC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,WAAA,IAAe,CAAC,CAAA;AAEpD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAC5D,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAAA,IAClD;AAEA,IAAA,SAAS,wBAAA,GAA2B;AAClC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,QAAA,EAAU,CAAA;AACpC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,QAAA,IAAY,CAAC,CAAA;AAEjD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AACzD,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAAA,IAC/C;AAEA,IAAA,SAAS,yBAAA,GAA4B;AACnC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,SAAA,EAAW,CAAA;AACrC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,SAAA,IAAa,CAAC,CAAA;AAElD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAC1D,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAAA,IAChD;AAEA,IAAA,SAAS,WAAW,KAAA,EAAc;AAChC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,QAAA,IAAY,CAAC,CAAA;AACjD,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAC7C,MAAA,MAAA,CAAO,IAAA,CAAK,CAAA,cAAA,EAAiB,IAAA,CAAK,SAAS,WAAW,KAAK,CAAA;AAAA,IAC7D;AAEA,IAAA,OAAO;AAAA,MACL,uBAAA;AAAA,MACA,2BAAA;AAAA,MACA,wBAAA;AAAA,MACA,yBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF;AAEA,EAAA,OAAO,EAAE,YAAA,EAAa;AACxB;;;;"}

package/dist/processing/DefaultCatalogProcessingOrchestrator.cjs.js

@@ -190,6 +190,9 @@ class DefaultCatalogProcessingOrchestrator {
             );
             if (thisValid) {
               valid = true;
+              if (this.options.legacySingleProcessorValidation) {
+                break;
+              }
             }
           } catch (e) {
             throw new errors.InputError(

package/dist/processing/DefaultCatalogProcessingOrchestrator.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultCatalogProcessingOrchestrator.cjs.js","sources":["../../src/processing/DefaultCatalogProcessingOrchestrator.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Span, trace } from '@opentelemetry/api';\nimport {\n  Entity,\n  EntityPolicy,\n  LocationEntity,\n  parseLocationRef,\n  stringifyEntityRef,\n  stringifyLocationRef,\n} from '@backstage/catalog-model';\nimport {\n  assertError,\n  ConflictError,\n  InputError,\n  NotAllowedError,\n} from '@backstage/errors';\nimport { JsonValue } from '@backstage/types';\nimport { ScmIntegrationRegistry } from '@backstage/integration';\nimport path from 'path';\nimport { LocationSpec } from '@backstage/plugin-catalog-common';\nimport {\n  CatalogProcessor,\n  CatalogProcessorParser,\n  processingResult,\n} from '@backstage/plugin-catalog-node';\nimport {\n  CatalogProcessingOrchestrator,\n  EntityProcessingRequest,\n  EntityProcessingResult,\n} from './types';\nimport { ProcessorOutputCollector } from './ProcessorOutputCollector';\nimport {\n  getEntityLocationRef,\n  getEntityOriginLocationRef,\n  isLocationEntity,\n  isObject,\n  toAbsoluteUrl,\n  validateEntity,\n  validateEntityEnvelope,\n} from './util';\nimport { CatalogRulesEnforcer } from '../ingestion/CatalogRules';\nimport { ProcessorCacheManager } from './ProcessorCacheManager';\nimport {\n  addEntityAttributes,\n  TRACER_ID,\n  withActiveSpan,\n} from '../util/opentelemetry';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\nconst tracer = trace.getTracer(TRACER_ID);\n\ntype Context = {\n  entityRef: string;\n  location: LocationSpec;\n  originLocation: LocationSpec;\n  collector: ProcessorOutputCollector;\n  cache: ProcessorCacheManager;\n};\n\nfunction addProcessorAttributes(\n  span: Span,\n  stage: string,\n  processor: CatalogProcessor,\n) {\n  span.setAttribute('backstage.catalog.processor.stage', stage);\n  span.setAttribute(\n    'backstage.catalog.processor.name',\n    processor.getProcessorName(),\n  );\n}\n\nexport class DefaultCatalogProcessingOrchestrator\n  implements CatalogProcessingOrchestrator\n{\n  constructor(\n    private readonly options: {\n      processors: CatalogProcessor[];\n      integrations: ScmIntegrationRegistry;\n      logger: LoggerService;\n      parser: CatalogProcessorParser;\n      policy: EntityPolicy;\n      rulesEnforcer: CatalogRulesEnforcer;\n    },\n  ) {}\n\n  async process(\n    request: EntityProcessingRequest,\n  ): Promise<EntityProcessingResult> {\n    return this.processSingleEntity(request.entity, request.state);\n  }\n\n  private async processSingleEntity(\n    unprocessedEntity: Entity,\n    state: JsonValue | undefined,\n  ): Promise<EntityProcessingResult> {\n    const collector = new ProcessorOutputCollector(\n      this.options.logger,\n      unprocessedEntity,\n    );\n\n    // Cache that is scoped to the entity and processor\n    const cache = new ProcessorCacheManager(\n      isObject(state) && isObject(state.cache) ? state.cache : {},\n    );\n\n    try {\n      // This will be checked and mutated step by step below\n      let entity: Entity = unprocessedEntity;\n\n      // NOTE: At this early point, we can only rely on the envelope having to\n      // be valid; full entity + kind validation happens after the (potentially\n      // mutative) pre-steps. This means that the code below can't make a lot\n      // of assumptions about the data despite it using the Entity type.\n      try {\n        validateEntityEnvelope(entity);\n      } catch (e) {\n        throw new InputError(\n          `Entity envelope failed validation before processing`,\n          e,\n        );\n      }\n\n      // TODO: which one do we actually use for the location?\n      // source-location? - maybe probably doesn't exist yet?\n      const context: Context = {\n        entityRef: stringifyEntityRef(entity),\n        location: parseLocationRef(getEntityLocationRef(entity)),\n        originLocation: parseLocationRef(getEntityOriginLocationRef(entity)),\n        cache,\n        collector,\n      };\n\n      // Run the steps\n      entity = await this.runPreProcessStep(entity, context);\n      entity = await this.runPolicyStep(entity);\n      await this.runValidateStep(entity, context);\n      if (isLocationEntity(entity)) {\n        await this.runSpecialLocationStep(entity, context);\n      }\n      entity = await this.runPostProcessStep(entity, context);\n\n      // Check that any emitted entities are permitted to originate from that\n      // particular location according to the catalog rules\n      const collectorResults = context.collector.results();\n      for (const deferredEntity of collectorResults.deferredEntities) {\n        if (\n          !this.options.rulesEnforcer.isAllowed(\n            deferredEntity.entity,\n            context.originLocation,\n          )\n        ) {\n          throw new NotAllowedError(\n            `Entity ${stringifyEntityRef(\n              deferredEntity.entity,\n            )} at ${stringifyLocationRef(\n              context.location,\n            )}, originated at ${stringifyLocationRef(\n              context.originLocation,\n            )}, is not of an allowed kind for that location`,\n          );\n        }\n      }\n\n      return {\n        ...collectorResults,\n        completedEntity: entity,\n        state: { cache: cache.collect() },\n        ok: collectorResults.errors.length === 0,\n      };\n    } catch (error) {\n      assertError(error);\n      return {\n        ok: false,\n        errors: collector.results().errors.concat(error),\n      };\n    }\n  }\n\n  // Pre-process phase, used to populate entities with data that is required\n  // during the main processing step\n  private async runPreProcessStep(\n    entity: Entity,\n    context: Context,\n  ): Promise<Entity> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute('backstage.catalog.processor.stage', 'preProcess');\n      let res = entity;\n\n      for (const processor of this.options.processors) {\n        if (processor.preProcessEntity) {\n          let innerRes = res;\n          res = await withActiveSpan(tracer, 'ProcessingStep', async span => {\n            addEntityAttributes(span, entity);\n            addProcessorAttributes(span, 'preProcessEntity', processor);\n            try {\n              innerRes = await processor.preProcessEntity!(\n                innerRes,\n                context.location,\n                context.collector.forProcessor(processor),\n                context.originLocation,\n                context.cache.forProcessor(processor),\n              );\n            } catch (e) {\n              throw new InputError(\n                `Processor ${processor.constructor.name} threw an error while preprocessing`,\n                e,\n              );\n            }\n            return innerRes;\n          });\n        }\n      }\n\n      return res;\n    });\n  }\n\n  /**\n   * Enforce entity policies making sure that entities conform to a general schema\n   */\n  private async runPolicyStep(entity: Entity): Promise<Entity> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute(\n        'backstage.catalog.processor.stage',\n        'enforcePolicy',\n      );\n      let policyEnforcedEntity: Entity | undefined;\n\n      try {\n        policyEnforcedEntity = await this.options.policy.enforce(entity);\n      } catch (e) {\n        throw new InputError(\n          `Policy check failed for ${stringifyEntityRef(entity)}`,\n          e,\n        );\n      }\n\n      if (!policyEnforcedEntity) {\n        throw new Error(\n          `Policy unexpectedly returned no data for ${stringifyEntityRef(\n            entity,\n          )}`,\n        );\n      }\n\n      return policyEnforcedEntity;\n    });\n  }\n\n  /**\n   * Validate the given entity\n   */\n  private async runValidateStep(\n    entity: Entity,\n    context: Context,\n  ): Promise<void> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute('backstage.catalog.processor.stage', 'validate');\n      // Double check that none of the previous steps tried to change something\n      // related to the entity ref, which would break downstream\n      if (stringifyEntityRef(entity) !== context.entityRef) {\n        throw new ConflictError(\n          'Fatal: The entity kind, namespace, or name changed during processing',\n        );\n      }\n\n      // Validate that the end result is a valid Entity at all\n      try {\n        validateEntity(entity);\n      } catch (e) {\n        throw new ConflictError(\n          `Entity envelope for ${context.entityRef} failed validation after preprocessing`,\n          e,\n        );\n      }\n\n      let valid = false;\n\n      for (const processor of this.options.processors) {\n        if (processor.validateEntityKind) {\n          try {\n            const thisValid = await withActiveSpan(\n              tracer,\n              'ProcessingStep',\n              async span => {\n                addEntityAttributes(span, entity);\n                addProcessorAttributes(span, 'validateEntityKind', processor);\n                return await processor.validateEntityKind!(entity);\n              },\n            );\n            if (thisValid) {\n              valid = true;\n            }\n          } catch (e) {\n            throw new InputError(\n              `Processor ${processor.constructor.name} threw an error while validating the entity ${context.entityRef}`,\n              e,\n            );\n          }\n        }\n      }\n\n      if (!valid) {\n        throw new InputError(\n          `No processor recognized the entity ${context.entityRef} as valid, possibly caused by a foreign kind or apiVersion`,\n        );\n      }\n    });\n  }\n\n  /**\n   * Backwards compatible processing of location entities\n   */\n  private async runSpecialLocationStep(\n    entity: LocationEntity,\n    context: Context,\n  ): Promise<void> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute(\n        'backstage.catalog.processor.stage',\n        'readLocation',\n      );\n      const { type = context.location.type, presence = 'required' } =\n        entity.spec;\n      const targets = new Array<string>();\n      if (entity.spec.target) {\n        targets.push(entity.spec.target);\n      }\n      if (entity.spec.targets) {\n        targets.push(...entity.spec.targets);\n      }\n\n      for (const maybeRelativeTarget of targets) {\n        if (type === 'file' && maybeRelativeTarget.endsWith(path.sep)) {\n          context.collector.generic()(\n            processingResult.inputError(\n              context.location,\n              `LocationEntityProcessor cannot handle ${type} type location with target ${context.location.target} that ends with a path separator`,\n            ),\n          );\n          continue;\n        }\n        const target = toAbsoluteUrl(\n          this.options.integrations,\n          context.location,\n          type,\n          maybeRelativeTarget,\n        );\n\n        let didRead = false;\n        for (const processor of this.options.processors) {\n          if (processor.readLocation) {\n            try {\n              const read = await withActiveSpan(\n                tracer,\n                'ProcessingStep',\n                async span => {\n                  addEntityAttributes(span, entity);\n                  addProcessorAttributes(span, 'readLocation', processor);\n                  return await processor.readLocation!(\n                    {\n                      type,\n                      target,\n                      presence,\n                    },\n                    presence === 'optional',\n                    context.collector.forProcessor(processor),\n                    this.options.parser,\n                    context.cache.forProcessor(processor, target),\n                  );\n                },\n              );\n              if (read) {\n                didRead = true;\n                break;\n              }\n            } catch (e) {\n              throw new InputError(\n                `Processor ${processor.constructor.name} threw an error while reading ${type}:${target}`,\n                e,\n              );\n            }\n          }\n        }\n        if (!didRead) {\n          throw new InputError(\n            `No processor was able to handle reading of ${type}:${target}`,\n          );\n        }\n      }\n    });\n  }\n\n  /**\n   * Main processing step of the entity\n   */\n  private async runPostProcessStep(\n    entity: Entity,\n    context: Context,\n  ): Promise<Entity> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute(\n        'backstage.catalog.processor.stage',\n        'postProcessEntity',\n      );\n      let res = entity;\n\n      for (const processor of this.options.processors) {\n        if (processor.postProcessEntity) {\n          let innerRes = res;\n          res = await withActiveSpan(tracer, 'ProcessingStep', async span => {\n            addEntityAttributes(span, entity);\n            addProcessorAttributes(span, 'postProcessEntity', processor);\n            try {\n              innerRes = await processor.postProcessEntity!(\n                innerRes,\n                context.location,\n                context.collector.forProcessor(processor),\n                context.cache.forProcessor(processor),\n              );\n            } catch (e) {\n              throw new InputError(\n                `Processor ${processor.constructor.name} threw an error while postprocessing`,\n                e,\n              );\n            }\n            return innerRes;\n          });\n        }\n      }\n\n      return res;\n    });\n  }\n}\n"],"names":["trace","TRACER_ID","ProcessorOutputCollector","ProcessorCacheManager","isObject","validateEntityEnvelope","InputError","stringifyEntityRef","parseLocationRef","getEntityLocationRef","getEntityOriginLocationRef","isLocationEntity","NotAllowedError","stringifyLocationRef","assertError","withActiveSpan","addEntityAttributes","ConflictError","validateEntity","path","processingResult","toAbsoluteUrl"],"mappings":";;;;;;;;;;;;;;;;AAgEA,MAAM,MAAA,GAASA,SAAA,CAAM,SAAA,CAAUC,uBAAS,CAAA;AAUxC,SAAS,sBAAA,CACP,IAAA,EACA,KAAA,EACA,SAAA,EACA;AACA,EAAA,IAAA,CAAK,YAAA,CAAa,qCAAqC,KAAK,CAAA;AAC5D,EAAA,IAAA,CAAK,YAAA;AAAA,IACH,kCAAA;AAAA,IACA,UAAU,gBAAA;AAAiB,GAC7B;AACF;AAEO,MAAM,oCAAA,CAEb;AAAA,EACE,YACmB,OAAA,EAQjB;AARiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA,EAQhB;AAAA,EAEH,MAAM,QACJ,OAAA,EACiC;AACjC,IAAA,OAAO,IAAA,CAAK,mBAAA,CAAoB,OAAA,CAAQ,MAAA,EAAQ,QAAQ,KAAK,CAAA;AAAA,EAC/D;AAAA,EAEA,MAAc,mBAAA,CACZ,iBAAA,EACA,KAAA,EACiC;AACjC,IAAA,MAAM,YAAY,IAAIC,iDAAA;AAAA,MACpB,KAAK,OAAA,CAAQ,MAAA;AAAA,MACb;AAAA,KACF;AAGA,IAAA,MAAM,QAAQ,IAAIC,2CAAA;AAAA,MAChBC,aAAA,CAAS,KAAK,CAAA,IAAKA,aAAA,CAAS,MAAM,KAAK,CAAA,GAAI,KAAA,CAAM,KAAA,GAAQ;AAAC,KAC5D;AAEA,IAAA,IAAI;AAEF,MAAA,IAAI,MAAA,GAAiB,iBAAA;AAMrB,MAAA,IAAI;AACF,QAAAC,2BAAA,CAAuB,MAAM,CAAA;AAAA,MAC/B,SAAS,CAAA,EAAG;AACV,QAAA,MAAM,IAAIC,iBAAA;AAAA,UACR,CAAA,mDAAA,CAAA;AAAA,UACA;AAAA,SACF;AAAA,MACF;AAIA,MAAA,MAAM,OAAA,GAAmB;AAAA,QACvB,SAAA,EAAWC,gCAAmB,MAAM,CAAA;AAAA,QACpC,QAAA,EAAUC,6BAAA,CAAiBC,yBAAA,CAAqB,MAAM,CAAC,CAAA;AAAA,QACvD,cAAA,EAAgBD,6BAAA,CAAiBE,+BAAA,CAA2B,MAAM,CAAC,CAAA;AAAA,QACnE,KAAA;AAAA,QACA;AAAA,OACF;AAGA,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,iBAAA,CAAkB,MAAA,EAAQ,OAAO,CAAA;AACrD,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,aAAA,CAAc,MAAM,CAAA;AACxC,MAAA,MAAM,IAAA,CAAK,eAAA,CAAgB,MAAA,EAAQ,OAAO,CAAA;AAC1C,MAAA,IAAIC,qBAAA,CAAiB,MAAM,CAAA,EAAG;AAC5B,QAAA,MAAM,IAAA,CAAK,sBAAA,CAAuB,MAAA,EAAQ,OAAO,CAAA;AAAA,MACnD;AACA,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,kBAAA,CAAmB,MAAA,EAAQ,OAAO,CAAA;AAItD,MAAA,MAAM,gBAAA,GAAmB,OAAA,CAAQ,SAAA,CAAU,OAAA,EAAQ;AACnD,MAAA,KAAA,MAAW,cAAA,IAAkB,iBAAiB,gBAAA,EAAkB;AAC9D,QAAA,IACE,CAAC,IAAA,CAAK,OAAA,CAAQ,aAAA,CAAc,SAAA;AAAA,UAC1B,cAAA,CAAe,MAAA;AAAA,UACf,OAAA,CAAQ;AAAA,SACV,EACA;AACA,UAAA,MAAM,IAAIC,sBAAA;AAAA,YACR,CAAA,OAAA,EAAUL,+BAAA;AAAA,cACR,cAAA,CAAe;AAAA,aAChB,CAAA,IAAA,EAAOM,iCAAA;AAAA,cACN,OAAA,CAAQ;AAAA,aACT,CAAA,gBAAA,EAAmBA,iCAAA;AAAA,cAClB,OAAA,CAAQ;AAAA,aACT,CAAA,6CAAA;AAAA,WACH;AAAA,QACF;AAAA,MACF;AAEA,MAAA,OAAO;AAAA,QACL,GAAG,gBAAA;AAAA,QACH,eAAA,EAAiB,MAAA;AAAA,QACjB,KAAA,EAAO,EAAE,KAAA,EAAO,KAAA,CAAM,SAAQ,EAAE;AAAA,QAChC,EAAA,EAAI,gBAAA,CAAiB,MAAA,CAAO,MAAA,KAAW;AAAA,OACzC;AAAA,IACF,SAAS,KAAA,EAAO;AACd,MAAAC,kBAAA,CAAY,KAAK,CAAA;AACjB,MAAA,OAAO;AAAA,QACL,EAAA,EAAI,KAAA;AAAA,QACJ,QAAQ,SAAA,CAAU,OAAA,EAAQ,CAAE,MAAA,CAAO,OAAO,KAAK;AAAA,OACjD;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA,EAIA,MAAc,iBAAA,CACZ,MAAA,EACA,OAAA,EACiB;AACjB,IAAA,OAAO,MAAMC,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA,CAAa,qCAAqC,YAAY,CAAA;AACxE,MAAA,IAAI,GAAA,GAAM,MAAA;AAEV,MAAA,KAAA,MAAW,SAAA,IAAa,IAAA,CAAK,OAAA,CAAQ,UAAA,EAAY;AAC/C,QAAA,IAAI,UAAU,gBAAA,EAAkB;AAC9B,UAAA,IAAI,QAAA,GAAW,GAAA;AACf,UAAA,GAAA,GAAM,MAAMD,4BAAA,CAAe,MAAA,EAAQ,gBAAA,EAAkB,OAAM,IAAA,KAAQ;AACjE,YAAAC,iCAAA,CAAoB,MAAM,MAAM,CAAA;AAChC,YAAA,sBAAA,CAAuB,IAAA,EAAM,oBAAoB,SAAS,CAAA;AAC1D,YAAA,IAAI;AACF,cAAA,QAAA,GAAW,MAAM,SAAA,CAAU,gBAAA;AAAA,gBACzB,QAAA;AAAA,gBACA,OAAA,CAAQ,QAAA;AAAA,gBACR,OAAA,CAAQ,SAAA,CAAU,YAAA,CAAa,SAAS,CAAA;AAAA,gBACxC,OAAA,CAAQ,cAAA;AAAA,gBACR,OAAA,CAAQ,KAAA,CAAM,YAAA,CAAa,SAAS;AAAA,eACtC;AAAA,YACF,SAAS,CAAA,EAAG;AACV,cAAA,MAAM,IAAIV,iBAAA;AAAA,gBACR,CAAA,UAAA,EAAa,SAAA,CAAU,WAAA,CAAY,IAAI,CAAA,mCAAA,CAAA;AAAA,gBACvC;AAAA,eACF;AAAA,YACF;AACA,YAAA,OAAO,QAAA;AAAA,UACT,CAAC,CAAA;AAAA,QACH;AAAA,MACF;AAEA,MAAA,OAAO,GAAA;AAAA,IACT,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,cAAc,MAAA,EAAiC;AAC3D,IAAA,OAAO,MAAMS,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA;AAAA,QACR,mCAAA;AAAA,QACA;AAAA,OACF;AACA,MAAA,IAAI,oBAAA;AAEJ,MAAA,IAAI;AACF,QAAA,oBAAA,GAAuB,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,QAAQ,MAAM,CAAA;AAAA,MACjE,SAAS,CAAA,EAAG;AACV,QAAA,MAAM,IAAIV,iBAAA;AAAA,UACR,CAAA,wBAAA,EAA2BC,+BAAA,CAAmB,MAAM,CAAC,CAAA,CAAA;AAAA,UACrD;AAAA,SACF;AAAA,MACF;AAEA,MAAA,IAAI,CAAC,oBAAA,EAAsB;AACzB,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,CAAA,yCAAA,EAA4CA,+BAAA;AAAA,YAC1C;AAAA,WACD,CAAA;AAAA,SACH;AAAA,MACF;AAEA,MAAA,OAAO,oBAAA;AAAA,IACT,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,eAAA,CACZ,MAAA,EACA,OAAA,EACe;AACf,IAAA,OAAO,MAAMQ,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA,CAAa,qCAAqC,UAAU,CAAA;AAGtE,MAAA,IAAIT,+BAAA,CAAmB,MAAM,CAAA,KAAM,OAAA,CAAQ,SAAA,EAAW;AACpD,QAAA,MAAM,IAAIU,oBAAA;AAAA,UACR;AAAA,SACF;AAAA,MACF;AAGA,MAAA,IAAI;AACF,QAAAC,mBAAA,CAAe,MAAM,CAAA;AAAA,MACvB,SAAS,CAAA,EAAG;AACV,QAAA,MAAM,IAAID,oBAAA;AAAA,UACR,CAAA,oBAAA,EAAuB,QAAQ,SAAS,CAAA,sCAAA,CAAA;AAAA,UACxC;AAAA,SACF;AAAA,MACF;AAEA,MAAA,IAAI,KAAA,GAAQ,KAAA;AAEZ,MAAA,KAAA,MAAW,SAAA,IAAa,IAAA,CAAK,OAAA,CAAQ,UAAA,EAAY;AAC/C,QAAA,IAAI,UAAU,kBAAA,EAAoB;AAChC,UAAA,IAAI;AACF,YAAA,MAAM,YAAY,MAAMF,4BAAA;AAAA,cACtB,MAAA;AAAA,cACA,gBAAA;AAAA,cACA,OAAM,IAAA,KAAQ;AACZ,gBAAAC,iCAAA,CAAoB,MAAM,MAAM,CAAA;AAChC,gBAAA,sBAAA,CAAuB,IAAA,EAAM,sBAAsB,SAAS,CAAA;AAC5D,gBAAA,OAAO,MAAM,SAAA,CAAU,kBAAA,CAAoB,MAAM,CAAA;AAAA,cACnD;AAAA,aACF;AACA,YAAA,IAAI,SAAA,EAAW;AACb,cAAA,KAAA,GAAQ,IAAA;AAAA,YACV;AAAA,UACF,SAAS,CAAA,EAAG;AACV,YAAA,MAAM,IAAIV,iBAAA;AAAA,cACR,aAAa,SAAA,CAAU,WAAA,CAAY,IAAI,CAAA,4CAAA,EAA+C,QAAQ,SAAS,CAAA,CAAA;AAAA,cACvG;AAAA,aACF;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAEA,MAAA,IAAI,CAAC,KAAA,EAAO;AACV,QAAA,MAAM,IAAIA,iBAAA;AAAA,UACR,CAAA,mCAAA,EAAsC,QAAQ,SAAS,CAAA,0DAAA;AAAA,SACzD;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,sBAAA,CACZ,MAAA,EACA,OAAA,EACe;AACf,IAAA,OAAO,MAAMS,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA;AAAA,QACR,mCAAA;AAAA,QACA;AAAA,OACF;AACA,MAAA,MAAM,EAAE,OAAO,OAAA,CAAQ,QAAA,CAAS,MAAM,QAAA,GAAW,UAAA,KAC/C,MAAA,CAAO,IAAA;AACT,MAAA,MAAM,OAAA,GAAU,IAAI,KAAA,EAAc;AAClC,MAAA,IAAI,MAAA,CAAO,KAAK,MAAA,EAAQ;AACtB,QAAA,OAAA,CAAQ,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,MAAM,CAAA;AAAA,MACjC;AACA,MAAA,IAAI,MAAA,CAAO,KAAK,OAAA,EAAS;AACvB,QAAA,OAAA,CAAQ,IAAA,CAAK,GAAG,MAAA,CAAO,IAAA,CAAK,OAAO,CAAA;AAAA,MACrC;AAEA,MAAA,KAAA,MAAW,uBAAuB,OAAA,EAAS;AACzC,QAAA,IAAI,SAAS,MAAA,IAAU,mBAAA,CAAoB,QAAA,CAASG,qBAAA,CAAK,GAAG,CAAA,EAAG;AAC7D,UAAA,OAAA,CAAQ,UAAU,OAAA,EAAQ;AAAA,YACxBC,kCAAA,CAAiB,UAAA;AAAA,cACf,OAAA,CAAQ,QAAA;AAAA,cACR,CAAA,sCAAA,EAAyC,IAAI,CAAA,2BAAA,EAA8B,OAAA,CAAQ,SAAS,MAAM,CAAA,gCAAA;AAAA;AACpG,WACF;AACA,UAAA;AAAA,QACF;AACA,QAAA,MAAM,MAAA,GAASC,kBAAA;AAAA,UACb,KAAK,OAAA,CAAQ,YAAA;AAAA,UACb,OAAA,CAAQ,QAAA;AAAA,UACR,IAAA;AAAA,UACA;AAAA,SACF;AAEA,QAAA,IAAI,OAAA,GAAU,KAAA;AACd,QAAA,KAAA,MAAW,SAAA,IAAa,IAAA,CAAK,OAAA,CAAQ,UAAA,EAAY;AAC/C,UAAA,IAAI,UAAU,YAAA,EAAc;AAC1B,YAAA,IAAI;AACF,cAAA,MAAM,OAAO,MAAMN,4BAAA;AAAA,gBACjB,MAAA;AAAA,gBACA,gBAAA;AAAA,gBACA,OAAM,IAAA,KAAQ;AACZ,kBAAAC,iCAAA,CAAoB,MAAM,MAAM,CAAA;AAChC,kBAAA,sBAAA,CAAuB,IAAA,EAAM,gBAAgB,SAAS,CAAA;AACtD,kBAAA,OAAO,MAAM,SAAA,CAAU,YAAA;AAAA,oBACrB;AAAA,sBACE,IAAA;AAAA,sBACA,MAAA;AAAA,sBACA;AAAA,qBACF;AAAA,oBACA,QAAA,KAAa,UAAA;AAAA,oBACb,OAAA,CAAQ,SAAA,CAAU,YAAA,CAAa,SAAS,CAAA;AAAA,oBACxC,KAAK,OAAA,CAAQ,MAAA;AAAA,oBACb,OAAA,CAAQ,KAAA,CAAM,YAAA,CAAa,SAAA,EAAW,MAAM;AAAA,mBAC9C;AAAA,gBACF;AAAA,eACF;AACA,cAAA,IAAI,IAAA,EAAM;AACR,gBAAA,OAAA,GAAU,IAAA;AACV,gBAAA;AAAA,cACF;AAAA,YACF,SAAS,CAAA,EAAG;AACV,cAAA,MAAM,IAAIV,iBAAA;AAAA,gBACR,aAAa,SAAA,CAAU,WAAA,CAAY,IAAI,CAAA,8BAAA,EAAiC,IAAI,IAAI,MAAM,CAAA,CAAA;AAAA,gBACtF;AAAA,eACF;AAAA,YACF;AAAA,UACF;AAAA,QACF;AACA,QAAA,IAAI,CAAC,OAAA,EAAS;AACZ,UAAA,MAAM,IAAIA,iBAAA;AAAA,YACR,CAAA,2CAAA,EAA8C,IAAI,CAAA,CAAA,EAAI,MAAM,CAAA;AAAA,WAC9D;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,kBAAA,CACZ,MAAA,EACA,OAAA,EACiB;AACjB,IAAA,OAAO,MAAMS,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA;AAAA,QACR,mCAAA;AAAA,QACA;AAAA,OACF;AACA,MAAA,IAAI,GAAA,GAAM,MAAA;AAEV,MAAA,KAAA,MAAW,SAAA,IAAa,IAAA,CAAK,OAAA,CAAQ,UAAA,EAAY;AAC/C,QAAA,IAAI,UAAU,iBAAA,EAAmB;AAC/B,UAAA,IAAI,QAAA,GAAW,GAAA;AACf,UAAA,GAAA,GAAM,MAAMD,4BAAA,CAAe,MAAA,EAAQ,gBAAA,EAAkB,OAAM,IAAA,KAAQ;AACjE,YAAAC,iCAAA,CAAoB,MAAM,MAAM,CAAA;AAChC,YAAA,sBAAA,CAAuB,IAAA,EAAM,qBAAqB,SAAS,CAAA;AAC3D,YAAA,IAAI;AACF,cAAA,QAAA,GAAW,MAAM,SAAA,CAAU,iBAAA;AAAA,gBACzB,QAAA;AAAA,gBACA,OAAA,CAAQ,QAAA;AAAA,gBACR,OAAA,CAAQ,SAAA,CAAU,YAAA,CAAa,SAAS,CAAA;AAAA,gBACxC,OAAA,CAAQ,KAAA,CAAM,YAAA,CAAa,SAAS;AAAA,eACtC;AAAA,YACF,SAAS,CAAA,EAAG;AACV,cAAA,MAAM,IAAIV,iBAAA;AAAA,gBACR,CAAA,UAAA,EAAa,SAAA,CAAU,WAAA,CAAY,IAAI,CAAA,oCAAA,CAAA;AAAA,gBACvC;AAAA,eACF;AAAA,YACF;AACA,YAAA,OAAO,QAAA;AAAA,UACT,CAAC,CAAA;AAAA,QACH;AAAA,MACF;AAEA,MAAA,OAAO,GAAA;AAAA,IACT,CAAC,CAAA;AAAA,EACH;AACF;;;;"}
+{"version":3,"file":"DefaultCatalogProcessingOrchestrator.cjs.js","sources":["../../src/processing/DefaultCatalogProcessingOrchestrator.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Span, trace } from '@opentelemetry/api';\nimport {\n  Entity,\n  EntityPolicy,\n  LocationEntity,\n  parseLocationRef,\n  stringifyEntityRef,\n  stringifyLocationRef,\n} from '@backstage/catalog-model';\nimport {\n  assertError,\n  ConflictError,\n  InputError,\n  NotAllowedError,\n} from '@backstage/errors';\nimport { JsonValue } from '@backstage/types';\nimport { ScmIntegrationRegistry } from '@backstage/integration';\nimport path from 'path';\nimport { LocationSpec } from '@backstage/plugin-catalog-common';\nimport {\n  CatalogProcessor,\n  CatalogProcessorParser,\n  processingResult,\n} from '@backstage/plugin-catalog-node';\nimport {\n  CatalogProcessingOrchestrator,\n  EntityProcessingRequest,\n  EntityProcessingResult,\n} from './types';\nimport { ProcessorOutputCollector } from './ProcessorOutputCollector';\nimport {\n  getEntityLocationRef,\n  getEntityOriginLocationRef,\n  isLocationEntity,\n  isObject,\n  toAbsoluteUrl,\n  validateEntity,\n  validateEntityEnvelope,\n} from './util';\nimport { CatalogRulesEnforcer } from '../ingestion/CatalogRules';\nimport { ProcessorCacheManager } from './ProcessorCacheManager';\nimport {\n  addEntityAttributes,\n  TRACER_ID,\n  withActiveSpan,\n} from '../util/opentelemetry';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\nconst tracer = trace.getTracer(TRACER_ID);\n\ntype Context = {\n  entityRef: string;\n  location: LocationSpec;\n  originLocation: LocationSpec;\n  collector: ProcessorOutputCollector;\n  cache: ProcessorCacheManager;\n};\n\nfunction addProcessorAttributes(\n  span: Span,\n  stage: string,\n  processor: CatalogProcessor,\n) {\n  span.setAttribute('backstage.catalog.processor.stage', stage);\n  span.setAttribute(\n    'backstage.catalog.processor.name',\n    processor.getProcessorName(),\n  );\n}\n\nexport class DefaultCatalogProcessingOrchestrator\n  implements CatalogProcessingOrchestrator\n{\n  constructor(\n    private readonly options: {\n      processors: CatalogProcessor[];\n      integrations: ScmIntegrationRegistry;\n      logger: LoggerService;\n      parser: CatalogProcessorParser;\n      policy: EntityPolicy;\n      rulesEnforcer: CatalogRulesEnforcer;\n      legacySingleProcessorValidation: boolean;\n    },\n  ) {}\n\n  async process(\n    request: EntityProcessingRequest,\n  ): Promise<EntityProcessingResult> {\n    return this.processSingleEntity(request.entity, request.state);\n  }\n\n  private async processSingleEntity(\n    unprocessedEntity: Entity,\n    state: JsonValue | undefined,\n  ): Promise<EntityProcessingResult> {\n    const collector = new ProcessorOutputCollector(\n      this.options.logger,\n      unprocessedEntity,\n    );\n\n    // Cache that is scoped to the entity and processor\n    const cache = new ProcessorCacheManager(\n      isObject(state) && isObject(state.cache) ? state.cache : {},\n    );\n\n    try {\n      // This will be checked and mutated step by step below\n      let entity: Entity = unprocessedEntity;\n\n      // NOTE: At this early point, we can only rely on the envelope having to\n      // be valid; full entity + kind validation happens after the (potentially\n      // mutative) pre-steps. This means that the code below can't make a lot\n      // of assumptions about the data despite it using the Entity type.\n      try {\n        validateEntityEnvelope(entity);\n      } catch (e) {\n        throw new InputError(\n          `Entity envelope failed validation before processing`,\n          e,\n        );\n      }\n\n      // TODO: which one do we actually use for the location?\n      // source-location? - maybe probably doesn't exist yet?\n      const context: Context = {\n        entityRef: stringifyEntityRef(entity),\n        location: parseLocationRef(getEntityLocationRef(entity)),\n        originLocation: parseLocationRef(getEntityOriginLocationRef(entity)),\n        cache,\n        collector,\n      };\n\n      // Run the steps\n      entity = await this.runPreProcessStep(entity, context);\n      entity = await this.runPolicyStep(entity);\n      await this.runValidateStep(entity, context);\n      if (isLocationEntity(entity)) {\n        await this.runSpecialLocationStep(entity, context);\n      }\n      entity = await this.runPostProcessStep(entity, context);\n\n      // Check that any emitted entities are permitted to originate from that\n      // particular location according to the catalog rules\n      const collectorResults = context.collector.results();\n      for (const deferredEntity of collectorResults.deferredEntities) {\n        if (\n          !this.options.rulesEnforcer.isAllowed(\n            deferredEntity.entity,\n            context.originLocation,\n          )\n        ) {\n          throw new NotAllowedError(\n            `Entity ${stringifyEntityRef(\n              deferredEntity.entity,\n            )} at ${stringifyLocationRef(\n              context.location,\n            )}, originated at ${stringifyLocationRef(\n              context.originLocation,\n            )}, is not of an allowed kind for that location`,\n          );\n        }\n      }\n\n      return {\n        ...collectorResults,\n        completedEntity: entity,\n        state: { cache: cache.collect() },\n        ok: collectorResults.errors.length === 0,\n      };\n    } catch (error) {\n      assertError(error);\n      return {\n        ok: false,\n        errors: collector.results().errors.concat(error),\n      };\n    }\n  }\n\n  // Pre-process phase, used to populate entities with data that is required\n  // during the main processing step\n  private async runPreProcessStep(\n    entity: Entity,\n    context: Context,\n  ): Promise<Entity> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute('backstage.catalog.processor.stage', 'preProcess');\n      let res = entity;\n\n      for (const processor of this.options.processors) {\n        if (processor.preProcessEntity) {\n          let innerRes = res;\n          res = await withActiveSpan(tracer, 'ProcessingStep', async span => {\n            addEntityAttributes(span, entity);\n            addProcessorAttributes(span, 'preProcessEntity', processor);\n            try {\n              innerRes = await processor.preProcessEntity!(\n                innerRes,\n                context.location,\n                context.collector.forProcessor(processor),\n                context.originLocation,\n                context.cache.forProcessor(processor),\n              );\n            } catch (e) {\n              throw new InputError(\n                `Processor ${processor.constructor.name} threw an error while preprocessing`,\n                e,\n              );\n            }\n            return innerRes;\n          });\n        }\n      }\n\n      return res;\n    });\n  }\n\n  /**\n   * Enforce entity policies making sure that entities conform to a general schema\n   */\n  private async runPolicyStep(entity: Entity): Promise<Entity> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute(\n        'backstage.catalog.processor.stage',\n        'enforcePolicy',\n      );\n      let policyEnforcedEntity: Entity | undefined;\n\n      try {\n        policyEnforcedEntity = await this.options.policy.enforce(entity);\n      } catch (e) {\n        throw new InputError(\n          `Policy check failed for ${stringifyEntityRef(entity)}`,\n          e,\n        );\n      }\n\n      if (!policyEnforcedEntity) {\n        throw new Error(\n          `Policy unexpectedly returned no data for ${stringifyEntityRef(\n            entity,\n          )}`,\n        );\n      }\n\n      return policyEnforcedEntity;\n    });\n  }\n\n  /**\n   * Validate the given entity\n   */\n  private async runValidateStep(\n    entity: Entity,\n    context: Context,\n  ): Promise<void> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute('backstage.catalog.processor.stage', 'validate');\n      // Double check that none of the previous steps tried to change something\n      // related to the entity ref, which would break downstream\n      if (stringifyEntityRef(entity) !== context.entityRef) {\n        throw new ConflictError(\n          'Fatal: The entity kind, namespace, or name changed during processing',\n        );\n      }\n\n      // Validate that the end result is a valid Entity at all\n      try {\n        validateEntity(entity);\n      } catch (e) {\n        throw new ConflictError(\n          `Entity envelope for ${context.entityRef} failed validation after preprocessing`,\n          e,\n        );\n      }\n\n      let valid = false;\n\n      for (const processor of this.options.processors) {\n        if (processor.validateEntityKind) {\n          try {\n            const thisValid = await withActiveSpan(\n              tracer,\n              'ProcessingStep',\n              async span => {\n                addEntityAttributes(span, entity);\n                addProcessorAttributes(span, 'validateEntityKind', processor);\n                return await processor.validateEntityKind!(entity);\n              },\n            );\n            if (thisValid) {\n              valid = true;\n              if (this.options.legacySingleProcessorValidation) {\n                break;\n              }\n            }\n          } catch (e) {\n            throw new InputError(\n              `Processor ${processor.constructor.name} threw an error while validating the entity ${context.entityRef}`,\n              e,\n            );\n          }\n        }\n      }\n\n      if (!valid) {\n        throw new InputError(\n          `No processor recognized the entity ${context.entityRef} as valid, possibly caused by a foreign kind or apiVersion`,\n        );\n      }\n    });\n  }\n\n  /**\n   * Backwards compatible processing of location entities\n   */\n  private async runSpecialLocationStep(\n    entity: LocationEntity,\n    context: Context,\n  ): Promise<void> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute(\n        'backstage.catalog.processor.stage',\n        'readLocation',\n      );\n      const { type = context.location.type, presence = 'required' } =\n        entity.spec;\n      const targets = new Array<string>();\n      if (entity.spec.target) {\n        targets.push(entity.spec.target);\n      }\n      if (entity.spec.targets) {\n        targets.push(...entity.spec.targets);\n      }\n\n      for (const maybeRelativeTarget of targets) {\n        if (type === 'file' && maybeRelativeTarget.endsWith(path.sep)) {\n          context.collector.generic()(\n            processingResult.inputError(\n              context.location,\n              `LocationEntityProcessor cannot handle ${type} type location with target ${context.location.target} that ends with a path separator`,\n            ),\n          );\n          continue;\n        }\n        const target = toAbsoluteUrl(\n          this.options.integrations,\n          context.location,\n          type,\n          maybeRelativeTarget,\n        );\n\n        let didRead = false;\n        for (const processor of this.options.processors) {\n          if (processor.readLocation) {\n            try {\n              const read = await withActiveSpan(\n                tracer,\n                'ProcessingStep',\n                async span => {\n                  addEntityAttributes(span, entity);\n                  addProcessorAttributes(span, 'readLocation', processor);\n                  return await processor.readLocation!(\n                    {\n                      type,\n                      target,\n                      presence,\n                    },\n                    presence === 'optional',\n                    context.collector.forProcessor(processor),\n                    this.options.parser,\n                    context.cache.forProcessor(processor, target),\n                  );\n                },\n              );\n              if (read) {\n                didRead = true;\n                break;\n              }\n            } catch (e) {\n              throw new InputError(\n                `Processor ${processor.constructor.name} threw an error while reading ${type}:${target}`,\n                e,\n              );\n            }\n          }\n        }\n        if (!didRead) {\n          throw new InputError(\n            `No processor was able to handle reading of ${type}:${target}`,\n          );\n        }\n      }\n    });\n  }\n\n  /**\n   * Main processing step of the entity\n   */\n  private async runPostProcessStep(\n    entity: Entity,\n    context: Context,\n  ): Promise<Entity> {\n    return await withActiveSpan(tracer, 'ProcessingStage', async stageSpan => {\n      addEntityAttributes(stageSpan, entity);\n      stageSpan.setAttribute(\n        'backstage.catalog.processor.stage',\n        'postProcessEntity',\n      );\n      let res = entity;\n\n      for (const processor of this.options.processors) {\n        if (processor.postProcessEntity) {\n          let innerRes = res;\n          res = await withActiveSpan(tracer, 'ProcessingStep', async span => {\n            addEntityAttributes(span, entity);\n            addProcessorAttributes(span, 'postProcessEntity', processor);\n            try {\n              innerRes = await processor.postProcessEntity!(\n                innerRes,\n                context.location,\n                context.collector.forProcessor(processor),\n                context.cache.forProcessor(processor),\n              );\n            } catch (e) {\n              throw new InputError(\n                `Processor ${processor.constructor.name} threw an error while postprocessing`,\n                e,\n              );\n            }\n            return innerRes;\n          });\n        }\n      }\n\n      return res;\n    });\n  }\n}\n"],"names":["trace","TRACER_ID","ProcessorOutputCollector","ProcessorCacheManager","isObject","validateEntityEnvelope","InputError","stringifyEntityRef","parseLocationRef","getEntityLocationRef","getEntityOriginLocationRef","isLocationEntity","NotAllowedError","stringifyLocationRef","assertError","withActiveSpan","addEntityAttributes","ConflictError","validateEntity","path","processingResult","toAbsoluteUrl"],"mappings":";;;;;;;;;;;;;;;;AAgEA,MAAM,MAAA,GAASA,SAAA,CAAM,SAAA,CAAUC,uBAAS,CAAA;AAUxC,SAAS,sBAAA,CACP,IAAA,EACA,KAAA,EACA,SAAA,EACA;AACA,EAAA,IAAA,CAAK,YAAA,CAAa,qCAAqC,KAAK,CAAA;AAC5D,EAAA,IAAA,CAAK,YAAA;AAAA,IACH,kCAAA;AAAA,IACA,UAAU,gBAAA;AAAiB,GAC7B;AACF;AAEO,MAAM,oCAAA,CAEb;AAAA,EACE,YACmB,OAAA,EASjB;AATiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA,EAShB;AAAA,EAEH,MAAM,QACJ,OAAA,EACiC;AACjC,IAAA,OAAO,IAAA,CAAK,mBAAA,CAAoB,OAAA,CAAQ,MAAA,EAAQ,QAAQ,KAAK,CAAA;AAAA,EAC/D;AAAA,EAEA,MAAc,mBAAA,CACZ,iBAAA,EACA,KAAA,EACiC;AACjC,IAAA,MAAM,YAAY,IAAIC,iDAAA;AAAA,MACpB,KAAK,OAAA,CAAQ,MAAA;AAAA,MACb;AAAA,KACF;AAGA,IAAA,MAAM,QAAQ,IAAIC,2CAAA;AAAA,MAChBC,aAAA,CAAS,KAAK,CAAA,IAAKA,aAAA,CAAS,MAAM,KAAK,CAAA,GAAI,KAAA,CAAM,KAAA,GAAQ;AAAC,KAC5D;AAEA,IAAA,IAAI;AAEF,MAAA,IAAI,MAAA,GAAiB,iBAAA;AAMrB,MAAA,IAAI;AACF,QAAAC,2BAAA,CAAuB,MAAM,CAAA;AAAA,MAC/B,SAAS,CAAA,EAAG;AACV,QAAA,MAAM,IAAIC,iBAAA;AAAA,UACR,CAAA,mDAAA,CAAA;AAAA,UACA;AAAA,SACF;AAAA,MACF;AAIA,MAAA,MAAM,OAAA,GAAmB;AAAA,QACvB,SAAA,EAAWC,gCAAmB,MAAM,CAAA;AAAA,QACpC,QAAA,EAAUC,6BAAA,CAAiBC,yBAAA,CAAqB,MAAM,CAAC,CAAA;AAAA,QACvD,cAAA,EAAgBD,6BAAA,CAAiBE,+BAAA,CAA2B,MAAM,CAAC,CAAA;AAAA,QACnE,KAAA;AAAA,QACA;AAAA,OACF;AAGA,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,iBAAA,CAAkB,MAAA,EAAQ,OAAO,CAAA;AACrD,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,aAAA,CAAc,MAAM,CAAA;AACxC,MAAA,MAAM,IAAA,CAAK,eAAA,CAAgB,MAAA,EAAQ,OAAO,CAAA;AAC1C,MAAA,IAAIC,qBAAA,CAAiB,MAAM,CAAA,EAAG;AAC5B,QAAA,MAAM,IAAA,CAAK,sBAAA,CAAuB,MAAA,EAAQ,OAAO,CAAA;AAAA,MACnD;AACA,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,kBAAA,CAAmB,MAAA,EAAQ,OAAO,CAAA;AAItD,MAAA,MAAM,gBAAA,GAAmB,OAAA,CAAQ,SAAA,CAAU,OAAA,EAAQ;AACnD,MAAA,KAAA,MAAW,cAAA,IAAkB,iBAAiB,gBAAA,EAAkB;AAC9D,QAAA,IACE,CAAC,IAAA,CAAK,OAAA,CAAQ,aAAA,CAAc,SAAA;AAAA,UAC1B,cAAA,CAAe,MAAA;AAAA,UACf,OAAA,CAAQ;AAAA,SACV,EACA;AACA,UAAA,MAAM,IAAIC,sBAAA;AAAA,YACR,CAAA,OAAA,EAAUL,+BAAA;AAAA,cACR,cAAA,CAAe;AAAA,aAChB,CAAA,IAAA,EAAOM,iCAAA;AAAA,cACN,OAAA,CAAQ;AAAA,aACT,CAAA,gBAAA,EAAmBA,iCAAA;AAAA,cAClB,OAAA,CAAQ;AAAA,aACT,CAAA,6CAAA;AAAA,WACH;AAAA,QACF;AAAA,MACF;AAEA,MAAA,OAAO;AAAA,QACL,GAAG,gBAAA;AAAA,QACH,eAAA,EAAiB,MAAA;AAAA,QACjB,KAAA,EAAO,EAAE,KAAA,EAAO,KAAA,CAAM,SAAQ,EAAE;AAAA,QAChC,EAAA,EAAI,gBAAA,CAAiB,MAAA,CAAO,MAAA,KAAW;AAAA,OACzC;AAAA,IACF,SAAS,KAAA,EAAO;AACd,MAAAC,kBAAA,CAAY,KAAK,CAAA;AACjB,MAAA,OAAO;AAAA,QACL,EAAA,EAAI,KAAA;AAAA,QACJ,QAAQ,SAAA,CAAU,OAAA,EAAQ,CAAE,MAAA,CAAO,OAAO,KAAK;AAAA,OACjD;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA,EAIA,MAAc,iBAAA,CACZ,MAAA,EACA,OAAA,EACiB;AACjB,IAAA,OAAO,MAAMC,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA,CAAa,qCAAqC,YAAY,CAAA;AACxE,MAAA,IAAI,GAAA,GAAM,MAAA;AAEV,MAAA,KAAA,MAAW,SAAA,IAAa,IAAA,CAAK,OAAA,CAAQ,UAAA,EAAY;AAC/C,QAAA,IAAI,UAAU,gBAAA,EAAkB;AAC9B,UAAA,IAAI,QAAA,GAAW,GAAA;AACf,UAAA,GAAA,GAAM,MAAMD,4BAAA,CAAe,MAAA,EAAQ,gBAAA,EAAkB,OAAM,IAAA,KAAQ;AACjE,YAAAC,iCAAA,CAAoB,MAAM,MAAM,CAAA;AAChC,YAAA,sBAAA,CAAuB,IAAA,EAAM,oBAAoB,SAAS,CAAA;AAC1D,YAAA,IAAI;AACF,cAAA,QAAA,GAAW,MAAM,SAAA,CAAU,gBAAA;AAAA,gBACzB,QAAA;AAAA,gBACA,OAAA,CAAQ,QAAA;AAAA,gBACR,OAAA,CAAQ,SAAA,CAAU,YAAA,CAAa,SAAS,CAAA;AAAA,gBACxC,OAAA,CAAQ,cAAA;AAAA,gBACR,OAAA,CAAQ,KAAA,CAAM,YAAA,CAAa,SAAS;AAAA,eACtC;AAAA,YACF,SAAS,CAAA,EAAG;AACV,cAAA,MAAM,IAAIV,iBAAA;AAAA,gBACR,CAAA,UAAA,EAAa,SAAA,CAAU,WAAA,CAAY,IAAI,CAAA,mCAAA,CAAA;AAAA,gBACvC;AAAA,eACF;AAAA,YACF;AACA,YAAA,OAAO,QAAA;AAAA,UACT,CAAC,CAAA;AAAA,QACH;AAAA,MACF;AAEA,MAAA,OAAO,GAAA;AAAA,IACT,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,cAAc,MAAA,EAAiC;AAC3D,IAAA,OAAO,MAAMS,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA;AAAA,QACR,mCAAA;AAAA,QACA;AAAA,OACF;AACA,MAAA,IAAI,oBAAA;AAEJ,MAAA,IAAI;AACF,QAAA,oBAAA,GAAuB,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,QAAQ,MAAM,CAAA;AAAA,MACjE,SAAS,CAAA,EAAG;AACV,QAAA,MAAM,IAAIV,iBAAA;AAAA,UACR,CAAA,wBAAA,EAA2BC,+BAAA,CAAmB,MAAM,CAAC,CAAA,CAAA;AAAA,UACrD;AAAA,SACF;AAAA,MACF;AAEA,MAAA,IAAI,CAAC,oBAAA,EAAsB;AACzB,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,CAAA,yCAAA,EAA4CA,+BAAA;AAAA,YAC1C;AAAA,WACD,CAAA;AAAA,SACH;AAAA,MACF;AAEA,MAAA,OAAO,oBAAA;AAAA,IACT,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,eAAA,CACZ,MAAA,EACA,OAAA,EACe;AACf,IAAA,OAAO,MAAMQ,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA,CAAa,qCAAqC,UAAU,CAAA;AAGtE,MAAA,IAAIT,+BAAA,CAAmB,MAAM,CAAA,KAAM,OAAA,CAAQ,SAAA,EAAW;AACpD,QAAA,MAAM,IAAIU,oBAAA;AAAA,UACR;AAAA,SACF;AAAA,MACF;AAGA,MAAA,IAAI;AACF,QAAAC,mBAAA,CAAe,MAAM,CAAA;AAAA,MACvB,SAAS,CAAA,EAAG;AACV,QAAA,MAAM,IAAID,oBAAA;AAAA,UACR,CAAA,oBAAA,EAAuB,QAAQ,SAAS,CAAA,sCAAA,CAAA;AAAA,UACxC;AAAA,SACF;AAAA,MACF;AAEA,MAAA,IAAI,KAAA,GAAQ,KAAA;AAEZ,MAAA,KAAA,MAAW,SAAA,IAAa,IAAA,CAAK,OAAA,CAAQ,UAAA,EAAY;AAC/C,QAAA,IAAI,UAAU,kBAAA,EAAoB;AAChC,UAAA,IAAI;AACF,YAAA,MAAM,YAAY,MAAMF,4BAAA;AAAA,cACtB,MAAA;AAAA,cACA,gBAAA;AAAA,cACA,OAAM,IAAA,KAAQ;AACZ,gBAAAC,iCAAA,CAAoB,MAAM,MAAM,CAAA;AAChC,gBAAA,sBAAA,CAAuB,IAAA,EAAM,sBAAsB,SAAS,CAAA;AAC5D,gBAAA,OAAO,MAAM,SAAA,CAAU,kBAAA,CAAoB,MAAM,CAAA;AAAA,cACnD;AAAA,aACF;AACA,YAAA,IAAI,SAAA,EAAW;AACb,cAAA,KAAA,GAAQ,IAAA;AACR,cAAA,IAAI,IAAA,CAAK,QAAQ,+BAAA,EAAiC;AAChD,gBAAA;AAAA,cACF;AAAA,YACF;AAAA,UACF,SAAS,CAAA,EAAG;AACV,YAAA,MAAM,IAAIV,iBAAA;AAAA,cACR,aAAa,SAAA,CAAU,WAAA,CAAY,IAAI,CAAA,4CAAA,EAA+C,QAAQ,SAAS,CAAA,CAAA;AAAA,cACvG;AAAA,aACF;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAEA,MAAA,IAAI,CAAC,KAAA,EAAO;AACV,QAAA,MAAM,IAAIA,iBAAA;AAAA,UACR,CAAA,mCAAA,EAAsC,QAAQ,SAAS,CAAA,0DAAA;AAAA,SACzD;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,sBAAA,CACZ,MAAA,EACA,OAAA,EACe;AACf,IAAA,OAAO,MAAMS,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA;AAAA,QACR,mCAAA;AAAA,QACA;AAAA,OACF;AACA,MAAA,MAAM,EAAE,OAAO,OAAA,CAAQ,QAAA,CAAS,MAAM,QAAA,GAAW,UAAA,KAC/C,MAAA,CAAO,IAAA;AACT,MAAA,MAAM,OAAA,GAAU,IAAI,KAAA,EAAc;AAClC,MAAA,IAAI,MAAA,CAAO,KAAK,MAAA,EAAQ;AACtB,QAAA,OAAA,CAAQ,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,MAAM,CAAA;AAAA,MACjC;AACA,MAAA,IAAI,MAAA,CAAO,KAAK,OAAA,EAAS;AACvB,QAAA,OAAA,CAAQ,IAAA,CAAK,GAAG,MAAA,CAAO,IAAA,CAAK,OAAO,CAAA;AAAA,MACrC;AAEA,MAAA,KAAA,MAAW,uBAAuB,OAAA,EAAS;AACzC,QAAA,IAAI,SAAS,MAAA,IAAU,mBAAA,CAAoB,QAAA,CAASG,qBAAA,CAAK,GAAG,CAAA,EAAG;AAC7D,UAAA,OAAA,CAAQ,UAAU,OAAA,EAAQ;AAAA,YACxBC,kCAAA,CAAiB,UAAA;AAAA,cACf,OAAA,CAAQ,QAAA;AAAA,cACR,CAAA,sCAAA,EAAyC,IAAI,CAAA,2BAAA,EAA8B,OAAA,CAAQ,SAAS,MAAM,CAAA,gCAAA;AAAA;AACpG,WACF;AACA,UAAA;AAAA,QACF;AACA,QAAA,MAAM,MAAA,GAASC,kBAAA;AAAA,UACb,KAAK,OAAA,CAAQ,YAAA;AAAA,UACb,OAAA,CAAQ,QAAA;AAAA,UACR,IAAA;AAAA,UACA;AAAA,SACF;AAEA,QAAA,IAAI,OAAA,GAAU,KAAA;AACd,QAAA,KAAA,MAAW,SAAA,IAAa,IAAA,CAAK,OAAA,CAAQ,UAAA,EAAY;AAC/C,UAAA,IAAI,UAAU,YAAA,EAAc;AAC1B,YAAA,IAAI;AACF,cAAA,MAAM,OAAO,MAAMN,4BAAA;AAAA,gBACjB,MAAA;AAAA,gBACA,gBAAA;AAAA,gBACA,OAAM,IAAA,KAAQ;AACZ,kBAAAC,iCAAA,CAAoB,MAAM,MAAM,CAAA;AAChC,kBAAA,sBAAA,CAAuB,IAAA,EAAM,gBAAgB,SAAS,CAAA;AACtD,kBAAA,OAAO,MAAM,SAAA,CAAU,YAAA;AAAA,oBACrB;AAAA,sBACE,IAAA;AAAA,sBACA,MAAA;AAAA,sBACA;AAAA,qBACF;AAAA,oBACA,QAAA,KAAa,UAAA;AAAA,oBACb,OAAA,CAAQ,SAAA,CAAU,YAAA,CAAa,SAAS,CAAA;AAAA,oBACxC,KAAK,OAAA,CAAQ,MAAA;AAAA,oBACb,OAAA,CAAQ,KAAA,CAAM,YAAA,CAAa,SAAA,EAAW,MAAM;AAAA,mBAC9C;AAAA,gBACF;AAAA,eACF;AACA,cAAA,IAAI,IAAA,EAAM;AACR,gBAAA,OAAA,GAAU,IAAA;AACV,gBAAA;AAAA,cACF;AAAA,YACF,SAAS,CAAA,EAAG;AACV,cAAA,MAAM,IAAIV,iBAAA;AAAA,gBACR,aAAa,SAAA,CAAU,WAAA,CAAY,IAAI,CAAA,8BAAA,EAAiC,IAAI,IAAI,MAAM,CAAA,CAAA;AAAA,gBACtF;AAAA,eACF;AAAA,YACF;AAAA,UACF;AAAA,QACF;AACA,QAAA,IAAI,CAAC,OAAA,EAAS;AACZ,UAAA,MAAM,IAAIA,iBAAA;AAAA,YACR,CAAA,2CAAA,EAA8C,IAAI,CAAA,CAAA,EAAI,MAAM,CAAA;AAAA,WAC9D;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,kBAAA,CACZ,MAAA,EACA,OAAA,EACiB;AACjB,IAAA,OAAO,MAAMS,4BAAA,CAAe,MAAA,EAAQ,iBAAA,EAAmB,OAAM,SAAA,KAAa;AACxE,MAAAC,iCAAA,CAAoB,WAAW,MAAM,CAAA;AACrC,MAAA,SAAA,CAAU,YAAA;AAAA,QACR,mCAAA;AAAA,QACA;AAAA,OACF;AACA,MAAA,IAAI,GAAA,GAAM,MAAA;AAEV,MAAA,KAAA,MAAW,SAAA,IAAa,IAAA,CAAK,OAAA,CAAQ,UAAA,EAAY;AAC/C,QAAA,IAAI,UAAU,iBAAA,EAAmB;AAC/B,UAAA,IAAI,QAAA,GAAW,GAAA;AACf,UAAA,GAAA,GAAM,MAAMD,4BAAA,CAAe,MAAA,EAAQ,gBAAA,EAAkB,OAAM,IAAA,KAAQ;AACjE,YAAAC,iCAAA,CAAoB,MAAM,MAAM,CAAA;AAChC,YAAA,sBAAA,CAAuB,IAAA,EAAM,qBAAqB,SAAS,CAAA;AAC3D,YAAA,IAAI;AACF,cAAA,QAAA,GAAW,MAAM,SAAA,CAAU,iBAAA;AAAA,gBACzB,QAAA;AAAA,gBACA,OAAA,CAAQ,QAAA;AAAA,gBACR,OAAA,CAAQ,SAAA,CAAU,YAAA,CAAa,SAAS,CAAA;AAAA,gBACxC,OAAA,CAAQ,KAAA,CAAM,YAAA,CAAa,SAAS;AAAA,eACtC;AAAA,YACF,SAAS,CAAA,EAAG;AACV,cAAA,MAAM,IAAIV,iBAAA;AAAA,gBACR,CAAA,UAAA,EAAa,SAAA,CAAU,WAAA,CAAY,IAAI,CAAA,oCAAA,CAAA;AAAA,gBACvC;AAAA,eACF;AAAA,YACF;AACA,YAAA,OAAO,QAAA;AAAA,UACT,CAAC,CAAA;AAAA,QACH;AAAA,MACF;AAEA,MAAA,OAAO,GAAA;AAAA,IACT,CAAC,CAAA;AAAA,EACH;AACF;;;;"}

package/dist/service/CatalogBuilder.cjs.js

@@ -67,6 +67,8 @@ class CatalogBuilder {
   permissions;
   permissionRules;
   allowedLocationType;
+  legacySingleProcessorValidation = false;
+  eventBroker;
   /**
    * Creates a catalog builder.
    */
@@ -106,6 +108,27 @@ class CatalogBuilder {
     this.entityPolicies.push(...policies.flat());
     return this;
   }
+  /**
+   * Processing interval determines how often entities should be processed.
+   * Seconds provided will be multiplied by 1.5
+   * The default processing interval is 100-150 seconds.
+   * setting this too low will potentially deplete request quotas to upstream services.
+   */
+  setProcessingIntervalSeconds(seconds) {
+    this.processingInterval = refresh.createRandomProcessingInterval({
+      minSeconds: seconds,
+      maxSeconds: seconds * 1.5
+    });
+    return this;
+  }
+  /**
+   * Overwrites the default processing interval function used to spread
+   * entity updates in the catalog.
+   */
+  setProcessingInterval(processingInterval) {
+    this.processingInterval = processingInterval;
+    return this;
+  }
   /**
    * Overwrites the default location analyzer.
    */
@@ -265,6 +288,21 @@ class CatalogBuilder {
     this.allowedLocationType = allowedLocationTypes;
     return this;
   }
+  /**
+   * Enables the legacy behaviour of canceling validation early whenever only a
+   * single processor declares an entity kind to be valid.
+   */
+  useLegacySingleProcessorValidation() {
+    this.legacySingleProcessorValidation = true;
+    return this;
+  }
+  /**
+   * Enables the publishing of events for conflicts in the DefaultProcessingDatabase
+   */
+  setEventBroker(broker) {
+    this.eventBroker = broker;
+    return this;
+  }
   /**
    * Wires up and returns all of the component parts of the catalog
    */
@@ -278,8 +316,7 @@ class CatalogBuilder {
       permissionsRegistry,
       auditor,
       auth,
-      httpAuth,
-      events
+      httpAuth
     } = this.env;
     const enableRelationsCompatibility = Boolean(
       config.getOptionalBoolean("catalog.enableRelationsCompatibility")
@@ -299,8 +336,8 @@ class CatalogBuilder {
     const processingDatabase = new DefaultProcessingDatabase.DefaultProcessingDatabase({
       database: dbClient,
       logger,
-      events,
-      refreshInterval: this.processingInterval
+      refreshInterval: this.processingInterval,
+      eventBroker: this.eventBroker
     });
     const providerDatabase = new DefaultProviderDatabase.DefaultProviderDatabase({
       database: dbClient,
@@ -333,7 +370,8 @@ class CatalogBuilder {
       rulesEnforcer,
       logger,
       parser,
-      policy
+      policy,
+      legacySingleProcessorValidation: this.legacySingleProcessorValidation
     });
     const entitiesCatalog = new AuthorizedEntitiesCatalog.AuthorizedEntitiesCatalog(
       unauthorizedEntitiesCatalog,
@@ -389,7 +427,7 @@ class CatalogBuilder {
       onProcessingError: (event) => {
         this.onProcessingError?.(event);
       },
-      events
+      eventBroker: this.eventBroker
     });
     const locationAnalyzer = this.locationAnalyzer ?? new AuthorizedLocationAnalyzer.AuthorizedLocationAnalyzer(
       new LocationAnalyzer.RepoLocationAnalyzer(logger, integrations, this.locationAnalyzers),