@backstage/plugin-catalog-backend 3.9.1 → 4.0.0-next.1

package/CHANGELOG.md

@@ -1,14 +1,43 @@
 # @backstage/plugin-catalog-backend
 
-## 3.9.1
+## 4.0.0-next.1
+
+### Major Changes
+
+- d2cf99f: **BREAKING**: Removed the deprecated `catalog.enableRelationsCompatibility` config option and its associated compatibility layer. Entity relations are now always returned in the standard format with only `targetRef`. If you were relying on the `target` field in relations, update your code to use `targetRef` instead.
+
+### Patch Changes
+
+- f5119cc: Improved PostgreSQL catalog query planning by correcting entity cardinality statistics for the search table.
+- 72c16f5: Use stable PostgreSQL array parameters for `$in` filter predicates.
+- 7629e5e: Use stable PostgreSQL array parameters for batched refresh state lookups and updates.
+- Updated dependencies
+  - @backstage/plugin-catalog-common@1.2.0-next.0
+  - @backstage/config@1.3.9-next.0
+  - @backstage/catalog-client@1.16.2-next.1
+  - @backstage/plugin-catalog-node@2.2.5-next.1
+  - @backstage/backend-plugin-api@1.10.1-next.1
+  - @backstage/filter-predicates@0.1.5-next.1
+  - @backstage/integration@2.1.2-next.1
+  - @backstage/plugin-permission-common@0.9.11-next.0
+  - @backstage/plugin-permission-node@0.11.4-next.1
+
+## 3.9.2-next.0
 
 ### Patch Changes
 
-- 61a10f1: Improved path validation for catalog entity placeholders.
-- 21424fa: Correct catalog property permission matching.
-- 23774b6: Allowed location type restrictions are now applied consistently during catalog processing.
+- 7761a50: Improved path validation for catalog entity placeholders.
+- fda0ef1: Correct catalog property permission matching.
+- e363ae2: Allowed location type restrictions are now applied consistently during catalog processing.
 - Updated dependencies
-  - @backstage/integration@2.1.1
+  - @backstage/integration@2.1.2-next.0
+  - @backstage/backend-openapi-utils@0.7.2-next.0
+  - @backstage/filter-predicates@0.1.5-next.0
+  - @backstage/plugin-permission-node@0.11.4-next.0
+  - @backstage/catalog-client@1.16.2-next.0
+  - @backstage/backend-plugin-api@1.10.1-next.0
+  - @backstage/plugin-catalog-node@2.2.5-next.0
+  - @backstage/plugin-events-node@0.4.26-next.0
 
 ## 3.9.0
 

package/config.schema.json

@@ -110,10 +110,6 @@
           },
           "description": "A set of static locations that the catalog shall always keep itself up-to-date with. This is commonly used for large, permanent integrations that are defined by the Backstage operators at an organization, rather than individual things that users register dynamically.\n\nThese have (optional) rules of their own. These override what the global rules above specify. This way, you can prevent everybody from register e.g. User and Group entities, except for one or a few static locations that have those two kinds explicitly allowed.\n\nFor example:\n\n```yaml rules:   - allow: [Component, API, Template, Location] locations:   - type: url     target: https://github.com/org/repo/blob/master/users.yaml     rules:       - allow: [User, Group]   - type: url     target: https://github.com/org/repo/blob/master/systems.yaml     rules:       - allow: [System] ```"
         },
-        "enableRelationsCompatibility": {
-          "type": "boolean",
-          "description": "Enables the compatibility layer for relations in returned entities that ensures that all relations objects have both `target` and `targetRef`.\n\nEnabling this option significantly increases the memory usage of the catalog, and slightly reduces performance, but may avoid breaking consumers that rely on the existence of `target` in the relations objects."
-        },
         "disableDefaultProcessors": {
           "type": "boolean",
           "description": "Disables the default backstage processors.\n\nEnabling this option allows more complete control of which processors are included in the backstage processing loop."

package/dist/database/DefaultProcessingDatabase.cjs.js

@@ -117,9 +117,11 @@ class DefaultProcessingDatabase {
         }
       });
       if (items2.length > 0) {
-        await tx("refresh_state").whereIn(
-          "entity_ref",
-          items2.map((i) => i.entity_ref)
+        await tx("refresh_state").where(
+          util.whereInArray(
+            "entity_ref",
+            items2.map((i) => i.entity_ref)
+          )
         ).update({
           next_update_at: nextUpdateAt(tx, interval)
         });

package/dist/database/DefaultProcessingDatabase.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultProcessingDatabase.cjs.js","sources":["../../src/database/DefaultProcessingDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError } from '@backstage/errors';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport { ProcessingIntervalFunction } from '../processing/refresh';\nimport { rethrowError, timestampToDateTime } from './conversion';\nimport { initDatabaseMetrics } from './metrics';\nimport {\n  DbRefreshKeysRow,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n} from './tables';\nimport {\n  GetProcessableEntitiesResult,\n  ListParentsOptions,\n  ListParentsResult,\n  ProcessingDatabase,\n  RefreshStateItem,\n  Transaction,\n  UpdateEntityCacheOptions,\n  UpdateProcessedEntityOptions,\n} from './types';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { generateStableHash, generateTargetKey } from './util';\nimport {\n  syncRelations,\n  SyncRelationsResult,\n} from './operations/relations/syncRelations';\nimport { EventParams, EventsService } from '@backstage/plugin-events-node';\nimport { DateTime } from 'luxon';\nimport { CATALOG_CONFLICTS_TOPIC } from '../constants';\nimport { CatalogConflictEventPayload } from '../catalog/types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\nimport { MetricsService } from '@backstage/backend-plugin-api/alpha';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProcessingDatabase implements ProcessingDatabase {\n  private readonly options: {\n    database: Knex;\n    logger: LoggerService;\n    refreshInterval: ProcessingIntervalFunction;\n    events: EventsService;\n    metrics: MetricsService;\n  };\n\n  constructor(options: {\n    database: Knex;\n    logger: LoggerService;\n    refreshInterval: ProcessingIntervalFunction;\n    events: EventsService;\n    metrics: MetricsService;\n  }) {\n    this.options = options;\n    initDatabaseMetrics(options.database, options.metrics);\n  }\n\n  async updateProcessedEntity(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<{ relationsChange: SyncRelationsResult }> {\n    const tx = txOpaque as Knex.Transaction;\n    const {\n      id,\n      processedEntity,\n      resultHash,\n      errors,\n      relations,\n      deferredEntities,\n      refreshKeys,\n      locationKey,\n    } = options;\n    const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        processed_entity: JSON.stringify(processedEntity),\n        result_hash: resultHash,\n        errors,\n        location_key: locationKey,\n      })\n      .where('entity_id', id)\n      .andWhere(inner => {\n        if (!locationKey) {\n          return inner.whereNull('location_key');\n        }\n        return inner\n          .where('location_key', locationKey)\n          .orWhereNull('location_key');\n      });\n    if (refreshResult === 0) {\n      throw new ConflictError(\n        `Conflicting write of processing result for ${id} with location key '${locationKey}'`,\n      );\n    }\n    const sourceEntityRef = stringifyEntityRef(processedEntity);\n\n    // Schedule all deferred entities for future processing.\n    await this.addUnprocessedEntities(tx, {\n      entities: deferredEntities,\n      sourceEntityRef,\n    });\n\n    // Sync relations using a diff-based approach — only touches rows that\n    // actually changed, eliminating steady-state write churn.\n    const relationRows: DbRelationsRow[] = relations.map(\n      ({ source, target, type }) => ({\n        originating_entity_id: id,\n        source_entity_ref: stringifyEntityRef(source),\n        target_entity_ref: stringifyEntityRef(target),\n        type,\n      }),\n    );\n\n    const relationsResult = await syncRelations(tx, id, relationRows);\n\n    // Delete old refresh keys\n    await tx<DbRefreshKeysRow>('refresh_keys')\n      .where({ entity_id: id })\n      .delete();\n\n    // Insert the refresh keys for the processed entity\n    await tx.batchInsert(\n      'refresh_keys',\n      refreshKeys.map(k => ({\n        entity_id: id,\n        key: generateTargetKey(k.key),\n      })),\n      BATCH_SIZE,\n    );\n\n    return {\n      relationsChange: relationsResult,\n    };\n  }\n\n  async updateProcessedEntityErrors(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, errors, resultHash } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        errors,\n        result_hash: resultHash,\n      })\n      .where('entity_id', id);\n  }\n\n  async updateEntityCache(\n    txOpaque: Transaction,\n    options: UpdateEntityCacheOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, state } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({ cache: JSON.stringify(state ?? {}) })\n      .where('entity_id', id);\n  }\n\n  async getProcessableEntities(\n    maybeTx: Transaction | Knex,\n    request: { processBatchSize: number },\n  ): Promise<GetProcessableEntitiesResult> {\n    const knex = maybeTx as Knex.Transaction | Knex;\n    const useLocking = ['mysql', 'mysql2', 'pg'].includes(\n      knex.client.config.client,\n    );\n\n    const interval = this.options.refreshInterval();\n\n    const nextUpdateAt = (\n      tx: Knex | Knex.Transaction,\n      refreshInterval: number,\n    ) => {\n      if (tx.client.config.client.includes('sqlite3')) {\n        return tx.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);\n      } else if (tx.client.config.client.includes('mysql')) {\n        return tx.raw(`now() + interval ${refreshInterval} second`);\n      }\n      return tx.raw(`now() + interval '${refreshInterval} seconds'`);\n    };\n\n    // The SELECT FOR UPDATE SKIP LOCKED + UPDATE must run inside a\n    // single transaction so that the row locks persist until\n    // next_update_at has been bumped.\n    const run = async (tx: Knex | Knex.Transaction) => {\n      const items: DbRefreshStateRow[] = await tx('refresh_state')\n        .select([\n          'entity_id',\n          'entity_ref',\n          'unprocessed_entity',\n          'result_hash',\n          'cache',\n          'errors',\n          'location_key',\n          'next_update_at',\n        ])\n        .where('next_update_at', '<=', tx.fn.now())\n        .limit(request.processBatchSize)\n        .orderBy('next_update_at', 'asc')\n        .modify(qb => {\n          if (useLocking) {\n            qb.forUpdate().skipLocked();\n          }\n        });\n\n      if (items.length > 0) {\n        await tx('refresh_state')\n          .whereIn(\n            'entity_ref',\n            items.map(i => i.entity_ref),\n          )\n          .update({\n            next_update_at: nextUpdateAt(tx, interval),\n          });\n      }\n\n      return items;\n    };\n\n    const items =\n      knex.isTransaction || !useLocking\n        ? await run(knex)\n        : await knex.transaction(run);\n\n    return {\n      items: items.map(\n        i =>\n          ({\n            id: i.entity_id,\n            entityRef: i.entity_ref,\n            unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity,\n            resultHash: i.result_hash || '',\n            nextUpdateAt: timestampToDateTime(i.next_update_at),\n            state: i.cache ? JSON.parse(i.cache) : undefined,\n            errors: i.errors,\n            locationKey: i.location_key,\n          } satisfies RefreshStateItem),\n      ),\n    };\n  }\n\n  async listParents(\n    txOpaque: Transaction,\n    options: ListParentsOptions,\n  ): Promise<ListParentsResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .whereIn('target_entity_ref', options.entityRefs)\n      .select();\n\n    const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);\n\n    return { entityRefs };\n  }\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  /**\n   * Add a set of deferred entities for processing.\n   * The entities will be added at the front of the processing queue.\n   */\n  private async addUnprocessedEntities(\n    txOpaque: Transaction,\n    options: {\n      sourceEntityRef: string;\n      entities: DeferredEntity[];\n    },\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n\n    // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards\n    const stateReferences = new Array<string>();\n\n    // Upsert all of the unprocessed entities into the refresh_state table, by\n    // their entity ref.\n    for (const { entity, locationKey } of options.entities) {\n      const entityRef = stringifyEntityRef(entity);\n      const hash = generateStableHash(entity);\n\n      const updated = await updateUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n      });\n      if (updated) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      const inserted = await insertUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n        logger: this.options.logger,\n      });\n      if (inserted) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      // If the row can't be inserted, we have a conflict, but it could be either\n      // because of a conflicting locationKey or a race with another instance, so check\n      // whether the conflicting entity has the same entityRef but a different locationKey\n      const conflictingKey = await checkLocationKeyConflict({\n        tx,\n        entityRef,\n        locationKey,\n      });\n      if (conflictingKey) {\n        this.options.logger.warn(\n          `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n        );\n        if (locationKey) {\n          const eventParams: EventParams<CatalogConflictEventPayload> = {\n            topic: CATALOG_CONFLICTS_TOPIC,\n            eventPayload: {\n              unprocessedEntity: entity,\n              entityRef,\n              newLocationKey: locationKey,\n              existingLocationKey: conflictingKey,\n              lastConflictAt: DateTime.now().toISO()!,\n            },\n          };\n          await this.options.events.publish(eventParams);\n        }\n      }\n    }\n\n    // Lastly, replace refresh state references for the originating entity and any successfully added entities\n    await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n      // Remove all existing references from the originating entity\n      .where({ source_entity_ref: options.sourceEntityRef })\n      // And remove any existing references to entities that we're inserting new references for\n      .orWhereIn('target_entity_ref', stateReferences)\n      .delete();\n    await tx.batchInsert(\n      'refresh_state_references',\n      stateReferences.map(entityRef => ({\n        source_entity_ref: options.sourceEntityRef,\n        target_entity_ref: entityRef,\n      })),\n      BATCH_SIZE,\n    );\n  }\n}\n"],"names":["initDatabaseMetrics","errors","ConflictError","stringifyEntityRef","syncRelations","generateTargetKey","items","timestampToDateTime","rethrowError","generateStableHash","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","CATALOG_CONFLICTS_TOPIC","DateTime"],"mappings":";;;;;;;;;;;;;;AA0DA,MAAM,UAAA,GAAa,EAAA;AAEZ,MAAM,yBAAA,CAAwD;AAAA,EAClD,OAAA;AAAA,EAQjB,YAAY,OAAA,EAMT;AACD,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA;AACf,IAAAA,2BAAA,CAAoB,OAAA,CAAQ,QAAA,EAAU,OAAA,CAAQ,OAAO,CAAA;AAAA,EACvD;AAAA,EAEA,MAAM,qBAAA,CACJ,QAAA,EACA,OAAA,EACmD;AACnD,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM;AAAA,MACJ,EAAA;AAAA,MACA,eAAA;AAAA,MACA,UAAA;AAAA,cACAC,QAAA;AAAA,MACA,SAAA;AAAA,MACA,gBAAA;AAAA,MACA,WAAA;AAAA,MACA;AAAA,KACF,GAAI,OAAA;AACJ,IAAA,MAAM,aAAA,GAAgB,MAAM,EAAA,CAAsB,eAAe,EAC9D,MAAA,CAAO;AAAA,MACN,gBAAA,EAAkB,IAAA,CAAK,SAAA,CAAU,eAAe,CAAA;AAAA,MAChD,WAAA,EAAa,UAAA;AAAA,cACbA,QAAA;AAAA,MACA,YAAA,EAAc;AAAA,KACf,CAAA,CACA,KAAA,CAAM,aAAa,EAAE,CAAA,CACrB,SAAS,CAAA,KAAA,KAAS;AACjB,MAAA,IAAI,CAAC,WAAA,EAAa;AAChB,QAAA,OAAO,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA,MACvC;AACA,MAAA,OAAO,MACJ,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,IAC/B,CAAC,CAAA;AACH,IAAA,IAAI,kBAAkB,CAAA,EAAG;AACvB,MAAA,MAAM,IAAIC,oBAAA;AAAA,QACR,CAAA,2CAAA,EAA8C,EAAE,CAAA,oBAAA,EAAuB,WAAW,CAAA,CAAA;AAAA,OACpF;AAAA,IACF;AACA,IAAA,MAAM,eAAA,GAAkBC,gCAAmB,eAAe,CAAA;AAG1D,IAAA,MAAM,IAAA,CAAK,uBAAuB,EAAA,EAAI;AAAA,MACpC,QAAA,EAAU,gBAAA;AAAA,MACV;AAAA,KACD,CAAA;AAID,IAAA,MAAM,eAAiC,SAAA,CAAU,GAAA;AAAA,MAC/C,CAAC,EAAE,MAAA,EAAQ,MAAA,EAAQ,MAAK,MAAO;AAAA,QAC7B,qBAAA,EAAuB,EAAA;AAAA,QACvB,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C;AAAA,OACF;AAAA,KACF;AAEA,IAAA,MAAM,eAAA,GAAkB,MAAMC,2BAAA,CAAc,EAAA,EAAI,IAAI,YAAY,CAAA;AAGhE,IAAA,MAAM,EAAA,CAAqB,cAAc,CAAA,CACtC,KAAA,CAAM,EAAE,SAAA,EAAW,EAAA,EAAI,CAAA,CACvB,MAAA,EAAO;AAGV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,cAAA;AAAA,MACA,WAAA,CAAY,IAAI,CAAA,CAAA,MAAM;AAAA,QACpB,SAAA,EAAW,EAAA;AAAA,QACX,GAAA,EAAKC,sBAAA,CAAkB,CAAA,CAAE,GAAG;AAAA,OAC9B,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAEA,IAAA,OAAO;AAAA,MACL,eAAA,EAAiB;AAAA,KACnB;AAAA,EACF;AAAA,EAEA,MAAM,2BAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,MAAA,EAAQ,UAAA,EAAW,GAAI,OAAA;AAEnC,IAAA,MAAM,EAAA,CAAsB,eAAe,CAAA,CACxC,MAAA,CAAO;AAAA,MACN,MAAA;AAAA,MACA,WAAA,EAAa;AAAA,KACd,CAAA,CACA,KAAA,CAAM,WAAA,EAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,iBAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,KAAA,EAAM,GAAI,OAAA;AAEtB,IAAA,MAAM,GAAsB,eAAe,CAAA,CACxC,MAAA,CAAO,EAAE,OAAO,IAAA,CAAK,SAAA,CAAU,KAAA,IAAS,EAAE,CAAA,EAAG,CAAA,CAC7C,KAAA,CAAM,aAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,sBAAA,CACJ,OAAA,EACA,OAAA,EACuC;AACvC,IAAA,MAAM,IAAA,GAAO,OAAA;AACb,IAAA,MAAM,UAAA,GAAa,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,QAAA;AAAA,MAC3C,IAAA,CAAK,OAAO,MAAA,CAAO;AAAA,KACrB;AAEA,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,OAAA,CAAQ,eAAA,EAAgB;AAE9C,IAAA,MAAM,YAAA,GAAe,CACnB,EAAA,EACA,eAAA,KACG;AACH,MAAA,IAAI,GAAG,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AAC/C,QAAA,OAAO,GAAG,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,eAAe,UAAU,CAAC,CAAA;AAAA,MACpE,WAAW,EAAA,CAAG,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACpD,QAAA,OAAO,EAAA,CAAG,GAAA,CAAI,CAAA,iBAAA,EAAoB,eAAe,CAAA,OAAA,CAAS,CAAA;AAAA,MAC5D;AACA,MAAA,OAAO,EAAA,CAAG,GAAA,CAAI,CAAA,kBAAA,EAAqB,eAAe,CAAA,SAAA,CAAW,CAAA;AAAA,IAC/D,CAAA;AAKA,IAAA,MAAM,GAAA,GAAM,OAAO,EAAA,KAAgC;AACjD,MAAA,MAAMC,MAAAA,GAA6B,MAAM,EAAA,CAAG,eAAe,EACxD,MAAA,CAAO;AAAA,QACN,WAAA;AAAA,QACA,YAAA;AAAA,QACA,oBAAA;AAAA,QACA,aAAA;AAAA,QACA,OAAA;AAAA,QACA,QAAA;AAAA,QACA,cAAA;AAAA,QACA;AAAA,OACD,CAAA,CACA,KAAA,CAAM,kBAAkB,IAAA,EAAM,EAAA,CAAG,GAAG,GAAA,EAAK,EACzC,KAAA,CAAM,OAAA,CAAQ,gBAAgB,CAAA,CAC9B,OAAA,CAAQ,kBAAkB,KAAK,CAAA,CAC/B,OAAO,CAAA,EAAA,KAAM;AACZ,QAAA,IAAI,UAAA,EAAY;AACd,UAAA,EAAA,CAAG,SAAA,GAAY,UAAA,EAAW;AAAA,QAC5B;AAAA,MACF,CAAC,CAAA;AAEH,MAAA,IAAIA,MAAAA,CAAM,SAAS,CAAA,EAAG;AACpB,QAAA,MAAM,EAAA,CAAG,eAAe,CAAA,CACrB,OAAA;AAAA,UACC,YAAA;AAAA,UACAA,MAAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,UAE5B,MAAA,CAAO;AAAA,UACN,cAAA,EAAgB,YAAA,CAAa,EAAA,EAAI,QAAQ;AAAA,SAC1C,CAAA;AAAA,MACL;AAEA,MAAA,OAAOA,MAAAA;AAAA,IACT,CAAA;AAEA,IAAA,MAAM,KAAA,GACJ,IAAA,CAAK,aAAA,IAAiB,CAAC,UAAA,GACnB,MAAM,GAAA,CAAI,IAAI,CAAA,GACd,MAAM,IAAA,CAAK,WAAA,CAAY,GAAG,CAAA;AAEhC,IAAA,OAAO;AAAA,MACL,OAAO,KAAA,CAAM,GAAA;AAAA,QACX,CAAA,CAAA,MACG;AAAA,UACC,IAAI,CAAA,CAAE,SAAA;AAAA,UACN,WAAW,CAAA,CAAE,UAAA;AAAA,UACb,iBAAA,EAAmB,IAAA,CAAK,KAAA,CAAM,CAAA,CAAE,kBAAkB,CAAA;AAAA,UAClD,UAAA,EAAY,EAAE,WAAA,IAAe,EAAA;AAAA,UAC7B,YAAA,EAAcC,8BAAA,CAAoB,CAAA,CAAE,cAAc,CAAA;AAAA,UAClD,OAAO,CAAA,CAAE,KAAA,GAAQ,KAAK,KAAA,CAAM,CAAA,CAAE,KAAK,CAAA,GAAI,MAAA;AAAA,UACvC,QAAQ,CAAA,CAAE,MAAA;AAAA,UACV,aAAa,CAAA,CAAE;AAAA,SACjB;AAAA;AACJ,KACF;AAAA,EACF;AAAA,EAEA,MAAM,WAAA,CACJ,QAAA,EACA,OAAA,EAC4B;AAC5B,IAAA,MAAM,EAAA,GAAK,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,MAEC,OAAA,CAAQ,mBAAA,EAAqB,OAAA,CAAQ,UAAU,EAC/C,MAAA,EAAO;AAEV,IAAA,MAAM,UAAA,GAAa,KAAK,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,iBAAkB,CAAA,CAAE,OAAO,OAAO,CAAA;AAErE,IAAA,OAAO,EAAE,UAAA,EAAW;AAAA,EACtB;AAAA,EAEA,MAAM,YAAe,EAAA,EAAiD;AACpE,IAAA,IAAI;AACF,MAAA,IAAI,MAAA,GAAwB,KAAA,CAAA;AAE5B,MAAA,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,WAAA;AAAA,QAC1B,OAAM,EAAA,KAAM;AAGV,UAAA,MAAA,GAAS,MAAM,GAAG,EAAE,CAAA;AAAA,QACtB,CAAA;AAAA,QACA;AAAA;AAAA,UAEE,qBAAA,EAAuB;AAAA;AACzB,OACF;AAEA,MAAA,OAAO,MAAA;AAAA,IACT,SAAS,CAAA,EAAG;AACV,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,0BAAA,EAA6B,CAAC,CAAA,CAAE,CAAA;AAC1D,MAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA,IACtB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,sBAAA,CACZ,QAAA,EACA,OAAA,EAIe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AAGX,IAAA,MAAM,eAAA,GAAkB,IAAI,KAAA,EAAc;AAI1C,IAAA,KAAA,MAAW,EAAE,MAAA,EAAQ,WAAA,EAAY,IAAK,QAAQ,QAAA,EAAU;AACtD,MAAA,MAAM,SAAA,GAAYL,gCAAmB,MAAM,CAAA;AAC3C,MAAA,MAAM,IAAA,GAAOM,wBAAmB,MAAM,CAAA;AAEtC,MAAA,MAAM,OAAA,GAAU,MAAMC,+CAAA,CAAwB;AAAA,QAC5C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,OAAA,EAAS;AACX,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,QAAA,GAAW,MAAMC,+CAAA,CAAwB;AAAA,QAC7C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA,WAAA;AAAA,QACA,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAKA,MAAA,MAAM,cAAA,GAAiB,MAAMC,iDAAA,CAAyB;AAAA,QACpD,EAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,cAAA,EAAgB;AAClB,QAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,UAClB,CAAA,+BAAA,EAAkC,SAAS,CAAA,uBAAA,EAA0B,cAAc,iBAAiB,WAAW,CAAA;AAAA,SACjH;AACA,QAAA,IAAI,WAAA,EAAa;AACf,UAAA,MAAM,WAAA,GAAwD;AAAA,YAC5D,KAAA,EAAOC,iCAAA;AAAA,YACP,YAAA,EAAc;AAAA,cACZ,iBAAA,EAAmB,MAAA;AAAA,cACnB,SAAA;AAAA,cACA,cAAA,EAAgB,WAAA;AAAA,cAChB,mBAAA,EAAqB,cAAA;AAAA,cACrB,cAAA,EAAgBC,cAAA,CAAS,GAAA,EAAI,CAAE,KAAA;AAAM;AACvC,WACF;AACA,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,OAAA,CAAQ,WAAW,CAAA;AAAA,QAC/C;AAAA,MACF;AAAA,IACF;AAGA,IAAA,MAAM,EAAA,CAAgC,0BAA0B,CAAA,CAE7D,KAAA,CAAM,EAAE,iBAAA,EAAmB,OAAA,CAAQ,eAAA,EAAiB,CAAA,CAEpD,SAAA,CAAU,mBAAA,EAAqB,eAAe,EAC9C,MAAA,EAAO;AACV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,0BAAA;AAAA,MACA,eAAA,CAAgB,IAAI,CAAA,SAAA,MAAc;AAAA,QAChC,mBAAmB,OAAA,CAAQ,eAAA;AAAA,QAC3B,iBAAA,EAAmB;AAAA,OACrB,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAAA,EACF;AACF;;;;"}
+{"version":3,"file":"DefaultProcessingDatabase.cjs.js","sources":["../../src/database/DefaultProcessingDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError } from '@backstage/errors';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport { ProcessingIntervalFunction } from '../processing/refresh';\nimport { rethrowError, timestampToDateTime } from './conversion';\nimport { initDatabaseMetrics } from './metrics';\nimport {\n  DbRefreshKeysRow,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n} from './tables';\nimport {\n  GetProcessableEntitiesResult,\n  ListParentsOptions,\n  ListParentsResult,\n  ProcessingDatabase,\n  RefreshStateItem,\n  Transaction,\n  UpdateEntityCacheOptions,\n  UpdateProcessedEntityOptions,\n} from './types';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { generateStableHash, generateTargetKey, whereInArray } from './util';\nimport {\n  syncRelations,\n  SyncRelationsResult,\n} from './operations/relations/syncRelations';\nimport { EventParams, EventsService } from '@backstage/plugin-events-node';\nimport { DateTime } from 'luxon';\nimport { CATALOG_CONFLICTS_TOPIC } from '../constants';\nimport { CatalogConflictEventPayload } from '../catalog/types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\nimport { MetricsService } from '@backstage/backend-plugin-api/alpha';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProcessingDatabase implements ProcessingDatabase {\n  private readonly options: {\n    database: Knex;\n    logger: LoggerService;\n    refreshInterval: ProcessingIntervalFunction;\n    events: EventsService;\n    metrics: MetricsService;\n  };\n\n  constructor(options: {\n    database: Knex;\n    logger: LoggerService;\n    refreshInterval: ProcessingIntervalFunction;\n    events: EventsService;\n    metrics: MetricsService;\n  }) {\n    this.options = options;\n    initDatabaseMetrics(options.database, options.metrics);\n  }\n\n  async updateProcessedEntity(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<{ relationsChange: SyncRelationsResult }> {\n    const tx = txOpaque as Knex.Transaction;\n    const {\n      id,\n      processedEntity,\n      resultHash,\n      errors,\n      relations,\n      deferredEntities,\n      refreshKeys,\n      locationKey,\n    } = options;\n    const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        processed_entity: JSON.stringify(processedEntity),\n        result_hash: resultHash,\n        errors,\n        location_key: locationKey,\n      })\n      .where('entity_id', id)\n      .andWhere(inner => {\n        if (!locationKey) {\n          return inner.whereNull('location_key');\n        }\n        return inner\n          .where('location_key', locationKey)\n          .orWhereNull('location_key');\n      });\n    if (refreshResult === 0) {\n      throw new ConflictError(\n        `Conflicting write of processing result for ${id} with location key '${locationKey}'`,\n      );\n    }\n    const sourceEntityRef = stringifyEntityRef(processedEntity);\n\n    // Schedule all deferred entities for future processing.\n    await this.addUnprocessedEntities(tx, {\n      entities: deferredEntities,\n      sourceEntityRef,\n    });\n\n    // Sync relations using a diff-based approach — only touches rows that\n    // actually changed, eliminating steady-state write churn.\n    const relationRows: DbRelationsRow[] = relations.map(\n      ({ source, target, type }) => ({\n        originating_entity_id: id,\n        source_entity_ref: stringifyEntityRef(source),\n        target_entity_ref: stringifyEntityRef(target),\n        type,\n      }),\n    );\n\n    const relationsResult = await syncRelations(tx, id, relationRows);\n\n    // Delete old refresh keys\n    await tx<DbRefreshKeysRow>('refresh_keys')\n      .where({ entity_id: id })\n      .delete();\n\n    // Insert the refresh keys for the processed entity\n    await tx.batchInsert(\n      'refresh_keys',\n      refreshKeys.map(k => ({\n        entity_id: id,\n        key: generateTargetKey(k.key),\n      })),\n      BATCH_SIZE,\n    );\n\n    return {\n      relationsChange: relationsResult,\n    };\n  }\n\n  async updateProcessedEntityErrors(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, errors, resultHash } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        errors,\n        result_hash: resultHash,\n      })\n      .where('entity_id', id);\n  }\n\n  async updateEntityCache(\n    txOpaque: Transaction,\n    options: UpdateEntityCacheOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, state } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({ cache: JSON.stringify(state ?? {}) })\n      .where('entity_id', id);\n  }\n\n  async getProcessableEntities(\n    maybeTx: Transaction | Knex,\n    request: { processBatchSize: number },\n  ): Promise<GetProcessableEntitiesResult> {\n    const knex = maybeTx as Knex.Transaction | Knex;\n    const useLocking = ['mysql', 'mysql2', 'pg'].includes(\n      knex.client.config.client,\n    );\n\n    const interval = this.options.refreshInterval();\n\n    const nextUpdateAt = (\n      tx: Knex | Knex.Transaction,\n      refreshInterval: number,\n    ) => {\n      if (tx.client.config.client.includes('sqlite3')) {\n        return tx.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);\n      } else if (tx.client.config.client.includes('mysql')) {\n        return tx.raw(`now() + interval ${refreshInterval} second`);\n      }\n      return tx.raw(`now() + interval '${refreshInterval} seconds'`);\n    };\n\n    // The SELECT FOR UPDATE SKIP LOCKED + UPDATE must run inside a\n    // single transaction so that the row locks persist until\n    // next_update_at has been bumped.\n    const run = async (tx: Knex | Knex.Transaction) => {\n      const items: DbRefreshStateRow[] = await tx('refresh_state')\n        .select([\n          'entity_id',\n          'entity_ref',\n          'unprocessed_entity',\n          'result_hash',\n          'cache',\n          'errors',\n          'location_key',\n          'next_update_at',\n        ])\n        .where('next_update_at', '<=', tx.fn.now())\n        .limit(request.processBatchSize)\n        .orderBy('next_update_at', 'asc')\n        .modify(qb => {\n          if (useLocking) {\n            qb.forUpdate().skipLocked();\n          }\n        });\n\n      if (items.length > 0) {\n        await tx('refresh_state')\n          .where(\n            whereInArray(\n              'entity_ref',\n              items.map(i => i.entity_ref),\n            ),\n          )\n          .update({\n            next_update_at: nextUpdateAt(tx, interval),\n          });\n      }\n\n      return items;\n    };\n\n    const items =\n      knex.isTransaction || !useLocking\n        ? await run(knex)\n        : await knex.transaction(run);\n\n    return {\n      items: items.map(\n        i =>\n          ({\n            id: i.entity_id,\n            entityRef: i.entity_ref,\n            unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity,\n            resultHash: i.result_hash || '',\n            nextUpdateAt: timestampToDateTime(i.next_update_at),\n            state: i.cache ? JSON.parse(i.cache) : undefined,\n            errors: i.errors,\n            locationKey: i.location_key,\n          } satisfies RefreshStateItem),\n      ),\n    };\n  }\n\n  async listParents(\n    txOpaque: Transaction,\n    options: ListParentsOptions,\n  ): Promise<ListParentsResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .whereIn('target_entity_ref', options.entityRefs)\n      .select();\n\n    const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);\n\n    return { entityRefs };\n  }\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  /**\n   * Add a set of deferred entities for processing.\n   * The entities will be added at the front of the processing queue.\n   */\n  private async addUnprocessedEntities(\n    txOpaque: Transaction,\n    options: {\n      sourceEntityRef: string;\n      entities: DeferredEntity[];\n    },\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n\n    // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards\n    const stateReferences = new Array<string>();\n\n    // Upsert all of the unprocessed entities into the refresh_state table, by\n    // their entity ref.\n    for (const { entity, locationKey } of options.entities) {\n      const entityRef = stringifyEntityRef(entity);\n      const hash = generateStableHash(entity);\n\n      const updated = await updateUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n      });\n      if (updated) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      const inserted = await insertUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n        logger: this.options.logger,\n      });\n      if (inserted) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      // If the row can't be inserted, we have a conflict, but it could be either\n      // because of a conflicting locationKey or a race with another instance, so check\n      // whether the conflicting entity has the same entityRef but a different locationKey\n      const conflictingKey = await checkLocationKeyConflict({\n        tx,\n        entityRef,\n        locationKey,\n      });\n      if (conflictingKey) {\n        this.options.logger.warn(\n          `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n        );\n        if (locationKey) {\n          const eventParams: EventParams<CatalogConflictEventPayload> = {\n            topic: CATALOG_CONFLICTS_TOPIC,\n            eventPayload: {\n              unprocessedEntity: entity,\n              entityRef,\n              newLocationKey: locationKey,\n              existingLocationKey: conflictingKey,\n              lastConflictAt: DateTime.now().toISO()!,\n            },\n          };\n          await this.options.events.publish(eventParams);\n        }\n      }\n    }\n\n    // Lastly, replace refresh state references for the originating entity and any successfully added entities\n    await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n      // Remove all existing references from the originating entity\n      .where({ source_entity_ref: options.sourceEntityRef })\n      // And remove any existing references to entities that we're inserting new references for\n      .orWhereIn('target_entity_ref', stateReferences)\n      .delete();\n    await tx.batchInsert(\n      'refresh_state_references',\n      stateReferences.map(entityRef => ({\n        source_entity_ref: options.sourceEntityRef,\n        target_entity_ref: entityRef,\n      })),\n      BATCH_SIZE,\n    );\n  }\n}\n"],"names":["initDatabaseMetrics","errors","ConflictError","stringifyEntityRef","syncRelations","generateTargetKey","items","whereInArray","timestampToDateTime","rethrowError","generateStableHash","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","CATALOG_CONFLICTS_TOPIC","DateTime"],"mappings":";;;;;;;;;;;;;;AA0DA,MAAM,UAAA,GAAa,EAAA;AAEZ,MAAM,yBAAA,CAAwD;AAAA,EAClD,OAAA;AAAA,EAQjB,YAAY,OAAA,EAMT;AACD,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA;AACf,IAAAA,2BAAA,CAAoB,OAAA,CAAQ,QAAA,EAAU,OAAA,CAAQ,OAAO,CAAA;AAAA,EACvD;AAAA,EAEA,MAAM,qBAAA,CACJ,QAAA,EACA,OAAA,EACmD;AACnD,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM;AAAA,MACJ,EAAA;AAAA,MACA,eAAA;AAAA,MACA,UAAA;AAAA,cACAC,QAAA;AAAA,MACA,SAAA;AAAA,MACA,gBAAA;AAAA,MACA,WAAA;AAAA,MACA;AAAA,KACF,GAAI,OAAA;AACJ,IAAA,MAAM,aAAA,GAAgB,MAAM,EAAA,CAAsB,eAAe,EAC9D,MAAA,CAAO;AAAA,MACN,gBAAA,EAAkB,IAAA,CAAK,SAAA,CAAU,eAAe,CAAA;AAAA,MAChD,WAAA,EAAa,UAAA;AAAA,cACbA,QAAA;AAAA,MACA,YAAA,EAAc;AAAA,KACf,CAAA,CACA,KAAA,CAAM,aAAa,EAAE,CAAA,CACrB,SAAS,CAAA,KAAA,KAAS;AACjB,MAAA,IAAI,CAAC,WAAA,EAAa;AAChB,QAAA,OAAO,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA,MACvC;AACA,MAAA,OAAO,MACJ,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,IAC/B,CAAC,CAAA;AACH,IAAA,IAAI,kBAAkB,CAAA,EAAG;AACvB,MAAA,MAAM,IAAIC,oBAAA;AAAA,QACR,CAAA,2CAAA,EAA8C,EAAE,CAAA,oBAAA,EAAuB,WAAW,CAAA,CAAA;AAAA,OACpF;AAAA,IACF;AACA,IAAA,MAAM,eAAA,GAAkBC,gCAAmB,eAAe,CAAA;AAG1D,IAAA,MAAM,IAAA,CAAK,uBAAuB,EAAA,EAAI;AAAA,MACpC,QAAA,EAAU,gBAAA;AAAA,MACV;AAAA,KACD,CAAA;AAID,IAAA,MAAM,eAAiC,SAAA,CAAU,GAAA;AAAA,MAC/C,CAAC,EAAE,MAAA,EAAQ,MAAA,EAAQ,MAAK,MAAO;AAAA,QAC7B,qBAAA,EAAuB,EAAA;AAAA,QACvB,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C;AAAA,OACF;AAAA,KACF;AAEA,IAAA,MAAM,eAAA,GAAkB,MAAMC,2BAAA,CAAc,EAAA,EAAI,IAAI,YAAY,CAAA;AAGhE,IAAA,MAAM,EAAA,CAAqB,cAAc,CAAA,CACtC,KAAA,CAAM,EAAE,SAAA,EAAW,EAAA,EAAI,CAAA,CACvB,MAAA,EAAO;AAGV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,cAAA;AAAA,MACA,WAAA,CAAY,IAAI,CAAA,CAAA,MAAM;AAAA,QACpB,SAAA,EAAW,EAAA;AAAA,QACX,GAAA,EAAKC,sBAAA,CAAkB,CAAA,CAAE,GAAG;AAAA,OAC9B,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAEA,IAAA,OAAO;AAAA,MACL,eAAA,EAAiB;AAAA,KACnB;AAAA,EACF;AAAA,EAEA,MAAM,2BAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,MAAA,EAAQ,UAAA,EAAW,GAAI,OAAA;AAEnC,IAAA,MAAM,EAAA,CAAsB,eAAe,CAAA,CACxC,MAAA,CAAO;AAAA,MACN,MAAA;AAAA,MACA,WAAA,EAAa;AAAA,KACd,CAAA,CACA,KAAA,CAAM,WAAA,EAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,iBAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,KAAA,EAAM,GAAI,OAAA;AAEtB,IAAA,MAAM,GAAsB,eAAe,CAAA,CACxC,MAAA,CAAO,EAAE,OAAO,IAAA,CAAK,SAAA,CAAU,KAAA,IAAS,EAAE,CAAA,EAAG,CAAA,CAC7C,KAAA,CAAM,aAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,sBAAA,CACJ,OAAA,EACA,OAAA,EACuC;AACvC,IAAA,MAAM,IAAA,GAAO,OAAA;AACb,IAAA,MAAM,UAAA,GAAa,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,QAAA;AAAA,MAC3C,IAAA,CAAK,OAAO,MAAA,CAAO;AAAA,KACrB;AAEA,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,OAAA,CAAQ,eAAA,EAAgB;AAE9C,IAAA,MAAM,YAAA,GAAe,CACnB,EAAA,EACA,eAAA,KACG;AACH,MAAA,IAAI,GAAG,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AAC/C,QAAA,OAAO,GAAG,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,eAAe,UAAU,CAAC,CAAA;AAAA,MACpE,WAAW,EAAA,CAAG,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACpD,QAAA,OAAO,EAAA,CAAG,GAAA,CAAI,CAAA,iBAAA,EAAoB,eAAe,CAAA,OAAA,CAAS,CAAA;AAAA,MAC5D;AACA,MAAA,OAAO,EAAA,CAAG,GAAA,CAAI,CAAA,kBAAA,EAAqB,eAAe,CAAA,SAAA,CAAW,CAAA;AAAA,IAC/D,CAAA;AAKA,IAAA,MAAM,GAAA,GAAM,OAAO,EAAA,KAAgC;AACjD,MAAA,MAAMC,MAAAA,GAA6B,MAAM,EAAA,CAAG,eAAe,EACxD,MAAA,CAAO;AAAA,QACN,WAAA;AAAA,QACA,YAAA;AAAA,QACA,oBAAA;AAAA,QACA,aAAA;AAAA,QACA,OAAA;AAAA,QACA,QAAA;AAAA,QACA,cAAA;AAAA,QACA;AAAA,OACD,CAAA,CACA,KAAA,CAAM,kBAAkB,IAAA,EAAM,EAAA,CAAG,GAAG,GAAA,EAAK,EACzC,KAAA,CAAM,OAAA,CAAQ,gBAAgB,CAAA,CAC9B,OAAA,CAAQ,kBAAkB,KAAK,CAAA,CAC/B,OAAO,CAAA,EAAA,KAAM;AACZ,QAAA,IAAI,UAAA,EAAY;AACd,UAAA,EAAA,CAAG,SAAA,GAAY,UAAA,EAAW;AAAA,QAC5B;AAAA,MACF,CAAC,CAAA;AAEH,MAAA,IAAIA,MAAAA,CAAM,SAAS,CAAA,EAAG;AACpB,QAAA,MAAM,EAAA,CAAG,eAAe,CAAA,CACrB,KAAA;AAAA,UACCC,iBAAA;AAAA,YACE,YAAA;AAAA,YACAD,MAAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA;AAC7B,UAED,MAAA,CAAO;AAAA,UACN,cAAA,EAAgB,YAAA,CAAa,EAAA,EAAI,QAAQ;AAAA,SAC1C,CAAA;AAAA,MACL;AAEA,MAAA,OAAOA,MAAAA;AAAA,IACT,CAAA;AAEA,IAAA,MAAM,KAAA,GACJ,IAAA,CAAK,aAAA,IAAiB,CAAC,UAAA,GACnB,MAAM,GAAA,CAAI,IAAI,CAAA,GACd,MAAM,IAAA,CAAK,WAAA,CAAY,GAAG,CAAA;AAEhC,IAAA,OAAO;AAAA,MACL,OAAO,KAAA,CAAM,GAAA;AAAA,QACX,CAAA,CAAA,MACG;AAAA,UACC,IAAI,CAAA,CAAE,SAAA;AAAA,UACN,WAAW,CAAA,CAAE,UAAA;AAAA,UACb,iBAAA,EAAmB,IAAA,CAAK,KAAA,CAAM,CAAA,CAAE,kBAAkB,CAAA;AAAA,UAClD,UAAA,EAAY,EAAE,WAAA,IAAe,EAAA;AAAA,UAC7B,YAAA,EAAcE,8BAAA,CAAoB,CAAA,CAAE,cAAc,CAAA;AAAA,UAClD,OAAO,CAAA,CAAE,KAAA,GAAQ,KAAK,KAAA,CAAM,CAAA,CAAE,KAAK,CAAA,GAAI,MAAA;AAAA,UACvC,QAAQ,CAAA,CAAE,MAAA;AAAA,UACV,aAAa,CAAA,CAAE;AAAA,SACjB;AAAA;AACJ,KACF;AAAA,EACF;AAAA,EAEA,MAAM,WAAA,CACJ,QAAA,EACA,OAAA,EAC4B;AAC5B,IAAA,MAAM,EAAA,GAAK,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,MAEC,OAAA,CAAQ,mBAAA,EAAqB,OAAA,CAAQ,UAAU,EAC/C,MAAA,EAAO;AAEV,IAAA,MAAM,UAAA,GAAa,KAAK,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,iBAAkB,CAAA,CAAE,OAAO,OAAO,CAAA;AAErE,IAAA,OAAO,EAAE,UAAA,EAAW;AAAA,EACtB;AAAA,EAEA,MAAM,YAAe,EAAA,EAAiD;AACpE,IAAA,IAAI;AACF,MAAA,IAAI,MAAA,GAAwB,KAAA,CAAA;AAE5B,MAAA,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,WAAA;AAAA,QAC1B,OAAM,EAAA,KAAM;AAGV,UAAA,MAAA,GAAS,MAAM,GAAG,EAAE,CAAA;AAAA,QACtB,CAAA;AAAA,QACA;AAAA;AAAA,UAEE,qBAAA,EAAuB;AAAA;AACzB,OACF;AAEA,MAAA,OAAO,MAAA;AAAA,IACT,SAAS,CAAA,EAAG;AACV,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,0BAAA,EAA6B,CAAC,CAAA,CAAE,CAAA;AAC1D,MAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA,IACtB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,sBAAA,CACZ,QAAA,EACA,OAAA,EAIe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AAGX,IAAA,MAAM,eAAA,GAAkB,IAAI,KAAA,EAAc;AAI1C,IAAA,KAAA,MAAW,EAAE,MAAA,EAAQ,WAAA,EAAY,IAAK,QAAQ,QAAA,EAAU;AACtD,MAAA,MAAM,SAAA,GAAYN,gCAAmB,MAAM,CAAA;AAC3C,MAAA,MAAM,IAAA,GAAOO,wBAAmB,MAAM,CAAA;AAEtC,MAAA,MAAM,OAAA,GAAU,MAAMC,+CAAA,CAAwB;AAAA,QAC5C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,OAAA,EAAS;AACX,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,QAAA,GAAW,MAAMC,+CAAA,CAAwB;AAAA,QAC7C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA,WAAA;AAAA,QACA,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAKA,MAAA,MAAM,cAAA,GAAiB,MAAMC,iDAAA,CAAyB;AAAA,QACpD,EAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,cAAA,EAAgB;AAClB,QAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,UAClB,CAAA,+BAAA,EAAkC,SAAS,CAAA,uBAAA,EAA0B,cAAc,iBAAiB,WAAW,CAAA;AAAA,SACjH;AACA,QAAA,IAAI,WAAA,EAAa;AACf,UAAA,MAAM,WAAA,GAAwD;AAAA,YAC5D,KAAA,EAAOC,iCAAA;AAAA,YACP,YAAA,EAAc;AAAA,cACZ,iBAAA,EAAmB,MAAA;AAAA,cACnB,SAAA;AAAA,cACA,cAAA,EAAgB,WAAA;AAAA,cAChB,mBAAA,EAAqB,cAAA;AAAA,cACrB,cAAA,EAAgBC,cAAA,CAAS,GAAA,EAAI,CAAE,KAAA;AAAM;AACvC,WACF;AACA,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,OAAA,CAAQ,WAAW,CAAA;AAAA,QAC/C;AAAA,MACF;AAAA,IACF;AAGA,IAAA,MAAM,EAAA,CAAgC,0BAA0B,CAAA,CAE7D,KAAA,CAAM,EAAE,iBAAA,EAAmB,OAAA,CAAQ,eAAA,EAAiB,CAAA,CAEpD,SAAA,CAAU,mBAAA,EAAqB,eAAe,EAC9C,MAAA,EAAO;AACV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,0BAAA;AAAA,MACA,eAAA,CAAgB,IAAI,CAAA,SAAA,MAAc;AAAA,QAChC,mBAAmB,OAAA,CAAQ,eAAA;AAAA,QAC3B,iBAAA,EAAmB;AAAA,OACrB,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAAA,EACF;AACF;;;;"}

package/dist/database/DefaultProviderDatabase.cjs.js

@@ -170,7 +170,7 @@ class DefaultProviderDatabase {
       const toRemove2 = options.removed.map((e) => e.entityRef);
       for (const chunk of lodash__default.default.chunk(options.added, 1e3)) {
         const entityRefs = chunk.map((e) => catalogModel.stringifyEntityRef(e.entity));
-        const rows = await tx("refresh_state").select(["entity_ref", "unprocessed_hash", "location_key"]).whereIn("entity_ref", entityRefs);
+        const rows = await tx("refresh_state").select(["entity_ref", "unprocessed_hash", "location_key"]).where(util.whereInArray("entity_ref", entityRefs));
         const oldStates = new Map(
           rows.map((row) => [
             row.entity_ref,

package/dist/database/DefaultProviderDatabase.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultProviderDatabase.cjs.js","sources":["../../src/database/DefaultProviderDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { stringifyEntityRef } from '@backstage/catalog-model';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { randomUUID as uuid } from 'node:crypto';\nimport { rethrowError } from './conversion';\nimport { deleteWithEagerPruningOfChildren } from './operations/provider/deleteWithEagerPruningOfChildren';\nimport { refreshByRefreshKeys } from './operations/provider/refreshByRefreshKeys';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { DbRefreshStateReferencesRow, DbRefreshStateRow } from './tables';\nimport {\n  ProviderDatabase,\n  RefreshByKeyOptions,\n  ReplaceUnprocessedEntitiesOptions,\n  Transaction,\n} from './types';\nimport { generateStableHash, isDeadlockError, retryOnDeadlock } from './util';\nimport {\n  LoggerService,\n  isDatabaseConflictError,\n} from '@backstage/backend-plugin-api';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProviderDatabase implements ProviderDatabase {\n  private readonly options: {\n    database: Knex;\n    logger: LoggerService;\n  };\n\n  constructor(options: { database: Knex; logger: LoggerService }) {\n    this.options = options;\n  }\n\n  /**\n   * Executes `fn` inside a database transaction, retrying automatically on\n   * deadlock (PostgreSQL error 40P01). Because the callback may be invoked\n   * more than once, it must not perform non-database side effects such as\n   * emitting events, mutating in-memory state, or calling external services.\n   */\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    return retryOnDeadlock(async () => {\n      try {\n        let result: T | undefined = undefined;\n        await this.options.database.transaction(\n          async tx => {\n            // We can't return here, as knex swallows the return type in case the\n            // transaction is rolled back:\n            // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n            result = await fn(tx);\n          },\n          {\n            // If we explicitly trigger a rollback, don't fail.\n            doNotRejectOnRollback: true,\n          },\n        );\n        return result!;\n      } catch (e) {\n        this.options.logger.debug(`Error during transaction, ${e}`);\n        throw rethrowError(e);\n      }\n    }, this.options.database);\n  }\n\n  async replaceUnprocessedEntities(\n    txOpaque: Knex | Transaction,\n    options: ReplaceUnprocessedEntitiesOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex | Knex.Transaction;\n    const { toAdd, toUpsert, toRemove } = await this.createDelta(tx, options);\n\n    if (toRemove.length) {\n      const removedCount = await deleteWithEagerPruningOfChildren({\n        knex: tx,\n        entityRefs: toRemove,\n        sourceKey: options.sourceKey,\n      });\n      this.options.logger.debug(\n        `removed, ${removedCount} entities: ${JSON.stringify(toRemove)}`,\n      );\n    }\n\n    if (toAdd.length) {\n      // The reason for this chunking, rather than just massively batch\n      // inserting the entire payload, is that we fall back to the individual\n      // upsert mechanism below on conflicts. That path is massively slower than\n      // the fast batch path, so we don't want to end up accidentally having to\n      // for example item-by-item upsert tens of thousands of entities in a\n      // large initial delivery dump. The implication is that the size of these\n      // chunks needs to weigh the benefit of fast successful inserts, against\n      // the drawback of super slow but more rare fallbacks. There's quickly\n      // diminishing returns though with turning up this value way high.\n      for (const chunk of lodash.chunk(toAdd, 50)) {\n        try {\n          await tx.batchInsert(\n            'refresh_state',\n            chunk.map(item => ({\n              entity_id: uuid(),\n              entity_ref: stringifyEntityRef(item.deferred.entity),\n              unprocessed_entity: JSON.stringify(item.deferred.entity),\n              unprocessed_hash: item.hash,\n              errors: '',\n              location_key: item.deferred.locationKey,\n              next_update_at: tx.fn.now(),\n              last_discovery_at: tx.fn.now(),\n            })),\n            BATCH_SIZE,\n          );\n          await tx.batchInsert(\n            'refresh_state_references',\n            chunk.map(item => ({\n              source_key: options.sourceKey,\n              target_entity_ref: stringifyEntityRef(item.deferred.entity),\n            })),\n            BATCH_SIZE,\n          );\n        } catch (error) {\n          if (!isDatabaseConflictError(error)) {\n            throw error;\n          } else {\n            this.options.logger.debug(\n              `Fast insert path failed, falling back to slow path, ${error}`,\n            );\n            toUpsert.push(...chunk);\n          }\n        }\n      }\n    }\n\n    if (toUpsert.length) {\n      for (const {\n        deferred: { entity, locationKey },\n        hash,\n      } of toUpsert) {\n        const entityRef = stringifyEntityRef(entity);\n\n        try {\n          let ok = await updateUnprocessedEntity({\n            tx,\n            entity,\n            hash,\n            locationKey,\n          });\n          if (!ok) {\n            ok = await insertUnprocessedEntity({\n              tx,\n              entity,\n              hash,\n              locationKey,\n              logger: this.options.logger,\n            });\n          }\n          if (ok) {\n            await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n              .where('target_entity_ref', entityRef)\n              .delete();\n\n            await tx<DbRefreshStateReferencesRow>(\n              'refresh_state_references',\n            ).insert({\n              source_key: options.sourceKey,\n              target_entity_ref: entityRef,\n            });\n          } else {\n            await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n              .where('target_entity_ref', entityRef)\n              .andWhere({ source_key: options.sourceKey })\n              .delete();\n\n            const conflictingKey = await checkLocationKeyConflict({\n              tx,\n              entityRef,\n              locationKey,\n            });\n            if (conflictingKey) {\n              this.options.logger.warn(\n                `Source ${options.sourceKey} detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n              );\n            }\n          }\n        } catch (error) {\n          if (isDeadlockError(tx, error)) {\n            throw error;\n          }\n          this.options.logger.error(\n            `Failed to add '${entityRef}' from source '${options.sourceKey}', ${error}`,\n          );\n        }\n      }\n    }\n  }\n\n  async listReferenceSourceKeys(txOpaque: Transaction): Promise<string[]> {\n    const tx = txOpaque as Knex | Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .distinct('source_key')\n      .whereNotNull('source_key');\n\n    return rows\n      .map(row => row.source_key)\n      .filter((key): key is string => !!key);\n  }\n\n  async refreshByRefreshKeys(\n    txOpaque: Transaction,\n    options: RefreshByKeyOptions,\n  ) {\n    const tx = txOpaque as Knex.Transaction;\n    await refreshByRefreshKeys({ tx, keys: options.keys });\n  }\n\n  private async createDelta(\n    tx: Knex | Knex.Transaction,\n    options: ReplaceUnprocessedEntitiesOptions,\n  ): Promise<{\n    toAdd: { deferred: DeferredEntity; hash: string }[];\n    toUpsert: { deferred: DeferredEntity; hash: string }[];\n    toRemove: string[];\n  }> {\n    if (options.type === 'delta') {\n      const toAdd = new Array<{ deferred: DeferredEntity; hash: string }>();\n      const toUpsert = new Array<{ deferred: DeferredEntity; hash: string }>();\n      const toRemove = options.removed.map(e => e.entityRef);\n\n      for (const chunk of lodash.chunk(options.added, 1000)) {\n        const entityRefs = chunk.map(e => stringifyEntityRef(e.entity));\n        const rows = await tx<DbRefreshStateRow>('refresh_state')\n          .select(['entity_ref', 'unprocessed_hash', 'location_key'])\n          .whereIn('entity_ref', entityRefs);\n        const oldStates = new Map(\n          rows.map(row => [\n            row.entity_ref,\n            {\n              unprocessed_hash: row.unprocessed_hash,\n              location_key: row.location_key,\n            },\n          ]),\n        );\n\n        chunk.forEach((deferred, i) => {\n          const entityRef = entityRefs[i];\n          const newHash = generateStableHash(deferred.entity);\n          const oldState = oldStates.get(entityRef);\n          if (oldState === undefined) {\n            // Add any entity that does not exist in the database\n            toAdd.push({ deferred, hash: newHash });\n          } else if (\n            (deferred.locationKey ?? null) !== (oldState.location_key ?? null)\n          ) {\n            // Remove and then re-add any entity that exists, but with a different location key\n            toRemove.push(entityRef);\n            toAdd.push({ deferred, hash: newHash });\n          } else if (newHash !== oldState.unprocessed_hash) {\n            // Entities with modifications should be pushed through too\n            toUpsert.push({ deferred, hash: newHash });\n          }\n        });\n      }\n\n      return { toAdd, toUpsert, toRemove };\n    }\n\n    // Grab all of the existing references from the same source, and their locationKeys as well\n    const oldRefs = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .leftJoin<DbRefreshStateRow>('refresh_state', {\n        target_entity_ref: 'entity_ref',\n      })\n      .where({ source_key: options.sourceKey })\n      .select({\n        target_entity_ref: 'refresh_state_references.target_entity_ref',\n        location_key: 'refresh_state.location_key',\n        unprocessed_hash: 'refresh_state.unprocessed_hash',\n      });\n\n    const items = options.items.map(deferred => ({\n      deferred,\n      ref: stringifyEntityRef(deferred.entity),\n      hash: generateStableHash(deferred.entity),\n    }));\n\n    const oldRefsSet = new Map(\n      oldRefs.map(r => [\n        r.target_entity_ref,\n        {\n          locationKey: r.location_key,\n          oldEntityHash: r.unprocessed_hash,\n        },\n      ]),\n    );\n    const newRefsSet = new Set(items.map(item => item.ref));\n\n    const toAdd = new Array<{ deferred: DeferredEntity; hash: string }>();\n    const toUpsert = new Array<{ deferred: DeferredEntity; hash: string }>();\n    const toRemove = oldRefs\n      .map(row => row.target_entity_ref)\n      .filter(ref => !newRefsSet.has(ref));\n\n    for (const item of items) {\n      const oldRef = oldRefsSet.get(item.ref);\n      const upsertItem = { deferred: item.deferred, hash: item.hash };\n      if (!oldRef) {\n        // Add any entity that does not exist in the database\n        toAdd.push(upsertItem);\n      } else if (\n        (oldRef.locationKey ?? undefined) !==\n        (item.deferred.locationKey ?? undefined)\n      ) {\n        // Remove and then re-add any entity that exists, but with a different location key\n        toRemove.push(item.ref);\n        toAdd.push(upsertItem);\n      } else if (oldRef.oldEntityHash !== item.hash) {\n        // Entities with modifications should be pushed through too\n        toUpsert.push(upsertItem);\n      }\n    }\n\n    return { toAdd, toUpsert, toRemove };\n  }\n}\n"],"names":["retryOnDeadlock","rethrowError","deleteWithEagerPruningOfChildren","lodash","uuid","stringifyEntityRef","isDatabaseConflictError","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","isDeadlockError","refreshByRefreshKeys","toAdd","toUpsert","toRemove","generateStableHash"],"mappings":";;;;;;;;;;;;;;;;;;AA4CA,MAAM,UAAA,GAAa,EAAA;AAEZ,MAAM,uBAAA,CAAoD;AAAA,EAC9C,OAAA;AAAA,EAKjB,YAAY,OAAA,EAAoD;AAC9D,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA;AAAA,EACjB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,YAAe,EAAA,EAAiD;AACpE,IAAA,OAAOA,qBAAgB,YAAY;AACjC,MAAA,IAAI;AACF,QAAA,IAAI,MAAA,GAAwB,KAAA,CAAA;AAC5B,QAAA,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,WAAA;AAAA,UAC1B,OAAM,EAAA,KAAM;AAIV,YAAA,MAAA,GAAS,MAAM,GAAG,EAAE,CAAA;AAAA,UACtB,CAAA;AAAA,UACA;AAAA;AAAA,YAEE,qBAAA,EAAuB;AAAA;AACzB,SACF;AACA,QAAA,OAAO,MAAA;AAAA,MACT,SAAS,CAAA,EAAG;AACV,QAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,0BAAA,EAA6B,CAAC,CAAA,CAAE,CAAA;AAC1D,QAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA,MACtB;AAAA,IACF,CAAA,EAAG,IAAA,CAAK,OAAA,CAAQ,QAAQ,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,0BAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,OAAO,QAAA,EAAU,QAAA,KAAa,MAAM,IAAA,CAAK,WAAA,CAAY,EAAA,EAAI,OAAO,CAAA;AAExE,IAAA,IAAI,SAAS,MAAA,EAAQ;AACnB,MAAA,MAAM,YAAA,GAAe,MAAMC,iEAAA,CAAiC;AAAA,QAC1D,IAAA,EAAM,EAAA;AAAA,QACN,UAAA,EAAY,QAAA;AAAA,QACZ,WAAW,OAAA,CAAQ;AAAA,OACpB,CAAA;AACD,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,QAClB,YAAY,YAAY,CAAA,WAAA,EAAc,IAAA,CAAK,SAAA,CAAU,QAAQ,CAAC,CAAA;AAAA,OAChE;AAAA,IACF;AAEA,IAAA,IAAI,MAAM,MAAA,EAAQ;AAUhB,MAAA,KAAA,MAAW,KAAA,IAASC,uBAAA,CAAO,KAAA,CAAM,KAAA,EAAO,EAAE,CAAA,EAAG;AAC3C,QAAA,IAAI;AACF,UAAA,MAAM,EAAA,CAAG,WAAA;AAAA,YACP,eAAA;AAAA,YACA,KAAA,CAAM,IAAI,CAAA,IAAA,MAAS;AAAA,cACjB,WAAWC,sBAAA,EAAK;AAAA,cAChB,UAAA,EAAYC,+BAAA,CAAmB,IAAA,CAAK,QAAA,CAAS,MAAM,CAAA;AAAA,cACnD,kBAAA,EAAoB,IAAA,CAAK,SAAA,CAAU,IAAA,CAAK,SAAS,MAAM,CAAA;AAAA,cACvD,kBAAkB,IAAA,CAAK,IAAA;AAAA,cACvB,MAAA,EAAQ,EAAA;AAAA,cACR,YAAA,EAAc,KAAK,QAAA,CAAS,WAAA;AAAA,cAC5B,cAAA,EAAgB,EAAA,CAAG,EAAA,CAAG,GAAA,EAAI;AAAA,cAC1B,iBAAA,EAAmB,EAAA,CAAG,EAAA,CAAG,GAAA;AAAI,aAC/B,CAAE,CAAA;AAAA,YACF;AAAA,WACF;AACA,UAAA,MAAM,EAAA,CAAG,WAAA;AAAA,YACP,0BAAA;AAAA,YACA,KAAA,CAAM,IAAI,CAAA,IAAA,MAAS;AAAA,cACjB,YAAY,OAAA,CAAQ,SAAA;AAAA,cACpB,iBAAA,EAAmBA,+BAAA,CAAmB,IAAA,CAAK,QAAA,CAAS,MAAM;AAAA,aAC5D,CAAE,CAAA;AAAA,YACF;AAAA,WACF;AAAA,QACF,SAAS,KAAA,EAAO;AACd,UAAA,IAAI,CAACC,wCAAA,CAAwB,KAAK,CAAA,EAAG;AACnC,YAAA,MAAM,KAAA;AAAA,UACR,CAAA,MAAO;AACL,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,cAClB,uDAAuD,KAAK,CAAA;AAAA,aAC9D;AACA,YAAA,QAAA,CAAS,IAAA,CAAK,GAAG,KAAK,CAAA;AAAA,UACxB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,IAAA,IAAI,SAAS,MAAA,EAAQ;AACnB,MAAA,KAAA,MAAW;AAAA,QACT,QAAA,EAAU,EAAE,MAAA,EAAQ,WAAA,EAAY;AAAA,QAChC;AAAA,WACG,QAAA,EAAU;AACb,QAAA,MAAM,SAAA,GAAYD,gCAAmB,MAAM,CAAA;AAE3C,QAAA,IAAI;AACF,UAAA,IAAI,EAAA,GAAK,MAAME,+CAAA,CAAwB;AAAA,YACrC,EAAA;AAAA,YACA,MAAA;AAAA,YACA,IAAA;AAAA,YACA;AAAA,WACD,CAAA;AACD,UAAA,IAAI,CAAC,EAAA,EAAI;AACP,YAAA,EAAA,GAAK,MAAMC,+CAAA,CAAwB;AAAA,cACjC,EAAA;AAAA,cACA,MAAA;AAAA,cACA,IAAA;AAAA,cACA,WAAA;AAAA,cACA,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,aACtB,CAAA;AAAA,UACH;AACA,UAAA,IAAI,EAAA,EAAI;AACN,YAAA,MAAM,GAAgC,0BAA0B,CAAA,CAC7D,MAAM,mBAAA,EAAqB,SAAS,EACpC,MAAA,EAAO;AAEV,YAAA,MAAM,EAAA;AAAA,cACJ;AAAA,cACA,MAAA,CAAO;AAAA,cACP,YAAY,OAAA,CAAQ,SAAA;AAAA,cACpB,iBAAA,EAAmB;AAAA,aACpB,CAAA;AAAA,UACH,CAAA,MAAO;AACL,YAAA,MAAM,EAAA,CAAgC,0BAA0B,CAAA,CAC7D,KAAA,CAAM,qBAAqB,SAAS,CAAA,CACpC,QAAA,CAAS,EAAE,UAAA,EAAY,OAAA,CAAQ,SAAA,EAAW,EAC1C,MAAA,EAAO;AAEV,YAAA,MAAM,cAAA,GAAiB,MAAMC,iDAAA,CAAyB;AAAA,cACpD,EAAA;AAAA,cACA,SAAA;AAAA,cACA;AAAA,aACD,CAAA;AACD,YAAA,IAAI,cAAA,EAAgB;AAClB,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,gBAClB,CAAA,OAAA,EAAU,QAAQ,SAAS,CAAA,gCAAA,EAAmC,SAAS,CAAA,uBAAA,EAA0B,cAAc,iBAAiB,WAAW,CAAA;AAAA,eAC7I;AAAA,YACF;AAAA,UACF;AAAA,QACF,SAAS,KAAA,EAAO;AACd,UAAA,IAAIC,oBAAA,CAAgB,EAAA,EAAI,KAAK,CAAA,EAAG;AAC9B,YAAA,MAAM,KAAA;AAAA,UACR;AACA,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,YAClB,kBAAkB,SAAS,CAAA,eAAA,EAAkB,OAAA,CAAQ,SAAS,MAAM,KAAK,CAAA;AAAA,WAC3E;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,wBAAwB,QAAA,EAA0C;AACtE,IAAA,MAAM,EAAA,GAAK,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,KACF,CACG,QAAA,CAAS,YAAY,CAAA,CACrB,aAAa,YAAY,CAAA;AAE5B,IAAA,OAAO,IAAA,CACJ,GAAA,CAAI,CAAA,GAAA,KAAO,GAAA,CAAI,UAAU,CAAA,CACzB,MAAA,CAAO,CAAC,GAAA,KAAuB,CAAC,CAAC,GAAG,CAAA;AAAA,EACzC;AAAA,EAEA,MAAM,oBAAA,CACJ,QAAA,EACA,OAAA,EACA;AACA,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAMC,0CAAqB,EAAE,EAAA,EAAI,IAAA,EAAM,OAAA,CAAQ,MAAM,CAAA;AAAA,EACvD;AAAA,EAEA,MAAc,WAAA,CACZ,EAAA,EACA,OAAA,EAKC;AACD,IAAA,IAAI,OAAA,CAAQ,SAAS,OAAA,EAAS;AAC5B,MAAA,MAAMC,MAAAA,GAAQ,IAAI,KAAA,EAAkD;AACpE,MAAA,MAAMC,SAAAA,GAAW,IAAI,KAAA,EAAkD;AACvE,MAAA,MAAMC,YAAW,OAAA,CAAQ,OAAA,CAAQ,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,SAAS,CAAA;AAErD,MAAA,KAAA,MAAW,SAASX,uBAAA,CAAO,KAAA,CAAM,OAAA,CAAQ,KAAA,EAAO,GAAI,CAAA,EAAG;AACrD,QAAA,MAAM,aAAa,KAAA,CAAM,GAAA,CAAI,OAAKE,+BAAA,CAAmB,CAAA,CAAE,MAAM,CAAC,CAAA;AAC9D,QAAA,MAAM,IAAA,GAAO,MAAM,EAAA,CAAsB,eAAe,EACrD,MAAA,CAAO,CAAC,YAAA,EAAc,kBAAA,EAAoB,cAAc,CAAC,CAAA,CACzD,OAAA,CAAQ,cAAc,UAAU,CAAA;AACnC,QAAA,MAAM,YAAY,IAAI,GAAA;AAAA,UACpB,IAAA,CAAK,IAAI,CAAA,GAAA,KAAO;AAAA,YACd,GAAA,CAAI,UAAA;AAAA,YACJ;AAAA,cACE,kBAAkB,GAAA,CAAI,gBAAA;AAAA,cACtB,cAAc,GAAA,CAAI;AAAA;AACpB,WACD;AAAA,SACH;AAEA,QAAA,KAAA,CAAM,OAAA,CAAQ,CAAC,QAAA,EAAU,CAAA,KAAM;AAC7B,UAAA,MAAM,SAAA,GAAY,WAAW,CAAC,CAAA;AAC9B,UAAA,MAAM,OAAA,GAAUU,uBAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAClD,UAAA,MAAM,QAAA,GAAW,SAAA,CAAU,GAAA,CAAI,SAAS,CAAA;AACxC,UAAA,IAAI,aAAa,MAAA,EAAW;AAE1B,YAAAH,OAAM,IAAA,CAAK,EAAE,QAAA,EAAU,IAAA,EAAM,SAAS,CAAA;AAAA,UACxC,YACG,QAAA,CAAS,WAAA,IAAe,IAAA,OAAW,QAAA,CAAS,gBAAgB,IAAA,CAAA,EAC7D;AAEA,YAAAE,SAAAA,CAAS,KAAK,SAAS,CAAA;AACvB,YAAAF,OAAM,IAAA,CAAK,EAAE,QAAA,EAAU,IAAA,EAAM,SAAS,CAAA;AAAA,UACxC,CAAA,MAAA,IAAW,OAAA,KAAY,QAAA,CAAS,gBAAA,EAAkB;AAEhD,YAAAC,UAAS,IAAA,CAAK,EAAE,QAAA,EAAU,IAAA,EAAM,SAAS,CAAA;AAAA,UAC3C;AAAA,QACF,CAAC,CAAA;AAAA,MACH;AAEA,MAAA,OAAO,EAAE,KAAA,EAAAD,MAAAA,EAAO,QAAA,EAAAC,SAAAA,EAAU,UAAAC,SAAAA,EAAS;AAAA,IACrC;AAGA,IAAA,MAAM,UAAU,MAAM,EAAA;AAAA,MACpB;AAAA,KACF,CACG,SAA4B,eAAA,EAAiB;AAAA,MAC5C,iBAAA,EAAmB;AAAA,KACpB,EACA,KAAA,CAAM,EAAE,YAAY,OAAA,CAAQ,SAAA,EAAW,CAAA,CACvC,MAAA,CAAO;AAAA,MACN,iBAAA,EAAmB,4CAAA;AAAA,MACnB,YAAA,EAAc,4BAAA;AAAA,MACd,gBAAA,EAAkB;AAAA,KACnB,CAAA;AAEH,IAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,KAAA,CAAM,GAAA,CAAI,CAAA,QAAA,MAAa;AAAA,MAC3C,QAAA;AAAA,MACA,GAAA,EAAKT,+BAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAAA,MACvC,IAAA,EAAMU,uBAAA,CAAmB,QAAA,CAAS,MAAM;AAAA,KAC1C,CAAE,CAAA;AAEF,IAAA,MAAM,aAAa,IAAI,GAAA;AAAA,MACrB,OAAA,CAAQ,IAAI,CAAA,CAAA,KAAK;AAAA,QACf,CAAA,CAAE,iBAAA;AAAA,QACF;AAAA,UACE,aAAa,CAAA,CAAE,YAAA;AAAA,UACf,eAAe,CAAA,CAAE;AAAA;AACnB,OACD;AAAA,KACH;AACA,IAAA,MAAM,UAAA,GAAa,IAAI,GAAA,CAAI,KAAA,CAAM,IAAI,CAAA,IAAA,KAAQ,IAAA,CAAK,GAAG,CAAC,CAAA;AAEtD,IAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAkD;AACpE,IAAA,MAAM,QAAA,GAAW,IAAI,KAAA,EAAkD;AACvE,IAAA,MAAM,QAAA,GAAW,OAAA,CACd,GAAA,CAAI,CAAA,GAAA,KAAO,GAAA,CAAI,iBAAiB,CAAA,CAChC,MAAA,CAAO,CAAA,GAAA,KAAO,CAAC,UAAA,CAAW,GAAA,CAAI,GAAG,CAAC,CAAA;AAErC,IAAA,KAAA,MAAW,QAAQ,KAAA,EAAO;AACxB,MAAA,MAAM,MAAA,GAAS,UAAA,CAAW,GAAA,CAAI,IAAA,CAAK,GAAG,CAAA;AACtC,MAAA,MAAM,aAAa,EAAE,QAAA,EAAU,KAAK,QAAA,EAAU,IAAA,EAAM,KAAK,IAAA,EAAK;AAC9D,MAAA,IAAI,CAAC,MAAA,EAAQ;AAEX,QAAA,KAAA,CAAM,KAAK,UAAU,CAAA;AAAA,MACvB,YACG,MAAA,CAAO,WAAA,IAAe,aACtB,IAAA,CAAK,QAAA,CAAS,eAAe,MAAA,CAAA,EAC9B;AAEA,QAAA,QAAA,CAAS,IAAA,CAAK,KAAK,GAAG,CAAA;AACtB,QAAA,KAAA,CAAM,KAAK,UAAU,CAAA;AAAA,MACvB,CAAA,MAAA,IAAW,MAAA,CAAO,aAAA,KAAkB,IAAA,CAAK,IAAA,EAAM;AAE7C,QAAA,QAAA,CAAS,KAAK,UAAU,CAAA;AAAA,MAC1B;AAAA,IACF;AAEA,IAAA,OAAO,EAAE,KAAA,EAAO,QAAA,EAAU,QAAA,EAAS;AAAA,EACrC;AACF;;;;"}
+{"version":3,"file":"DefaultProviderDatabase.cjs.js","sources":["../../src/database/DefaultProviderDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { stringifyEntityRef } from '@backstage/catalog-model';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { randomUUID as uuid } from 'node:crypto';\nimport { rethrowError } from './conversion';\nimport { deleteWithEagerPruningOfChildren } from './operations/provider/deleteWithEagerPruningOfChildren';\nimport { refreshByRefreshKeys } from './operations/provider/refreshByRefreshKeys';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { DbRefreshStateReferencesRow, DbRefreshStateRow } from './tables';\nimport {\n  ProviderDatabase,\n  RefreshByKeyOptions,\n  ReplaceUnprocessedEntitiesOptions,\n  Transaction,\n} from './types';\nimport {\n  generateStableHash,\n  isDeadlockError,\n  retryOnDeadlock,\n  whereInArray,\n} from './util';\nimport {\n  LoggerService,\n  isDatabaseConflictError,\n} from '@backstage/backend-plugin-api';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProviderDatabase implements ProviderDatabase {\n  private readonly options: {\n    database: Knex;\n    logger: LoggerService;\n  };\n\n  constructor(options: { database: Knex; logger: LoggerService }) {\n    this.options = options;\n  }\n\n  /**\n   * Executes `fn` inside a database transaction, retrying automatically on\n   * deadlock (PostgreSQL error 40P01). Because the callback may be invoked\n   * more than once, it must not perform non-database side effects such as\n   * emitting events, mutating in-memory state, or calling external services.\n   */\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    return retryOnDeadlock(async () => {\n      try {\n        let result: T | undefined = undefined;\n        await this.options.database.transaction(\n          async tx => {\n            // We can't return here, as knex swallows the return type in case the\n            // transaction is rolled back:\n            // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n            result = await fn(tx);\n          },\n          {\n            // If we explicitly trigger a rollback, don't fail.\n            doNotRejectOnRollback: true,\n          },\n        );\n        return result!;\n      } catch (e) {\n        this.options.logger.debug(`Error during transaction, ${e}`);\n        throw rethrowError(e);\n      }\n    }, this.options.database);\n  }\n\n  async replaceUnprocessedEntities(\n    txOpaque: Knex | Transaction,\n    options: ReplaceUnprocessedEntitiesOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex | Knex.Transaction;\n    const { toAdd, toUpsert, toRemove } = await this.createDelta(tx, options);\n\n    if (toRemove.length) {\n      const removedCount = await deleteWithEagerPruningOfChildren({\n        knex: tx,\n        entityRefs: toRemove,\n        sourceKey: options.sourceKey,\n      });\n      this.options.logger.debug(\n        `removed, ${removedCount} entities: ${JSON.stringify(toRemove)}`,\n      );\n    }\n\n    if (toAdd.length) {\n      // The reason for this chunking, rather than just massively batch\n      // inserting the entire payload, is that we fall back to the individual\n      // upsert mechanism below on conflicts. That path is massively slower than\n      // the fast batch path, so we don't want to end up accidentally having to\n      // for example item-by-item upsert tens of thousands of entities in a\n      // large initial delivery dump. The implication is that the size of these\n      // chunks needs to weigh the benefit of fast successful inserts, against\n      // the drawback of super slow but more rare fallbacks. There's quickly\n      // diminishing returns though with turning up this value way high.\n      for (const chunk of lodash.chunk(toAdd, 50)) {\n        try {\n          await tx.batchInsert(\n            'refresh_state',\n            chunk.map(item => ({\n              entity_id: uuid(),\n              entity_ref: stringifyEntityRef(item.deferred.entity),\n              unprocessed_entity: JSON.stringify(item.deferred.entity),\n              unprocessed_hash: item.hash,\n              errors: '',\n              location_key: item.deferred.locationKey,\n              next_update_at: tx.fn.now(),\n              last_discovery_at: tx.fn.now(),\n            })),\n            BATCH_SIZE,\n          );\n          await tx.batchInsert(\n            'refresh_state_references',\n            chunk.map(item => ({\n              source_key: options.sourceKey,\n              target_entity_ref: stringifyEntityRef(item.deferred.entity),\n            })),\n            BATCH_SIZE,\n          );\n        } catch (error) {\n          if (!isDatabaseConflictError(error)) {\n            throw error;\n          } else {\n            this.options.logger.debug(\n              `Fast insert path failed, falling back to slow path, ${error}`,\n            );\n            toUpsert.push(...chunk);\n          }\n        }\n      }\n    }\n\n    if (toUpsert.length) {\n      for (const {\n        deferred: { entity, locationKey },\n        hash,\n      } of toUpsert) {\n        const entityRef = stringifyEntityRef(entity);\n\n        try {\n          let ok = await updateUnprocessedEntity({\n            tx,\n            entity,\n            hash,\n            locationKey,\n          });\n          if (!ok) {\n            ok = await insertUnprocessedEntity({\n              tx,\n              entity,\n              hash,\n              locationKey,\n              logger: this.options.logger,\n            });\n          }\n          if (ok) {\n            await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n              .where('target_entity_ref', entityRef)\n              .delete();\n\n            await tx<DbRefreshStateReferencesRow>(\n              'refresh_state_references',\n            ).insert({\n              source_key: options.sourceKey,\n              target_entity_ref: entityRef,\n            });\n          } else {\n            await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n              .where('target_entity_ref', entityRef)\n              .andWhere({ source_key: options.sourceKey })\n              .delete();\n\n            const conflictingKey = await checkLocationKeyConflict({\n              tx,\n              entityRef,\n              locationKey,\n            });\n            if (conflictingKey) {\n              this.options.logger.warn(\n                `Source ${options.sourceKey} detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n              );\n            }\n          }\n        } catch (error) {\n          if (isDeadlockError(tx, error)) {\n            throw error;\n          }\n          this.options.logger.error(\n            `Failed to add '${entityRef}' from source '${options.sourceKey}', ${error}`,\n          );\n        }\n      }\n    }\n  }\n\n  async listReferenceSourceKeys(txOpaque: Transaction): Promise<string[]> {\n    const tx = txOpaque as Knex | Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .distinct('source_key')\n      .whereNotNull('source_key');\n\n    return rows\n      .map(row => row.source_key)\n      .filter((key): key is string => !!key);\n  }\n\n  async refreshByRefreshKeys(\n    txOpaque: Transaction,\n    options: RefreshByKeyOptions,\n  ) {\n    const tx = txOpaque as Knex.Transaction;\n    await refreshByRefreshKeys({ tx, keys: options.keys });\n  }\n\n  private async createDelta(\n    tx: Knex | Knex.Transaction,\n    options: ReplaceUnprocessedEntitiesOptions,\n  ): Promise<{\n    toAdd: { deferred: DeferredEntity; hash: string }[];\n    toUpsert: { deferred: DeferredEntity; hash: string }[];\n    toRemove: string[];\n  }> {\n    if (options.type === 'delta') {\n      const toAdd = new Array<{ deferred: DeferredEntity; hash: string }>();\n      const toUpsert = new Array<{ deferred: DeferredEntity; hash: string }>();\n      const toRemove = options.removed.map(e => e.entityRef);\n\n      for (const chunk of lodash.chunk(options.added, 1000)) {\n        const entityRefs = chunk.map(e => stringifyEntityRef(e.entity));\n        const rows = await tx<DbRefreshStateRow>('refresh_state')\n          .select(['entity_ref', 'unprocessed_hash', 'location_key'])\n          .where(whereInArray('entity_ref', entityRefs));\n        const oldStates = new Map(\n          rows.map(row => [\n            row.entity_ref,\n            {\n              unprocessed_hash: row.unprocessed_hash,\n              location_key: row.location_key,\n            },\n          ]),\n        );\n\n        chunk.forEach((deferred, i) => {\n          const entityRef = entityRefs[i];\n          const newHash = generateStableHash(deferred.entity);\n          const oldState = oldStates.get(entityRef);\n          if (oldState === undefined) {\n            // Add any entity that does not exist in the database\n            toAdd.push({ deferred, hash: newHash });\n          } else if (\n            (deferred.locationKey ?? null) !== (oldState.location_key ?? null)\n          ) {\n            // Remove and then re-add any entity that exists, but with a different location key\n            toRemove.push(entityRef);\n            toAdd.push({ deferred, hash: newHash });\n          } else if (newHash !== oldState.unprocessed_hash) {\n            // Entities with modifications should be pushed through too\n            toUpsert.push({ deferred, hash: newHash });\n          }\n        });\n      }\n\n      return { toAdd, toUpsert, toRemove };\n    }\n\n    // Grab all of the existing references from the same source, and their locationKeys as well\n    const oldRefs = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .leftJoin<DbRefreshStateRow>('refresh_state', {\n        target_entity_ref: 'entity_ref',\n      })\n      .where({ source_key: options.sourceKey })\n      .select({\n        target_entity_ref: 'refresh_state_references.target_entity_ref',\n        location_key: 'refresh_state.location_key',\n        unprocessed_hash: 'refresh_state.unprocessed_hash',\n      });\n\n    const items = options.items.map(deferred => ({\n      deferred,\n      ref: stringifyEntityRef(deferred.entity),\n      hash: generateStableHash(deferred.entity),\n    }));\n\n    const oldRefsSet = new Map(\n      oldRefs.map(r => [\n        r.target_entity_ref,\n        {\n          locationKey: r.location_key,\n          oldEntityHash: r.unprocessed_hash,\n        },\n      ]),\n    );\n    const newRefsSet = new Set(items.map(item => item.ref));\n\n    const toAdd = new Array<{ deferred: DeferredEntity; hash: string }>();\n    const toUpsert = new Array<{ deferred: DeferredEntity; hash: string }>();\n    const toRemove = oldRefs\n      .map(row => row.target_entity_ref)\n      .filter(ref => !newRefsSet.has(ref));\n\n    for (const item of items) {\n      const oldRef = oldRefsSet.get(item.ref);\n      const upsertItem = { deferred: item.deferred, hash: item.hash };\n      if (!oldRef) {\n        // Add any entity that does not exist in the database\n        toAdd.push(upsertItem);\n      } else if (\n        (oldRef.locationKey ?? undefined) !==\n        (item.deferred.locationKey ?? undefined)\n      ) {\n        // Remove and then re-add any entity that exists, but with a different location key\n        toRemove.push(item.ref);\n        toAdd.push(upsertItem);\n      } else if (oldRef.oldEntityHash !== item.hash) {\n        // Entities with modifications should be pushed through too\n        toUpsert.push(upsertItem);\n      }\n    }\n\n    return { toAdd, toUpsert, toRemove };\n  }\n}\n"],"names":["retryOnDeadlock","rethrowError","deleteWithEagerPruningOfChildren","lodash","uuid","stringifyEntityRef","isDatabaseConflictError","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","isDeadlockError","refreshByRefreshKeys","toAdd","toUpsert","toRemove","whereInArray","generateStableHash"],"mappings":";;;;;;;;;;;;;;;;;;AAiDA,MAAM,UAAA,GAAa,EAAA;AAEZ,MAAM,uBAAA,CAAoD;AAAA,EAC9C,OAAA;AAAA,EAKjB,YAAY,OAAA,EAAoD;AAC9D,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA;AAAA,EACjB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,YAAe,EAAA,EAAiD;AACpE,IAAA,OAAOA,qBAAgB,YAAY;AACjC,MAAA,IAAI;AACF,QAAA,IAAI,MAAA,GAAwB,KAAA,CAAA;AAC5B,QAAA,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,WAAA;AAAA,UAC1B,OAAM,EAAA,KAAM;AAIV,YAAA,MAAA,GAAS,MAAM,GAAG,EAAE,CAAA;AAAA,UACtB,CAAA;AAAA,UACA;AAAA;AAAA,YAEE,qBAAA,EAAuB;AAAA;AACzB,SACF;AACA,QAAA,OAAO,MAAA;AAAA,MACT,SAAS,CAAA,EAAG;AACV,QAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,0BAAA,EAA6B,CAAC,CAAA,CAAE,CAAA;AAC1D,QAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA,MACtB;AAAA,IACF,CAAA,EAAG,IAAA,CAAK,OAAA,CAAQ,QAAQ,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,0BAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,OAAO,QAAA,EAAU,QAAA,KAAa,MAAM,IAAA,CAAK,WAAA,CAAY,EAAA,EAAI,OAAO,CAAA;AAExE,IAAA,IAAI,SAAS,MAAA,EAAQ;AACnB,MAAA,MAAM,YAAA,GAAe,MAAMC,iEAAA,CAAiC;AAAA,QAC1D,IAAA,EAAM,EAAA;AAAA,QACN,UAAA,EAAY,QAAA;AAAA,QACZ,WAAW,OAAA,CAAQ;AAAA,OACpB,CAAA;AACD,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,QAClB,YAAY,YAAY,CAAA,WAAA,EAAc,IAAA,CAAK,SAAA,CAAU,QAAQ,CAAC,CAAA;AAAA,OAChE;AAAA,IACF;AAEA,IAAA,IAAI,MAAM,MAAA,EAAQ;AAUhB,MAAA,KAAA,MAAW,KAAA,IAASC,uBAAA,CAAO,KAAA,CAAM,KAAA,EAAO,EAAE,CAAA,EAAG;AAC3C,QAAA,IAAI;AACF,UAAA,MAAM,EAAA,CAAG,WAAA;AAAA,YACP,eAAA;AAAA,YACA,KAAA,CAAM,IAAI,CAAA,IAAA,MAAS;AAAA,cACjB,WAAWC,sBAAA,EAAK;AAAA,cAChB,UAAA,EAAYC,+BAAA,CAAmB,IAAA,CAAK,QAAA,CAAS,MAAM,CAAA;AAAA,cACnD,kBAAA,EAAoB,IAAA,CAAK,SAAA,CAAU,IAAA,CAAK,SAAS,MAAM,CAAA;AAAA,cACvD,kBAAkB,IAAA,CAAK,IAAA;AAAA,cACvB,MAAA,EAAQ,EAAA;AAAA,cACR,YAAA,EAAc,KAAK,QAAA,CAAS,WAAA;AAAA,cAC5B,cAAA,EAAgB,EAAA,CAAG,EAAA,CAAG,GAAA,EAAI;AAAA,cAC1B,iBAAA,EAAmB,EAAA,CAAG,EAAA,CAAG,GAAA;AAAI,aAC/B,CAAE,CAAA;AAAA,YACF;AAAA,WACF;AACA,UAAA,MAAM,EAAA,CAAG,WAAA;AAAA,YACP,0BAAA;AAAA,YACA,KAAA,CAAM,IAAI,CAAA,IAAA,MAAS;AAAA,cACjB,YAAY,OAAA,CAAQ,SAAA;AAAA,cACpB,iBAAA,EAAmBA,+BAAA,CAAmB,IAAA,CAAK,QAAA,CAAS,MAAM;AAAA,aAC5D,CAAE,CAAA;AAAA,YACF;AAAA,WACF;AAAA,QACF,SAAS,KAAA,EAAO;AACd,UAAA,IAAI,CAACC,wCAAA,CAAwB,KAAK,CAAA,EAAG;AACnC,YAAA,MAAM,KAAA;AAAA,UACR,CAAA,MAAO;AACL,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,cAClB,uDAAuD,KAAK,CAAA;AAAA,aAC9D;AACA,YAAA,QAAA,CAAS,IAAA,CAAK,GAAG,KAAK,CAAA;AAAA,UACxB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,IAAA,IAAI,SAAS,MAAA,EAAQ;AACnB,MAAA,KAAA,MAAW;AAAA,QACT,QAAA,EAAU,EAAE,MAAA,EAAQ,WAAA,EAAY;AAAA,QAChC;AAAA,WACG,QAAA,EAAU;AACb,QAAA,MAAM,SAAA,GAAYD,gCAAmB,MAAM,CAAA;AAE3C,QAAA,IAAI;AACF,UAAA,IAAI,EAAA,GAAK,MAAME,+CAAA,CAAwB;AAAA,YACrC,EAAA;AAAA,YACA,MAAA;AAAA,YACA,IAAA;AAAA,YACA;AAAA,WACD,CAAA;AACD,UAAA,IAAI,CAAC,EAAA,EAAI;AACP,YAAA,EAAA,GAAK,MAAMC,+CAAA,CAAwB;AAAA,cACjC,EAAA;AAAA,cACA,MAAA;AAAA,cACA,IAAA;AAAA,cACA,WAAA;AAAA,cACA,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,aACtB,CAAA;AAAA,UACH;AACA,UAAA,IAAI,EAAA,EAAI;AACN,YAAA,MAAM,GAAgC,0BAA0B,CAAA,CAC7D,MAAM,mBAAA,EAAqB,SAAS,EACpC,MAAA,EAAO;AAEV,YAAA,MAAM,EAAA;AAAA,cACJ;AAAA,cACA,MAAA,CAAO;AAAA,cACP,YAAY,OAAA,CAAQ,SAAA;AAAA,cACpB,iBAAA,EAAmB;AAAA,aACpB,CAAA;AAAA,UACH,CAAA,MAAO;AACL,YAAA,MAAM,EAAA,CAAgC,0BAA0B,CAAA,CAC7D,KAAA,CAAM,qBAAqB,SAAS,CAAA,CACpC,QAAA,CAAS,EAAE,UAAA,EAAY,OAAA,CAAQ,SAAA,EAAW,EAC1C,MAAA,EAAO;AAEV,YAAA,MAAM,cAAA,GAAiB,MAAMC,iDAAA,CAAyB;AAAA,cACpD,EAAA;AAAA,cACA,SAAA;AAAA,cACA;AAAA,aACD,CAAA;AACD,YAAA,IAAI,cAAA,EAAgB;AAClB,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,gBAClB,CAAA,OAAA,EAAU,QAAQ,SAAS,CAAA,gCAAA,EAAmC,SAAS,CAAA,uBAAA,EAA0B,cAAc,iBAAiB,WAAW,CAAA;AAAA,eAC7I;AAAA,YACF;AAAA,UACF;AAAA,QACF,SAAS,KAAA,EAAO;AACd,UAAA,IAAIC,oBAAA,CAAgB,EAAA,EAAI,KAAK,CAAA,EAAG;AAC9B,YAAA,MAAM,KAAA;AAAA,UACR;AACA,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,YAClB,kBAAkB,SAAS,CAAA,eAAA,EAAkB,OAAA,CAAQ,SAAS,MAAM,KAAK,CAAA;AAAA,WAC3E;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,wBAAwB,QAAA,EAA0C;AACtE,IAAA,MAAM,EAAA,GAAK,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,KACF,CACG,QAAA,CAAS,YAAY,CAAA,CACrB,aAAa,YAAY,CAAA;AAE5B,IAAA,OAAO,IAAA,CACJ,GAAA,CAAI,CAAA,GAAA,KAAO,GAAA,CAAI,UAAU,CAAA,CACzB,MAAA,CAAO,CAAC,GAAA,KAAuB,CAAC,CAAC,GAAG,CAAA;AAAA,EACzC;AAAA,EAEA,MAAM,oBAAA,CACJ,QAAA,EACA,OAAA,EACA;AACA,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAMC,0CAAqB,EAAE,EAAA,EAAI,IAAA,EAAM,OAAA,CAAQ,MAAM,CAAA;AAAA,EACvD;AAAA,EAEA,MAAc,WAAA,CACZ,EAAA,EACA,OAAA,EAKC;AACD,IAAA,IAAI,OAAA,CAAQ,SAAS,OAAA,EAAS;AAC5B,MAAA,MAAMC,MAAAA,GAAQ,IAAI,KAAA,EAAkD;AACpE,MAAA,MAAMC,SAAAA,GAAW,IAAI,KAAA,EAAkD;AACvE,MAAA,MAAMC,YAAW,OAAA,CAAQ,OAAA,CAAQ,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,SAAS,CAAA;AAErD,MAAA,KAAA,MAAW,SAASX,uBAAA,CAAO,KAAA,CAAM,OAAA,CAAQ,KAAA,EAAO,GAAI,CAAA,EAAG;AACrD,QAAA,MAAM,aAAa,KAAA,CAAM,GAAA,CAAI,OAAKE,+BAAA,CAAmB,CAAA,CAAE,MAAM,CAAC,CAAA;AAC9D,QAAA,MAAM,OAAO,MAAM,EAAA,CAAsB,eAAe,CAAA,CACrD,OAAO,CAAC,YAAA,EAAc,kBAAA,EAAoB,cAAc,CAAC,CAAA,CACzD,KAAA,CAAMU,iBAAA,CAAa,YAAA,EAAc,UAAU,CAAC,CAAA;AAC/C,QAAA,MAAM,YAAY,IAAI,GAAA;AAAA,UACpB,IAAA,CAAK,IAAI,CAAA,GAAA,KAAO;AAAA,YACd,GAAA,CAAI,UAAA;AAAA,YACJ;AAAA,cACE,kBAAkB,GAAA,CAAI,gBAAA;AAAA,cACtB,cAAc,GAAA,CAAI;AAAA;AACpB,WACD;AAAA,SACH;AAEA,QAAA,KAAA,CAAM,OAAA,CAAQ,CAAC,QAAA,EAAU,CAAA,KAAM;AAC7B,UAAA,MAAM,SAAA,GAAY,WAAW,CAAC,CAAA;AAC9B,UAAA,MAAM,OAAA,GAAUC,uBAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAClD,UAAA,MAAM,QAAA,GAAW,SAAA,CAAU,GAAA,CAAI,SAAS,CAAA;AACxC,UAAA,IAAI,aAAa,MAAA,EAAW;AAE1B,YAAAJ,OAAM,IAAA,CAAK,EAAE,QAAA,EAAU,IAAA,EAAM,SAAS,CAAA;AAAA,UACxC,YACG,QAAA,CAAS,WAAA,IAAe,IAAA,OAAW,QAAA,CAAS,gBAAgB,IAAA,CAAA,EAC7D;AAEA,YAAAE,SAAAA,CAAS,KAAK,SAAS,CAAA;AACvB,YAAAF,OAAM,IAAA,CAAK,EAAE,QAAA,EAAU,IAAA,EAAM,SAAS,CAAA;AAAA,UACxC,CAAA,MAAA,IAAW,OAAA,KAAY,QAAA,CAAS,gBAAA,EAAkB;AAEhD,YAAAC,UAAS,IAAA,CAAK,EAAE,QAAA,EAAU,IAAA,EAAM,SAAS,CAAA;AAAA,UAC3C;AAAA,QACF,CAAC,CAAA;AAAA,MACH;AAEA,MAAA,OAAO,EAAE,KAAA,EAAAD,MAAAA,EAAO,QAAA,EAAAC,SAAAA,EAAU,UAAAC,SAAAA,EAAS;AAAA,IACrC;AAGA,IAAA,MAAM,UAAU,MAAM,EAAA;AAAA,MACpB;AAAA,KACF,CACG,SAA4B,eAAA,EAAiB;AAAA,MAC5C,iBAAA,EAAmB;AAAA,KACpB,EACA,KAAA,CAAM,EAAE,YAAY,OAAA,CAAQ,SAAA,EAAW,CAAA,CACvC,MAAA,CAAO;AAAA,MACN,iBAAA,EAAmB,4CAAA;AAAA,MACnB,YAAA,EAAc,4BAAA;AAAA,MACd,gBAAA,EAAkB;AAAA,KACnB,CAAA;AAEH,IAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,KAAA,CAAM,GAAA,CAAI,CAAA,QAAA,MAAa;AAAA,MAC3C,QAAA;AAAA,MACA,GAAA,EAAKT,+BAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAAA,MACvC,IAAA,EAAMW,uBAAA,CAAmB,QAAA,CAAS,MAAM;AAAA,KAC1C,CAAE,CAAA;AAEF,IAAA,MAAM,aAAa,IAAI,GAAA;AAAA,MACrB,OAAA,CAAQ,IAAI,CAAA,CAAA,KAAK;AAAA,QACf,CAAA,CAAE,iBAAA;AAAA,QACF;AAAA,UACE,aAAa,CAAA,CAAE,YAAA;AAAA,UACf,eAAe,CAAA,CAAE;AAAA;AACnB,OACD;AAAA,KACH;AACA,IAAA,MAAM,UAAA,GAAa,IAAI,GAAA,CAAI,KAAA,CAAM,IAAI,CAAA,IAAA,KAAQ,IAAA,CAAK,GAAG,CAAC,CAAA;AAEtD,IAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAkD;AACpE,IAAA,MAAM,QAAA,GAAW,IAAI,KAAA,EAAkD;AACvE,IAAA,MAAM,QAAA,GAAW,OAAA,CACd,GAAA,CAAI,CAAA,GAAA,KAAO,GAAA,CAAI,iBAAiB,CAAA,CAChC,MAAA,CAAO,CAAA,GAAA,KAAO,CAAC,UAAA,CAAW,GAAA,CAAI,GAAG,CAAC,CAAA;AAErC,IAAA,KAAA,MAAW,QAAQ,KAAA,EAAO;AACxB,MAAA,MAAM,MAAA,GAAS,UAAA,CAAW,GAAA,CAAI,IAAA,CAAK,GAAG,CAAA;AACtC,MAAA,MAAM,aAAa,EAAE,QAAA,EAAU,KAAK,QAAA,EAAU,IAAA,EAAM,KAAK,IAAA,EAAK;AAC9D,MAAA,IAAI,CAAC,MAAA,EAAQ;AAEX,QAAA,KAAA,CAAM,KAAK,UAAU,CAAA;AAAA,MACvB,YACG,MAAA,CAAO,WAAA,IAAe,aACtB,IAAA,CAAK,QAAA,CAAS,eAAe,MAAA,CAAA,EAC9B;AAEA,QAAA,QAAA,CAAS,IAAA,CAAK,KAAK,GAAG,CAAA;AACtB,QAAA,KAAA,CAAM,KAAK,UAAU,CAAA;AAAA,MACvB,CAAA,MAAA,IAAW,MAAA,CAAO,aAAA,KAAkB,IAAA,CAAK,IAAA,EAAM;AAE7C,QAAA,QAAA,CAAS,KAAK,UAAU,CAAA;AAAA,MAC1B;AAAA,IACF;AAEA,IAAA,OAAO,EAAE,KAAA,EAAO,QAAA,EAAU,QAAA,EAAS;AAAA,EACrC;AACF;;;;"}

package/dist/database/util.cjs.js

@@ -15,6 +15,15 @@ function generateStableHash(entity) {
 function generateTargetKey(target) {
   return target.length > 255 ? `${target.slice(0, 180)}#sha256:${node_crypto.createHash("sha256").update(target).digest("hex")}` : target;
 }
+function whereInArray(column, values) {
+  return (qb) => {
+    if (qb.client.config.client === "pg") {
+      qb.whereRaw("?? = ANY(?::text[])", [column, values]);
+    } else {
+      qb.whereIn(column, values);
+    }
+  };
+}
 async function retryOnDeadlock(fn, knex, retries = 3, baseMs = 25) {
   let attempt = 0;
   for (; ; ) {
@@ -44,4 +53,5 @@ exports.generateStableHash = generateStableHash;
 exports.generateTargetKey = generateTargetKey;
 exports.isDeadlockError = isDeadlockError;
 exports.retryOnDeadlock = retryOnDeadlock;
+exports.whereInArray = whereInArray;
 //# sourceMappingURL=util.cjs.js.map

package/dist/database/util.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.cjs.js","sources":["../../src/database/util.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity } from '@backstage/catalog-model';\nimport { ErrorLike, isError } from '@backstage/errors';\nimport stableStringify from 'fast-json-stable-stringify';\nimport { Knex } from 'knex';\nimport { createHash } from 'node:crypto';\nimport { setTimeout as sleep } from 'node:timers/promises';\n\nexport function generateStableHash(entity: Entity) {\n  return createHash('sha1')\n    .update(stableStringify({ ...entity }))\n    .digest('hex');\n}\n\nexport function generateTargetKey(target: string) {\n  return target.length > 255\n    ? `${target.slice(0, 180)}#sha256:${createHash('sha256')\n        .update(target)\n        .digest('hex')}`\n    : target;\n}\n\n/**\n * Retries an operation on database deadlock errors.\n */\nexport async function retryOnDeadlock<T>(\n  fn: () => Promise<T>,\n  knex: Knex | Knex.Transaction,\n  retries = 3,\n  baseMs = 25,\n): Promise<T> {\n  let attempt = 0;\n  for (;;) {\n    try {\n      return await fn();\n    } catch (e: unknown) {\n      if (isDeadlockError(knex, e) && attempt < retries) {\n        await sleep(baseMs * Math.pow(2, attempt));\n        attempt++;\n        continue;\n      }\n      throw e;\n    }\n  }\n}\n\n/**\n * Checks if the given error is a deadlock error for the database engine in use.\n */\nexport function isDeadlockError(\n  knex: Knex | Knex.Transaction,\n  e: unknown,\n): e is ErrorLike {\n  if (knex.client.config.client.includes('pg')) {\n    // PostgreSQL deadlock detection via error code\n    return isError(e) && e.code === '40P01';\n  }\n\n  if (knex.client.config.client.includes('mysql')) {\n    return isError(e) && (e as any).errno === 1213;\n  }\n\n  return false;\n}\n"],"names":["createHash","stableStringify","sleep","isError"],"mappings":";;;;;;;;;;;AAuBO,SAAS,mBAAmB,MAAA,EAAgB;AACjD,EAAA,OAAOA,sBAAA,CAAW,MAAM,CAAA,CACrB,MAAA,CAAOC,gCAAA,CAAgB,EAAE,GAAG,MAAA,EAAQ,CAAC,CAAA,CACrC,MAAA,CAAO,KAAK,CAAA;AACjB;AAEO,SAAS,kBAAkB,MAAA,EAAgB;AAChD,EAAA,OAAO,OAAO,MAAA,GAAS,GAAA,GACnB,GAAG,MAAA,CAAO,KAAA,CAAM,GAAG,GAAG,CAAC,WAAWD,sBAAA,CAAW,QAAQ,EAClD,MAAA,CAAO,MAAM,EACb,MAAA,CAAO,KAAK,CAAC,CAAA,CAAA,GAChB,MAAA;AACN;AAKA,eAAsB,gBACpB,EAAA,EACA,IAAA,EACA,OAAA,GAAU,CAAA,EACV,SAAS,EAAA,EACG;AACZ,EAAA,IAAI,OAAA,GAAU,CAAA;AACd,EAAA,WAAS;AACP,IAAA,IAAI;AACF,MAAA,OAAO,MAAM,EAAA,EAAG;AAAA,IAClB,SAAS,CAAA,EAAY;AACnB,MAAA,IAAI,eAAA,CAAgB,IAAA,EAAM,CAAC,CAAA,IAAK,UAAU,OAAA,EAAS;AACjD,QAAA,MAAME,oBAAM,MAAA,GAAS,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,OAAO,CAAC,CAAA;AACzC,QAAA,OAAA,EAAA;AACA,QAAA;AAAA,MACF;AACA,MAAA,MAAM,CAAA;AAAA,IACR;AAAA,EACF;AACF;AAKO,SAAS,eAAA,CACd,MACA,CAAA,EACgB;AAChB,EAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,IAAI,CAAA,EAAG;AAE5C,IAAA,OAAOC,cAAA,CAAQ,CAAC,CAAA,IAAK,CAAA,CAAE,IAAA,KAAS,OAAA;AAAA,EAClC;AAEA,EAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC/C,IAAA,OAAOA,cAAA,CAAQ,CAAC,CAAA,IAAM,CAAA,CAAU,KAAA,KAAU,IAAA;AAAA,EAC5C;AAEA,EAAA,OAAO,KAAA;AACT;;;;;;;"}
+{"version":3,"file":"util.cjs.js","sources":["../../src/database/util.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity } from '@backstage/catalog-model';\nimport { ErrorLike, isError } from '@backstage/errors';\nimport stableStringify from 'fast-json-stable-stringify';\nimport { Knex } from 'knex';\nimport { createHash } from 'node:crypto';\nimport { setTimeout as sleep } from 'node:timers/promises';\n\nexport function generateStableHash(entity: Entity) {\n  return createHash('sha1')\n    .update(stableStringify({ ...entity }))\n    .digest('hex');\n}\n\nexport function generateTargetKey(target: string) {\n  return target.length > 255\n    ? `${target.slice(0, 180)}#sha256:${createHash('sha256')\n        .update(target)\n        .digest('hex')}`\n    : target;\n}\n\n/** Applies an array-backed WHERE clause with a stable query shape on PostgreSQL. */\nexport function whereInArray(column: string, values: string[]) {\n  return (qb: Knex.QueryBuilder) => {\n    if (qb.client.config.client === 'pg') {\n      qb.whereRaw('?? = ANY(?::text[])', [column, values]);\n    } else {\n      qb.whereIn(column, values);\n    }\n  };\n}\n\n/**\n * Retries an operation on database deadlock errors.\n */\nexport async function retryOnDeadlock<T>(\n  fn: () => Promise<T>,\n  knex: Knex | Knex.Transaction,\n  retries = 3,\n  baseMs = 25,\n): Promise<T> {\n  let attempt = 0;\n  for (;;) {\n    try {\n      return await fn();\n    } catch (e: unknown) {\n      if (isDeadlockError(knex, e) && attempt < retries) {\n        await sleep(baseMs * Math.pow(2, attempt));\n        attempt++;\n        continue;\n      }\n      throw e;\n    }\n  }\n}\n\n/**\n * Checks if the given error is a deadlock error for the database engine in use.\n */\nexport function isDeadlockError(\n  knex: Knex | Knex.Transaction,\n  e: unknown,\n): e is ErrorLike {\n  if (knex.client.config.client.includes('pg')) {\n    // PostgreSQL deadlock detection via error code\n    return isError(e) && e.code === '40P01';\n  }\n\n  if (knex.client.config.client.includes('mysql')) {\n    return isError(e) && (e as any).errno === 1213;\n  }\n\n  return false;\n}\n"],"names":["createHash","stableStringify","sleep","isError"],"mappings":";;;;;;;;;;;AAuBO,SAAS,mBAAmB,MAAA,EAAgB;AACjD,EAAA,OAAOA,sBAAA,CAAW,MAAM,CAAA,CACrB,MAAA,CAAOC,gCAAA,CAAgB,EAAE,GAAG,MAAA,EAAQ,CAAC,CAAA,CACrC,MAAA,CAAO,KAAK,CAAA;AACjB;AAEO,SAAS,kBAAkB,MAAA,EAAgB;AAChD,EAAA,OAAO,OAAO,MAAA,GAAS,GAAA,GACnB,GAAG,MAAA,CAAO,KAAA,CAAM,GAAG,GAAG,CAAC,WAAWD,sBAAA,CAAW,QAAQ,EAClD,MAAA,CAAO,MAAM,EACb,MAAA,CAAO,KAAK,CAAC,CAAA,CAAA,GAChB,MAAA;AACN;AAGO,SAAS,YAAA,CAAa,QAAgB,MAAA,EAAkB;AAC7D,EAAA,OAAO,CAAC,EAAA,KAA0B;AAChC,IAAA,IAAI,EAAA,CAAG,MAAA,CAAO,MAAA,CAAO,MAAA,KAAW,IAAA,EAAM;AACpC,MAAA,EAAA,CAAG,QAAA,CAAS,qBAAA,EAAuB,CAAC,MAAA,EAAQ,MAAM,CAAC,CAAA;AAAA,IACrD,CAAA,MAAO;AACL,MAAA,EAAA,CAAG,OAAA,CAAQ,QAAQ,MAAM,CAAA;AAAA,IAC3B;AAAA,EACF,CAAA;AACF;AAKA,eAAsB,gBACpB,EAAA,EACA,IAAA,EACA,OAAA,GAAU,CAAA,EACV,SAAS,EAAA,EACG;AACZ,EAAA,IAAI,OAAA,GAAU,CAAA;AACd,EAAA,WAAS;AACP,IAAA,IAAI;AACF,MAAA,OAAO,MAAM,EAAA,EAAG;AAAA,IAClB,SAAS,CAAA,EAAY;AACnB,MAAA,IAAI,eAAA,CAAgB,IAAA,EAAM,CAAC,CAAA,IAAK,UAAU,OAAA,EAAS;AACjD,QAAA,MAAME,oBAAM,MAAA,GAAS,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,OAAO,CAAC,CAAA;AACzC,QAAA,OAAA,EAAA;AACA,QAAA;AAAA,MACF;AACA,MAAA,MAAM,CAAA;AAAA,IACR;AAAA,EACF;AACF;AAKO,SAAS,eAAA,CACd,MACA,CAAA,EACgB;AAChB,EAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,IAAI,CAAA,EAAG;AAE5C,IAAA,OAAOC,cAAA,CAAQ,CAAC,CAAA,IAAK,CAAA,CAAE,IAAA,KAAS,OAAA;AAAA,EAClC;AAEA,EAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC/C,IAAA,OAAOA,cAAA,CAAQ,CAAC,CAAA,IAAM,CAAA,CAAU,KAAA,KAAU,IAAA;AAAA,EAC5C;AAEA,EAAA,OAAO,KAAA;AACT;;;;;;;;"}

package/dist/service/CatalogBuilder.cjs.js

@@ -260,9 +260,6 @@ class CatalogBuilder {
       catalogScmEvents,
       metrics
     } = this.env;
-    const enableRelationsCompatibility = Boolean(
-      config.getOptionalBoolean("catalog.enableRelationsCompatibility")
-    );
     const policy = this.buildEntityPolicy();
     const processors = this.buildProcessors();
     const parser = this.parser || parse.defaultEntityDataParser;
@@ -295,8 +292,7 @@ class CatalogBuilder {
     const rulesEnforcer = CatalogRules.DefaultCatalogRulesEnforcer.fromConfig(config);
     const unauthorizedEntitiesCatalog = new DefaultEntitiesCatalog.DefaultEntitiesCatalog({
       database: dbClient,
-      logger,
-      enableRelationsCompatibility
+      logger
     });
     const orchestrator = new DefaultCatalogProcessingOrchestrator.DefaultCatalogProcessingOrchestrator({
       processors,
@@ -400,8 +396,7 @@ class CatalogBuilder {
       auth,
       httpAuth,
       permissionsService: permissions,
-      auditor,
-      enableRelationsCompatibility
+      auditor
     });
     await connectEntityProviders.connectEntityProviders(providerDatabase, enabledProviderEntries);
     return {

package/dist/service/CatalogBuilder.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"CatalogBuilder.cjs.js","sources":["../../src/service/CatalogBuilder.ts"],"sourcesContent":["/*\n * Copyright 2020 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  DefaultNamespaceEntityPolicy,\n  Entity,\n  EntityPolicies,\n  EntityPolicy,\n  FieldFormatEntityPolicy,\n  makeValidator,\n  NoForeignRootFieldsEntityPolicy,\n  SchemaValidEntityPolicy,\n  Validators,\n} from '@backstage/catalog-model';\nimport { ScmIntegrations } from '@backstage/integration';\nimport { createHash } from 'node:crypto';\nimport { Router } from 'express';\nimport lodash from 'lodash';\n\nimport {\n  AuditorService,\n  AuthService,\n  DatabaseService,\n  HttpAuthService,\n  LoggerService,\n  PermissionsRegistryService,\n  PermissionsService,\n  RootConfigService,\n  SchedulerService,\n  UrlReaderService,\n} from '@backstage/backend-plugin-api';\nimport { Config, readDurationFromConfig } from '@backstage/config';\nimport { catalogPermissions } from '@backstage/plugin-catalog-common/alpha';\nimport {\n  CatalogProcessor,\n  CatalogProcessorParser,\n  LocationAnalyzer,\n  PlaceholderResolver,\n  ScmLocationAnalyzer,\n} from '@backstage/plugin-catalog-node';\nimport { EventsService } from '@backstage/plugin-events-node';\nimport { createConditionTransformer } from '@backstage/plugin-permission-node';\nimport { durationToMilliseconds } from '@backstage/types';\nimport { DefaultCatalogDatabase } from '../database/DefaultCatalogDatabase';\nimport { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase';\nimport { DefaultProviderDatabase } from '../database/DefaultProviderDatabase';\nimport { applyDatabaseMigrations } from '../database/migrations';\nimport { DefaultCatalogRulesEnforcer } from '../ingestion/CatalogRules';\nimport { RepoLocationAnalyzer } from '../ingestion/LocationAnalyzer';\nimport { permissionRules as catalogPermissionRules } from '../permissions/rules';\nimport { CatalogProcessingEngine } from '../processing/types';\nimport {\n  createRandomProcessingInterval,\n  ProcessingIntervalFunction,\n} from '../processing/refresh';\nimport {\n  connectEntityProviders,\n  EntityProviderEntry,\n} from '../processing/connectEntityProviders';\nimport { evictEntitiesFromOrphanedProviders } from '../processing/evictEntitiesFromOrphanedProviders';\nimport { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine';\nimport { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator';\nimport {\n  AnnotateLocationEntityProcessor,\n  BuiltinKindsEntityProcessor,\n  FileReaderProcessor,\n  PlaceholderProcessor,\n  UrlReaderProcessor,\n} from '../processors';\nimport {\n  jsonPlaceholderResolver,\n  textPlaceholderResolver,\n  yamlPlaceholderResolver,\n} from '../processors/PlaceholderProcessor';\nimport { ConfigLocationEntityProvider } from '../providers/ConfigLocationEntityProvider';\nimport { DefaultLocationStore } from '../providers/DefaultLocationStore';\nimport { DefaultStitcher } from '../stitching/DefaultStitcher';\nimport { defaultEntityDataParser } from '../util/parse';\nimport { AuthorizedEntitiesCatalog } from './AuthorizedEntitiesCatalog';\nimport { AuthorizedLocationAnalyzer } from './AuthorizedLocationAnalyzer';\nimport { AuthorizedLocationService } from './AuthorizedLocationService';\nimport { AuthorizedRefreshService } from './AuthorizedRefreshService';\nimport { createRouter } from './createRouter';\nimport { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog';\nimport { DefaultLocationService } from './DefaultLocationService';\nimport { DefaultRefreshService } from './DefaultRefreshService';\nimport { entitiesResponseToObjects } from './response';\nimport {\n  catalogEntityPermissionResourceRef,\n  CatalogScmEventsService,\n} from '@backstage/plugin-catalog-node/alpha';\nimport { filterAndSortProcessors, filterProviders } from './util';\nimport { GenericScmEventRefreshProvider } from '../providers/GenericScmEventRefreshProvider';\nimport { readScmEventHandlingConfig } from '../util/readScmEventHandlingConfig';\nimport { MetricsService } from '@backstage/backend-plugin-api/alpha';\nimport { ModelProcessor } from '../processors/ModelProcessor';\nimport { ModelHolder } from '../model/ModelHolder';\n\nexport type CatalogEnvironment = {\n  logger: LoggerService;\n  modelHolder?: ModelHolder;\n  database: DatabaseService;\n  config: RootConfigService;\n  reader: UrlReaderService;\n  permissions: PermissionsService;\n  permissionsRegistry: PermissionsRegistryService;\n  scheduler: SchedulerService;\n  auth: AuthService;\n  httpAuth: HttpAuthService;\n  auditor: AuditorService;\n  events: EventsService;\n  catalogScmEvents: CatalogScmEventsService;\n  metrics: MetricsService;\n};\n\n/**\n * A builder that helps wire up all of the component parts of the catalog.\n *\n * The touch points where you can replace or extend behavior are as follows:\n *\n * - Entity policies can be added or replaced. These are automatically run\n *   after the processors' pre-processing steps. All policies are given the\n *   chance to inspect the entity, and all of them have to pass in order for\n *   the entity to be considered valid from an overall point of view.\n * - Location analyzers can be added. These are responsible for analyzing\n *   repositories when onboarding them into the catalog, by finding\n *   catalog-info.yaml files and other artifacts that can help automatically\n *   register or create catalog data on the user's behalf.\n * - Placeholder resolvers can be replaced or added. These run on the raw\n *   structured data between the parsing and pre-processing steps, to replace\n *   dollar-prefixed entries with their actual values (like $file).\n * - Field format validators can be replaced. These check the format of\n *   individual core fields such as metadata.name, to ensure that they adhere\n *   to certain rules.\n * - Processors can be added or replaced. These implement the functionality of\n *   reading, parsing, validating, and processing the entity data before it is\n *   persisted in the catalog.\n *\n * @internal\n */\nexport class CatalogBuilder {\n  private readonly env: CatalogEnvironment;\n  private entityPolicies: EntityPolicy[];\n  private entityPoliciesReplace: boolean;\n  private placeholderResolvers: Record<string, PlaceholderResolver>;\n  private fieldFormatValidators: Partial<Validators>;\n  private entityProviders: EntityProviderEntry[];\n  private processors: CatalogProcessor[];\n  private locationAnalyzers: ScmLocationAnalyzer[];\n  private processorsReplace: boolean;\n  private parser: CatalogProcessorParser | undefined;\n  private onProcessingError?: (event: {\n    unprocessedEntity: Entity;\n    errors: Error[];\n  }) => Promise<void> | void;\n  private processingInterval: ProcessingIntervalFunction;\n  private locationAnalyzer: LocationAnalyzer | undefined = undefined;\n  private allowedLocationType: string[];\n\n  /**\n   * Creates a catalog builder.\n   */\n  static create(env: CatalogEnvironment): CatalogBuilder {\n    return new CatalogBuilder(env);\n  }\n\n  private constructor(env: CatalogEnvironment) {\n    this.env = env;\n    this.entityPolicies = [];\n    this.entityPoliciesReplace = false;\n    this.placeholderResolvers = {};\n    this.fieldFormatValidators = {};\n    this.entityProviders = [];\n    this.processors = [];\n    this.locationAnalyzers = [];\n    this.processorsReplace = false;\n    this.parser = undefined;\n    this.allowedLocationType = ['url'];\n\n    this.processingInterval = CatalogBuilder.getDefaultProcessingInterval(\n      env.config,\n    );\n  }\n\n  /**\n   * Adds policies that are used to validate entities between the pre-\n   * processing and post-processing stages. All such policies must pass for the\n   * entity to be considered valid.\n   *\n   * If what you want to do is to replace the rules for what format is allowed\n   * in various core entity fields (such as metadata.name), you may want to use\n   * {@link CatalogBuilder#setFieldFormatValidators} instead.\n   *\n   * @param policies - One or more policies\n   */\n  addEntityPolicy(\n    ...policies: Array<EntityPolicy | Array<EntityPolicy>>\n  ): CatalogBuilder {\n    this.entityPolicies.push(...policies.flat());\n    return this;\n  }\n\n  /**\n   * Overwrites the default location analyzer.\n   */\n  setLocationAnalyzer(locationAnalyzer: LocationAnalyzer): CatalogBuilder {\n    this.locationAnalyzer = locationAnalyzer;\n    return this;\n  }\n\n  /**\n   * Sets what policies to use for validation of entities between the pre-\n   * processing and post-processing stages. All such policies must pass for the\n   * entity to be considered valid.\n   *\n   * If what you want to do is to replace the rules for what format is allowed\n   * in various core entity fields (such as metadata.name), you may want to use\n   * {@link CatalogBuilder#setFieldFormatValidators} instead.\n   *\n   * This function replaces the default set of policies; use with care.\n   *\n   * @param policies - One or more policies\n   */\n  replaceEntityPolicies(policies: EntityPolicy[]): CatalogBuilder {\n    this.entityPolicies = [...policies];\n    this.entityPoliciesReplace = true;\n    return this;\n  }\n\n  /**\n   * Adds, or overwrites, a handler for placeholders (e.g. $file) in entity\n   * definition files.\n   *\n   * @param key - The key that identifies the placeholder, e.g. \"file\"\n   * @param resolver - The resolver that gets values for this placeholder\n   */\n  setPlaceholderResolver(\n    key: string,\n    resolver: PlaceholderResolver,\n  ): CatalogBuilder {\n    this.placeholderResolvers[key] = resolver;\n    return this;\n  }\n\n  /**\n   * Sets the validator function to use for one or more special fields of an\n   * entity. This is useful if the default rules for formatting of fields are\n   * not sufficient.\n   *\n   * This function has no effect if used together with\n   * {@link CatalogBuilder#replaceEntityPolicies}.\n   *\n   * @param validators - The (subset of) validators to set\n   */\n  setFieldFormatValidators(validators: Partial<Validators>): CatalogBuilder {\n    lodash.merge(this.fieldFormatValidators, validators);\n    return this;\n  }\n\n  /**\n   * Adds or replaces entity providers. These are responsible for bootstrapping\n   * the list of entities out of original data sources. For example, there is\n   * one entity source for the config locations, and one for the database\n   * stored locations. If you ingest entities out of a third party system, you\n   * may want to implement that in terms of an entity provider as well.\n   *\n   * @param providers - One or more entity providers\n   */\n  addEntityProvider(\n    ...providers: Array<EntityProviderEntry | Array<EntityProviderEntry>>\n  ): CatalogBuilder {\n    this.entityProviders.push(...providers.flat());\n    return this;\n  }\n\n  /**\n   * Adds entity processors. These are responsible for reading, parsing, and\n   * processing entities before they are persisted in the catalog.\n   *\n   * @param processors - One or more processors\n   */\n  addProcessor(\n    ...processors: Array<CatalogProcessor | Array<CatalogProcessor>>\n  ): CatalogBuilder {\n    this.processors.push(...processors.flat());\n    return this;\n  }\n\n  /**\n   * Sets what entity processors to use. These are responsible for reading,\n   * parsing, and processing entities before they are persisted in the catalog.\n   *\n   * This function replaces the default set of processors, consider using with\n   * {@link CatalogBuilder#getDefaultProcessors}; use with care.\n   *\n   * @param processors - One or more processors\n   */\n  replaceProcessors(processors: CatalogProcessor[]): CatalogBuilder {\n    this.processors = [...processors];\n    this.processorsReplace = true;\n    return this;\n  }\n\n  /**\n   * Returns the default list of entity processors. These are responsible for reading,\n   * parsing, and processing entities before they are persisted in the catalog. Changing\n   * the order of processing can give more control to custom processors.\n   *\n   * Consider using with {@link CatalogBuilder#replaceProcessors}\n   *\n   */\n  getDefaultProcessors(): CatalogProcessor[] {\n    const { config, logger, reader } = this.env;\n    const integrations = ScmIntegrations.fromConfig(config);\n\n    return [\n      new FileReaderProcessor(),\n      new UrlReaderProcessor({ reader, logger }),\n      new AnnotateLocationEntityProcessor({ integrations }),\n    ];\n  }\n\n  /**\n   * Adds Location Analyzers. These are responsible for analyzing\n   * repositories when onboarding them into the catalog, by finding\n   * catalog-info.yaml files and other artifacts that can help automatically\n   * register or create catalog data on the user's behalf.\n   *\n   * @param locationAnalyzers - One or more location analyzers\n   */\n  addLocationAnalyzers(\n    ...analyzers: Array<ScmLocationAnalyzer | Array<ScmLocationAnalyzer>>\n  ): CatalogBuilder {\n    this.locationAnalyzers.push(...analyzers.flat());\n    return this;\n  }\n\n  /**\n   * Sets up the catalog to use a custom parser for entity data.\n   *\n   * This is the function that gets called immediately after some raw entity\n   * specification data has been read from a remote source, and needs to be\n   * parsed and emitted as structured data.\n   *\n   * @param parser - The custom parser\n   */\n  setEntityDataParser(parser: CatalogProcessorParser): CatalogBuilder {\n    this.parser = parser;\n    return this;\n  }\n\n  /**\n   * Sets up the allowed location types from being registered via the location service.\n   *\n   * @param allowedLocationTypes - the allowed location types\n   */\n  setAllowedLocationTypes(allowedLocationTypes: string[]): CatalogBuilder {\n    this.allowedLocationType = allowedLocationTypes;\n    return this;\n  }\n\n  /**\n   * Wires up and returns all of the component parts of the catalog\n   */\n  async build(): Promise<{\n    processingEngine: CatalogProcessingEngine;\n    router: Router;\n  }> {\n    const {\n      config,\n      database,\n      logger,\n      permissions,\n      scheduler,\n      permissionsRegistry,\n      auditor,\n      auth,\n      httpAuth,\n      events,\n      catalogScmEvents,\n      metrics,\n    } = this.env;\n\n    const enableRelationsCompatibility = Boolean(\n      config.getOptionalBoolean('catalog.enableRelationsCompatibility'),\n    );\n\n    const policy = this.buildEntityPolicy();\n    const processors = this.buildProcessors();\n    const parser = this.parser || defaultEntityDataParser;\n\n    const dbClient = await database.getClient();\n    if (!database.migrations?.skip) {\n      logger.info('Performing database migration');\n      await applyDatabaseMigrations(dbClient);\n    }\n\n    const stitcher = DefaultStitcher.fromConfig(config, {\n      knex: dbClient,\n      logger,\n      metrics,\n    });\n\n    const processingDatabase = new DefaultProcessingDatabase({\n      database: dbClient,\n      logger,\n      events,\n      refreshInterval: this.processingInterval,\n      metrics,\n    });\n    const providerDatabase = new DefaultProviderDatabase({\n      database: dbClient,\n      logger,\n    });\n    const catalogDatabase = new DefaultCatalogDatabase({\n      database: dbClient,\n      logger,\n    });\n    const integrations = ScmIntegrations.fromConfig(config);\n    const rulesEnforcer = DefaultCatalogRulesEnforcer.fromConfig(config);\n\n    const unauthorizedEntitiesCatalog = new DefaultEntitiesCatalog({\n      database: dbClient,\n      logger,\n      enableRelationsCompatibility,\n    });\n\n    const orchestrator = new DefaultCatalogProcessingOrchestrator({\n      processors,\n      integrations,\n      rulesEnforcer,\n      logger,\n      parser,\n      policy,\n      allowedLocationTypes: this.allowedLocationType,\n    });\n\n    const entitiesCatalog = new AuthorizedEntitiesCatalog(\n      unauthorizedEntitiesCatalog,\n      permissions,\n      createConditionTransformer(\n        permissionsRegistry.getPermissionRuleset(\n          catalogEntityPermissionResourceRef,\n        ),\n      ),\n    );\n\n    const getResources = async (resourceRefs: string[]) => {\n      const { items } = await unauthorizedEntitiesCatalog.entitiesBatch({\n        credentials: await auth.getOwnServiceCredentials(),\n        entityRefs: resourceRefs,\n      });\n\n      return entitiesResponseToObjects(items).map(e => e || undefined);\n    };\n\n    permissionsRegistry.addResourceType({\n      resourceRef: catalogEntityPermissionResourceRef,\n      getResources,\n      permissions: [...catalogPermissions],\n      rules: Object.values(catalogPermissionRules),\n    });\n\n    const scmEventHandlingConfig = readScmEventHandlingConfig(config);\n    const locationStore = new DefaultLocationStore(\n      dbClient,\n      catalogScmEvents,\n      scmEventHandlingConfig,\n    );\n    const configLocationProvider = new ConfigLocationEntityProvider(config);\n    const scmEvents = new GenericScmEventRefreshProvider(\n      dbClient,\n      catalogScmEvents,\n      scmEventHandlingConfig,\n    );\n\n    const entityProviderEntries = lodash.uniqBy(\n      [\n        ...this.entityProviders,\n        { provider: locationStore },\n        { provider: configLocationProvider },\n        { provider: scmEvents },\n      ],\n      entry => entry.provider.getProviderName(),\n    );\n    const enabledProviderEntries = filterProviders(\n      entityProviderEntries,\n      config,\n    );\n    const enabledProviders = enabledProviderEntries.map(\n      entry => entry.provider,\n    );\n\n    const processingEngine = new DefaultCatalogProcessingEngine({\n      config,\n      scheduler,\n      logger,\n      knex: dbClient,\n      processingDatabase,\n      orchestrator,\n      createHash: () => createHash('sha1'),\n      pollingIntervalMs: 1000,\n      onProcessingError: event => {\n        this.onProcessingError?.(event);\n      },\n      events,\n      metrics,\n    });\n\n    const locationAnalyzer =\n      this.locationAnalyzer ??\n      new AuthorizedLocationAnalyzer(\n        new RepoLocationAnalyzer(logger, integrations, this.locationAnalyzers),\n        permissions,\n      );\n    const locationService = new AuthorizedLocationService(\n      new DefaultLocationService(locationStore, orchestrator, {\n        allowedLocationTypes: this.allowedLocationType,\n        defaultLocationConflictStrategy:\n          (config.getOptionalString(\n            'catalog.defaultLocationConflictStrategy',\n          ) as 'refresh' | 'reject') || 'reject',\n      }),\n      permissions,\n    );\n    const refreshService = new AuthorizedRefreshService(\n      new DefaultRefreshService({ database: catalogDatabase }),\n      permissions,\n    );\n\n    const router = await createRouter({\n      entitiesCatalog,\n      locationAnalyzer,\n      locationService,\n      orchestrator,\n      refreshService,\n      logger,\n      config,\n      auth,\n      httpAuth,\n      permissionsService: permissions,\n      auditor,\n      enableRelationsCompatibility,\n    });\n\n    await connectEntityProviders(providerDatabase, enabledProviderEntries);\n\n    return {\n      processingEngine: {\n        async start() {\n          if (\n            config.getOptionalString('catalog.orphanProviderStrategy') !==\n            'keep'\n          ) {\n            await evictEntitiesFromOrphanedProviders({\n              db: providerDatabase,\n              providers: enabledProviders,\n              logger,\n            });\n          }\n          await processingEngine.start();\n          await stitcher.start();\n        },\n        async stop() {\n          await processingEngine.stop();\n          await stitcher.stop();\n        },\n      },\n      router,\n    };\n  }\n\n  subscribe(options: {\n    onProcessingError: (event: {\n      unprocessedEntity: Entity;\n      errors: Error[];\n    }) => Promise<void> | void;\n  }) {\n    this.onProcessingError = options.onProcessingError;\n  }\n\n  private buildEntityPolicy(): EntityPolicy {\n    const entityPolicies: EntityPolicy[] = this.entityPoliciesReplace\n      ? [new SchemaValidEntityPolicy(), ...this.entityPolicies]\n      : [\n          new SchemaValidEntityPolicy(),\n          new DefaultNamespaceEntityPolicy(),\n          new NoForeignRootFieldsEntityPolicy(),\n          new FieldFormatEntityPolicy(\n            makeValidator(this.fieldFormatValidators),\n          ),\n          ...this.entityPolicies,\n        ];\n\n    return EntityPolicies.allOf(entityPolicies);\n  }\n\n  private buildProcessors(): CatalogProcessor[] {\n    const { config, reader } = this.env;\n    const integrations = ScmIntegrations.fromConfig(config);\n\n    this.checkDeprecatedReaderProcessors();\n\n    const placeholderResolvers: Record<string, PlaceholderResolver> = {\n      json: jsonPlaceholderResolver,\n      yaml: yamlPlaceholderResolver,\n      text: textPlaceholderResolver,\n      ...this.placeholderResolvers,\n    };\n\n    // The placeholder is always there no matter what\n    const processors: CatalogProcessor[] = [\n      new PlaceholderProcessor({\n        resolvers: placeholderResolvers,\n        reader,\n        integrations,\n      }),\n    ];\n\n    const builtinKindsEntityProcessor = new BuiltinKindsEntityProcessor();\n    // If the user adds a processor named 'BuiltinKindsEntityProcessor', skip\n    // inclusion of the catalog-backend version. Same if there's a model\n    // registered - then we are using the new model flow.\n    if (\n      !this.env.modelHolder &&\n      !this.processors.some(\n        processor =>\n          processor.getProcessorName() ===\n          builtinKindsEntityProcessor.getProcessorName(),\n      )\n    ) {\n      processors.push(builtinKindsEntityProcessor);\n    }\n    if (this.env.modelHolder) {\n      processors.push(new ModelProcessor(this.env.modelHolder));\n    }\n\n    const disableDefaultProcessors = config.getOptionalBoolean(\n      'catalog.disableDefaultProcessors',\n    );\n\n    // Add default processors if:\n    // - processors have NOT been explicitly replaced\n    // - and default processors are NOT disabled via config\n    if (!this.processorsReplace && !disableDefaultProcessors) {\n      processors.push(...this.getDefaultProcessors());\n    }\n\n    // Add the ones (if any) that the user added\n    processors.push(...this.processors);\n\n    this.checkMissingExternalProcessors(processors);\n\n    return filterAndSortProcessors(processors, config);\n  }\n\n  // TODO(Rugvip): These old processors are removed, for a while we'll be throwing\n  //               errors here to make sure people know where to move the config\n  private checkDeprecatedReaderProcessors() {\n    const pc = this.env.config.getOptionalConfig('catalog.processors');\n    if (pc?.has('github')) {\n      throw new Error(\n        `Using deprecated configuration for catalog.processors.github, move to using integrations.github instead`,\n      );\n    }\n    if (pc?.has('gitlabApi')) {\n      throw new Error(\n        `Using deprecated configuration for catalog.processors.gitlabApi, move to using integrations.gitlab instead`,\n      );\n    }\n    if (pc?.has('bitbucketApi')) {\n      throw new Error(\n        `Using deprecated configuration for catalog.processors.bitbucketApi, move to using integrations.bitbucket instead`,\n      );\n    }\n    if (pc?.has('azureApi')) {\n      throw new Error(\n        `Using deprecated configuration for catalog.processors.azureApi, move to using integrations.azure instead`,\n      );\n    }\n  }\n\n  // TODO(freben): This can be removed no sooner than June 2022, after adopters have had some time to adapt to the new package structure\n  private checkMissingExternalProcessors(processors: CatalogProcessor[]) {\n    const skipCheckVarName = 'BACKSTAGE_CATALOG_SKIP_MISSING_PROCESSORS_CHECK';\n    if (process.env[skipCheckVarName]) {\n      return;\n    }\n\n    const locationTypes = new Set(\n      this.env.config\n        .getOptionalConfigArray('catalog.locations')\n        ?.map(l => l.getString('type')) ?? [],\n    );\n    const processorNames = new Set(processors.map(p => p.getProcessorName()));\n\n    function check(\n      locationType: string,\n      processorName: string,\n      installationUrl: string,\n    ) {\n      if (\n        locationTypes.has(locationType) &&\n        !processorNames.has(processorName)\n      ) {\n        throw new Error(\n          [\n            `Your config contains a \"catalog.locations\" entry of type ${locationType},`,\n            `but does not have the corresponding catalog processor ${processorName} installed.`,\n            `This processor used to be built into the catalog itself, but is now moved to an`,\n            `external module that has to be installed manually. Please follow the installation`,\n            `instructions at ${installationUrl} if you are using this ability, or remove the`,\n            `location from your app config if you do not. You can also silence this check entirely`,\n            `by setting the environment variable ${skipCheckVarName} to 'true'.`,\n          ].join(' '),\n        );\n      }\n    }\n\n    check(\n      'aws-cloud-accounts',\n      'AwsOrganizationCloudAccountProcessor',\n      'https://backstage.io/docs/integrations',\n    );\n    check(\n      's3-discovery',\n      'AwsS3DiscoveryProcessor',\n      'https://backstage.io/docs/integrations/aws-s3/discovery',\n    );\n    check(\n      'azure-discovery',\n      'AzureDevOpsDiscoveryProcessor',\n      'https://backstage.io/docs/integrations/azure/discovery',\n    );\n    check(\n      'bitbucket-discovery',\n      'BitbucketDiscoveryProcessor',\n      'https://backstage.io/docs/integrations/bitbucket/discovery',\n    );\n    check(\n      'github-discovery',\n      'GithubDiscoveryProcessor',\n      'https://backstage.io/docs/integrations/github/discovery',\n    );\n    check(\n      'github-org',\n      'GithubOrgReaderProcessor',\n      'https://backstage.io/docs/integrations/github/org',\n    );\n    check(\n      'gitlab-discovery',\n      'GitLabDiscoveryProcessor',\n      'https://backstage.io/docs/integrations/gitlab/discovery',\n    );\n    check(\n      'ldap-org',\n      'LdapOrgReaderProcessor',\n      'https://backstage.io/docs/integrations/ldap/org',\n    );\n    check(\n      'microsoft-graph-org',\n      'MicrosoftGraphOrgReaderProcessor',\n      'https://backstage.io/docs/integrations/azure/org',\n    );\n  }\n\n  private static getDefaultProcessingInterval(\n    config: Config,\n  ): ProcessingIntervalFunction {\n    const processingIntervalKey = 'catalog.processingInterval';\n\n    if (!config.has(processingIntervalKey)) {\n      return createRandomProcessingInterval({\n        minSeconds: 100,\n        maxSeconds: 150,\n      });\n    }\n\n    if (!Boolean(config.get('catalog.processingInterval'))) {\n      return () => {\n        throw new Error(\n          'catalog.processingInterval is set to false, processing is disabled.',\n        );\n      };\n    }\n\n    const duration = readDurationFromConfig(config, {\n      key: processingIntervalKey,\n    });\n\n    const seconds = Math.max(\n      1,\n      Math.round(durationToMilliseconds(duration) / 1000),\n    );\n\n    return createRandomProcessingInterval({\n      minSeconds: seconds,\n      maxSeconds: seconds * 1.5,\n    });\n  }\n}\n"],"names":["lodash","ScmIntegrations","FileReaderProcessor","UrlReaderProcessor","AnnotateLocationEntityProcessor","defaultEntityDataParser","applyDatabaseMigrations","DefaultStitcher","DefaultProcessingDatabase","DefaultProviderDatabase","DefaultCatalogDatabase","DefaultCatalogRulesEnforcer","DefaultEntitiesCatalog","DefaultCatalogProcessingOrchestrator","AuthorizedEntitiesCatalog","createConditionTransformer","catalogEntityPermissionResourceRef","entitiesResponseToObjects","catalogPermissions","catalogPermissionRules","readScmEventHandlingConfig","DefaultLocationStore","ConfigLocationEntityProvider","GenericScmEventRefreshProvider","filterProviders","DefaultCatalogProcessingEngine","createHash","AuthorizedLocationAnalyzer","RepoLocationAnalyzer","AuthorizedLocationService","DefaultLocationService","AuthorizedRefreshService","DefaultRefreshService","createRouter","connectEntityProviders","evictEntitiesFromOrphanedProviders","SchemaValidEntityPolicy","DefaultNamespaceEntityPolicy","NoForeignRootFieldsEntityPolicy","FieldFormatEntityPolicy","makeValidator","EntityPolicies","jsonPlaceholderResolver","yamlPlaceholderResolver","textPlaceholderResolver","PlaceholderProcessor","BuiltinKindsEntityProcessor","ModelProcessor","filterAndSortProcessors","config","createRandomProcessingInterval","readDurationFromConfig","durationToMilliseconds"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyJO,MAAM,cAAA,CAAe;AAAA,EACT,GAAA;AAAA,EACT,cAAA;AAAA,EACA,qBAAA;AAAA,EACA,oBAAA;AAAA,EACA,qBAAA;AAAA,EACA,eAAA;AAAA,EACA,UAAA;AAAA,EACA,iBAAA;AAAA,EACA,iBAAA;AAAA,EACA,MAAA;AAAA,EACA,iBAAA;AAAA,EAIA,kBAAA;AAAA,EACA,gBAAA,GAAiD,MAAA;AAAA,EACjD,mBAAA;AAAA;AAAA;AAAA;AAAA,EAKR,OAAO,OAAO,GAAA,EAAyC;AACrD,IAAA,OAAO,IAAI,eAAe,GAAG,CAAA;AAAA,EAC/B;AAAA,EAEQ,YAAY,GAAA,EAAyB;AAC3C,IAAA,IAAA,CAAK,GAAA,GAAM,GAAA;AACX,IAAA,IAAA,CAAK,iBAAiB,EAAC;AACvB,IAAA,IAAA,CAAK,qBAAA,GAAwB,KAAA;AAC7B,IAAA,IAAA,CAAK,uBAAuB,EAAC;AAC7B,IAAA,IAAA,CAAK,wBAAwB,EAAC;AAC9B,IAAA,IAAA,CAAK,kBAAkB,EAAC;AACxB,IAAA,IAAA,CAAK,aAAa,EAAC;AACnB,IAAA,IAAA,CAAK,oBAAoB,EAAC;AAC1B,IAAA,IAAA,CAAK,iBAAA,GAAoB,KAAA;AACzB,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AACd,IAAA,IAAA,CAAK,mBAAA,GAAsB,CAAC,KAAK,CAAA;AAEjC,IAAA,IAAA,CAAK,qBAAqB,cAAA,CAAe,4BAAA;AAAA,MACvC,GAAA,CAAI;AAAA,KACN;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAaA,mBACK,QAAA,EACa;AAChB,IAAA,IAAA,CAAK,cAAA,CAAe,IAAA,CAAK,GAAG,QAAA,CAAS,MAAM,CAAA;AAC3C,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,oBAAoB,gBAAA,EAAoD;AACtE,IAAA,IAAA,CAAK,gBAAA,GAAmB,gBAAA;AACxB,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAeA,sBAAsB,QAAA,EAA0C;AAC9D,IAAA,IAAA,CAAK,cAAA,GAAiB,CAAC,GAAG,QAAQ,CAAA;AAClC,IAAA,IAAA,CAAK,qBAAA,GAAwB,IAAA;AAC7B,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,sBAAA,CACE,KACA,QAAA,EACgB;AAChB,IAAA,IAAA,CAAK,oBAAA,CAAqB,GAAG,CAAA,GAAI,QAAA;AACjC,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,yBAAyB,UAAA,EAAiD;AACxE,IAAAA,uBAAA,CAAO,KAAA,CAAM,IAAA,CAAK,qBAAA,EAAuB,UAAU,CAAA;AACnD,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,qBACK,SAAA,EACa;AAChB,IAAA,IAAA,CAAK,eAAA,CAAgB,IAAA,CAAK,GAAG,SAAA,CAAU,MAAM,CAAA;AAC7C,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,gBACK,UAAA,EACa;AAChB,IAAA,IAAA,CAAK,UAAA,CAAW,IAAA,CAAK,GAAG,UAAA,CAAW,MAAM,CAAA;AACzC,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,kBAAkB,UAAA,EAAgD;AAChE,IAAA,IAAA,CAAK,UAAA,GAAa,CAAC,GAAG,UAAU,CAAA;AAChC,IAAA,IAAA,CAAK,iBAAA,GAAoB,IAAA;AACzB,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,oBAAA,GAA2C;AACzC,IAAA,MAAM,EAAE,MAAA,EAAQ,MAAA,EAAQ,MAAA,KAAW,IAAA,CAAK,GAAA;AACxC,IAAA,MAAM,YAAA,GAAeC,2BAAA,CAAgB,UAAA,CAAW,MAAM,CAAA;AAEtD,IAAA,OAAO;AAAA,MACL,IAAIC,uCAAA,EAAoB;AAAA,MACxB,IAAIC,qCAAA,CAAmB,EAAE,MAAA,EAAQ,QAAQ,CAAA;AAAA,MACzC,IAAIC,+DAAA,CAAgC,EAAE,YAAA,EAAc;AAAA,KACtD;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,wBACK,SAAA,EACa;AAChB,IAAA,IAAA,CAAK,iBAAA,CAAkB,IAAA,CAAK,GAAG,SAAA,CAAU,MAAM,CAAA;AAC/C,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,oBAAoB,MAAA,EAAgD;AAClE,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AACd,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,wBAAwB,oBAAA,EAAgD;AACtE,IAAA,IAAA,CAAK,mBAAA,GAAsB,oBAAA;AAC3B,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,KAAA,GAGH;AACD,IAAA,MAAM;AAAA,MACJ,MAAA;AAAA,MACA,QAAA;AAAA,MACA,MAAA;AAAA,MACA,WAAA;AAAA,MACA,SAAA;AAAA,MACA,mBAAA;AAAA,MACA,OAAA;AAAA,MACA,IAAA;AAAA,MACA,QAAA;AAAA,MACA,MAAA;AAAA,MACA,gBAAA;AAAA,MACA;AAAA,QACE,IAAA,CAAK,GAAA;AAET,IAAA,MAAM,4BAAA,GAA+B,OAAA;AAAA,MACnC,MAAA,CAAO,mBAAmB,sCAAsC;AAAA,KAClE;AAEA,IAAA,MAAM,MAAA,GAAS,KAAK,iBAAA,EAAkB;AACtC,IAAA,MAAM,UAAA,GAAa,KAAK,eAAA,EAAgB;AACxC,IAAA,MAAM,MAAA,GAAS,KAAK,MAAA,IAAUC,6BAAA;AAE9B,IAAA,MAAM,QAAA,GAAW,MAAM,QAAA,CAAS,SAAA,EAAU;AAC1C,IAAA,IAAI,CAAC,QAAA,CAAS,UAAA,EAAY,IAAA,EAAM;AAC9B,MAAA,MAAA,CAAO,KAAK,+BAA+B,CAAA;AAC3C,MAAA,MAAMC,mCAAwB,QAAQ,CAAA;AAAA,IACxC;AAEA,IAAA,MAAM,QAAA,GAAWC,+BAAA,CAAgB,UAAA,CAAW,MAAA,EAAQ;AAAA,MAClD,IAAA,EAAM,QAAA;AAAA,MACN,MAAA;AAAA,MACA;AAAA,KACD,CAAA;AAED,IAAA,MAAM,kBAAA,GAAqB,IAAIC,mDAAA,CAA0B;AAAA,MACvD,QAAA,EAAU,QAAA;AAAA,MACV,MAAA;AAAA,MACA,MAAA;AAAA,MACA,iBAAiB,IAAA,CAAK,kBAAA;AAAA,MACtB;AAAA,KACD,CAAA;AACD,IAAA,MAAM,gBAAA,GAAmB,IAAIC,+CAAA,CAAwB;AAAA,MACnD,QAAA,EAAU,QAAA;AAAA,MACV;AAAA,KACD,CAAA;AACD,IAAA,MAAM,eAAA,GAAkB,IAAIC,6CAAA,CAAuB;AAAA,MACjD,QAAA,EAAU,QAAA;AAAA,MACV;AAAA,KACD,CAAA;AACD,IAAA,MAAM,YAAA,GAAeT,2BAAA,CAAgB,UAAA,CAAW,MAAM,CAAA;AACtD,IAAA,MAAM,aAAA,GAAgBU,wCAAA,CAA4B,UAAA,CAAW,MAAM,CAAA;AAEnE,IAAA,MAAM,2BAAA,GAA8B,IAAIC,6CAAA,CAAuB;AAAA,MAC7D,QAAA,EAAU,QAAA;AAAA,MACV,MAAA;AAAA,MACA;AAAA,KACD,CAAA;AAED,IAAA,MAAM,YAAA,GAAe,IAAIC,yEAAA,CAAqC;AAAA,MAC5D,UAAA;AAAA,MACA,YAAA;AAAA,MACA,aAAA;AAAA,MACA,MAAA;AAAA,MACA,MAAA;AAAA,MACA,MAAA;AAAA,MACA,sBAAsB,IAAA,CAAK;AAAA,KAC5B,CAAA;AAED,IAAA,MAAM,kBAAkB,IAAIC,mDAAA;AAAA,MAC1B,2BAAA;AAAA,MACA,WAAA;AAAA,MACAC,+CAAA;AAAA,QACE,mBAAA,CAAoB,oBAAA;AAAA,UAClBC;AAAA;AACF;AACF,KACF;AAEA,IAAA,MAAM,YAAA,GAAe,OAAO,YAAA,KAA2B;AACrD,MAAA,MAAM,EAAE,KAAA,EAAM,GAAI,MAAM,4BAA4B,aAAA,CAAc;AAAA,QAChE,WAAA,EAAa,MAAM,IAAA,CAAK,wBAAA,EAAyB;AAAA,QACjD,UAAA,EAAY;AAAA,OACb,CAAA;AAED,MAAA,OAAOC,oCAA0B,KAAK,CAAA,CAAE,GAAA,CAAI,CAAA,CAAA,KAAK,KAAK,MAAS,CAAA;AAAA,IACjE,CAAA;AAEA,IAAA,mBAAA,CAAoB,eAAA,CAAgB;AAAA,MAClC,WAAA,EAAaD,wCAAA;AAAA,MACb,YAAA;AAAA,MACA,WAAA,EAAa,CAAC,GAAGE,0BAAkB,CAAA;AAAA,MACnC,KAAA,EAAO,MAAA,CAAO,MAAA,CAAOC,qBAAsB;AAAA,KAC5C,CAAA;AAED,IAAA,MAAM,sBAAA,GAAyBC,sDAA2B,MAAM,CAAA;AAChE,IAAA,MAAM,gBAAgB,IAAIC,yCAAA;AAAA,MACxB,QAAA;AAAA,MACA,gBAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,MAAM,sBAAA,GAAyB,IAAIC,yDAAA,CAA6B,MAAM,CAAA;AACtE,IAAA,MAAM,YAAY,IAAIC,6DAAA;AAAA,MACpB,QAAA;AAAA,MACA,gBAAA;AAAA,MACA;AAAA,KACF;AAEA,IAAA,MAAM,wBAAwBvB,uBAAA,CAAO,MAAA;AAAA,MACnC;AAAA,QACE,GAAG,IAAA,CAAK,eAAA;AAAA,QACR,EAAE,UAAU,aAAA,EAAc;AAAA,QAC1B,EAAE,UAAU,sBAAA,EAAuB;AAAA,QACnC,EAAE,UAAU,SAAA;AAAU,OACxB;AAAA,MACA,CAAA,KAAA,KAAS,KAAA,CAAM,QAAA,CAAS,eAAA;AAAgB,KAC1C;AACA,IAAA,MAAM,sBAAA,GAAyBwB,oBAAA;AAAA,MAC7B,qBAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,MAAM,mBAAmB,sBAAA,CAAuB,GAAA;AAAA,MAC9C,WAAS,KAAA,CAAM;AAAA,KACjB;AAEA,IAAA,MAAM,gBAAA,GAAmB,IAAIC,6DAAA,CAA+B;AAAA,MAC1D,MAAA;AAAA,MACA,SAAA;AAAA,MACA,MAAA;AAAA,MACA,IAAA,EAAM,QAAA;AAAA,MACN,kBAAA;AAAA,MACA,YAAA;AAAA,MACA,UAAA,EAAY,MAAMC,sBAAA,CAAW,MAAM,CAAA;AAAA,MACnC,iBAAA,EAAmB,GAAA;AAAA,MACnB,mBAAmB,CAAA,KAAA,KAAS;AAC1B,QAAA,IAAA,CAAK,oBAAoB,KAAK,CAAA;AAAA,MAChC,CAAA;AAAA,MACA,MAAA;AAAA,MACA;AAAA,KACD,CAAA;AAED,IAAA,MAAM,gBAAA,GACJ,IAAA,CAAK,gBAAA,IACL,IAAIC,qDAAA;AAAA,MACF,IAAIC,qCAAA,CAAqB,MAAA,EAAQ,YAAA,EAAc,KAAK,iBAAiB,CAAA;AAAA,MACrE;AAAA,KACF;AACF,IAAA,MAAM,kBAAkB,IAAIC,mDAAA;AAAA,MAC1B,IAAIC,6CAAA,CAAuB,aAAA,EAAe,YAAA,EAAc;AAAA,QACtD,sBAAsB,IAAA,CAAK,mBAAA;AAAA,QAC3B,iCACG,MAAA,CAAO,iBAAA;AAAA,UACN;AAAA,SACF,IAA8B;AAAA,OACjC,CAAA;AAAA,MACD;AAAA,KACF;AACA,IAAA,MAAM,iBAAiB,IAAIC,iDAAA;AAAA,MACzB,IAAIC,2CAAA,CAAsB,EAAE,QAAA,EAAU,iBAAiB,CAAA;AAAA,MACvD;AAAA,KACF;AAEA,IAAA,MAAM,MAAA,GAAS,MAAMC,yBAAA,CAAa;AAAA,MAChC,eAAA;AAAA,MACA,gBAAA;AAAA,MACA,eAAA;AAAA,MACA,YAAA;AAAA,MACA,cAAA;AAAA,MACA,MAAA;AAAA,MACA,MAAA;AAAA,MACA,IAAA;AAAA,MACA,QAAA;AAAA,MACA,kBAAA,EAAoB,WAAA;AAAA,MACpB,OAAA;AAAA,MACA;AAAA,KACD,CAAA;AAED,IAAA,MAAMC,6CAAA,CAAuB,kBAAkB,sBAAsB,CAAA;AAErE,IAAA,OAAO;AAAA,MACL,gBAAA,EAAkB;AAAA,QAChB,MAAM,KAAA,GAAQ;AACZ,UAAA,IACE,MAAA,CAAO,iBAAA,CAAkB,gCAAgC,CAAA,KACzD,MAAA,EACA;AACA,YAAA,MAAMC,qEAAA,CAAmC;AAAA,cACvC,EAAA,EAAI,gBAAA;AAAA,cACJ,SAAA,EAAW,gBAAA;AAAA,cACX;AAAA,aACD,CAAA;AAAA,UACH;AACA,UAAA,MAAM,iBAAiB,KAAA,EAAM;AAC7B,UAAA,MAAM,SAAS,KAAA,EAAM;AAAA,QACvB,CAAA;AAAA,QACA,MAAM,IAAA,GAAO;AACX,UAAA,MAAM,iBAAiB,IAAA,EAAK;AAC5B,UAAA,MAAM,SAAS,IAAA,EAAK;AAAA,QACtB;AAAA,OACF;AAAA,MACA;AAAA,KACF;AAAA,EACF;AAAA,EAEA,UAAU,OAAA,EAKP;AACD,IAAA,IAAA,CAAK,oBAAoB,OAAA,CAAQ,iBAAA;AAAA,EACnC;AAAA,EAEQ,iBAAA,GAAkC;AACxC,IAAA,MAAM,cAAA,GAAiC,IAAA,CAAK,qBAAA,GACxC,CAAC,IAAIC,sCAAwB,EAAG,GAAG,IAAA,CAAK,cAAc,CAAA,GACtD;AAAA,MACE,IAAIA,oCAAA,EAAwB;AAAA,MAC5B,IAAIC,yCAAA,EAA6B;AAAA,MACjC,IAAIC,4CAAA,EAAgC;AAAA,MACpC,IAAIC,oCAAA;AAAA,QACFC,0BAAA,CAAc,KAAK,qBAAqB;AAAA,OAC1C;AAAA,MACA,GAAG,IAAA,CAAK;AAAA,KACV;AAEJ,IAAA,OAAOC,2BAAA,CAAe,MAAM,cAAc,CAAA;AAAA,EAC5C;AAAA,EAEQ,eAAA,GAAsC;AAC5C,IAAA,MAAM,EAAE,MAAA,EAAQ,MAAA,EAAO,GAAI,IAAA,CAAK,GAAA;AAChC,IAAA,MAAM,YAAA,GAAexC,2BAAA,CAAgB,UAAA,CAAW,MAAM,CAAA;AAEtD,IAAA,IAAA,CAAK,+BAAA,EAAgC;AAErC,IAAA,MAAM,oBAAA,GAA4D;AAAA,MAChE,IAAA,EAAMyC,4CAAA;AAAA,MACN,IAAA,EAAMC,4CAAA;AAAA,MACN,IAAA,EAAMC,4CAAA;AAAA,MACN,GAAG,IAAA,CAAK;AAAA,KACV;AAGA,IAAA,MAAM,UAAA,GAAiC;AAAA,MACrC,IAAIC,yCAAA,CAAqB;AAAA,QACvB,SAAA,EAAW,oBAAA;AAAA,QACX,MAAA;AAAA,QACA;AAAA,OACD;AAAA,KACH;AAEA,IAAA,MAAM,2BAAA,GAA8B,IAAIC,uDAAA,EAA4B;AAIpE,IAAA,IACE,CAAC,IAAA,CAAK,GAAA,CAAI,WAAA,IACV,CAAC,KAAK,UAAA,CAAW,IAAA;AAAA,MACf,CAAA,SAAA,KACE,SAAA,CAAU,gBAAA,EAAiB,KAC3B,4BAA4B,gBAAA;AAAiB,KACjD,EACA;AACA,MAAA,UAAA,CAAW,KAAK,2BAA2B,CAAA;AAAA,IAC7C;AACA,IAAA,IAAI,IAAA,CAAK,IAAI,WAAA,EAAa;AACxB,MAAA,UAAA,CAAW,KAAK,IAAIC,6BAAA,CAAe,IAAA,CAAK,GAAA,CAAI,WAAW,CAAC,CAAA;AAAA,IAC1D;AAEA,IAAA,MAAM,2BAA2B,MAAA,CAAO,kBAAA;AAAA,MACtC;AAAA,KACF;AAKA,IAAA,IAAI,CAAC,IAAA,CAAK,iBAAA,IAAqB,CAAC,wBAAA,EAA0B;AACxD,MAAA,UAAA,CAAW,IAAA,CAAK,GAAG,IAAA,CAAK,oBAAA,EAAsB,CAAA;AAAA,IAChD;AAGA,IAAA,UAAA,CAAW,IAAA,CAAK,GAAG,IAAA,CAAK,UAAU,CAAA;AAElC,IAAA,IAAA,CAAK,+BAA+B,UAAU,CAAA;AAE9C,IAAA,OAAOC,4BAAA,CAAwB,YAAY,MAAM,CAAA;AAAA,EACnD;AAAA;AAAA;AAAA,EAIQ,+BAAA,GAAkC;AACxC,IAAA,MAAM,EAAA,GAAK,IAAA,CAAK,GAAA,CAAI,MAAA,CAAO,kBAAkB,oBAAoB,CAAA;AACjE,IAAA,IAAI,EAAA,EAAI,GAAA,CAAI,QAAQ,CAAA,EAAG;AACrB,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,CAAA,uGAAA;AAAA,OACF;AAAA,IACF;AACA,IAAA,IAAI,EAAA,EAAI,GAAA,CAAI,WAAW,CAAA,EAAG;AACxB,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,CAAA,0GAAA;AAAA,OACF;AAAA,IACF;AACA,IAAA,IAAI,EAAA,EAAI,GAAA,CAAI,cAAc,CAAA,EAAG;AAC3B,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,CAAA,gHAAA;AAAA,OACF;AAAA,IACF;AACA,IAAA,IAAI,EAAA,EAAI,GAAA,CAAI,UAAU,CAAA,EAAG;AACvB,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,CAAA,wGAAA;AAAA,OACF;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGQ,+BAA+B,UAAA,EAAgC;AACrE,IAAA,MAAM,gBAAA,GAAmB,iDAAA;AACzB,IAAA,IAAI,OAAA,CAAQ,GAAA,CAAI,gBAAgB,CAAA,EAAG;AACjC,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,gBAAgB,IAAI,GAAA;AAAA,MACxB,IAAA,CAAK,GAAA,CAAI,MAAA,CACN,sBAAA,CAAuB,mBAAmB,CAAA,EACzC,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,SAAA,CAAU,MAAM,CAAC,KAAK;AAAC,KACxC;AACA,IAAA,MAAM,cAAA,GAAiB,IAAI,GAAA,CAAI,UAAA,CAAW,IAAI,CAAA,CAAA,KAAK,CAAA,CAAE,gBAAA,EAAkB,CAAC,CAAA;AAExE,IAAA,SAAS,KAAA,CACP,YAAA,EACA,aAAA,EACA,eAAA,EACA;AACA,MAAA,IACE,aAAA,CAAc,IAAI,YAAY,CAAA,IAC9B,CAAC,cAAA,CAAe,GAAA,CAAI,aAAa,CAAA,EACjC;AACA,QAAA,MAAM,IAAI,KAAA;AAAA,UACR;AAAA,YACE,4DAA4D,YAAY,CAAA,CAAA,CAAA;AAAA,YACxE,yDAAyD,aAAa,CAAA,WAAA,CAAA;AAAA,YACtE,CAAA,+EAAA,CAAA;AAAA,YACA,CAAA,iFAAA,CAAA;AAAA,YACA,mBAAmB,eAAe,CAAA,6CAAA,CAAA;AAAA,YAClC,CAAA,qFAAA,CAAA;AAAA,YACA,uCAAuC,gBAAgB,CAAA,WAAA;AAAA,WACzD,CAAE,KAAK,GAAG;AAAA,SACZ;AAAA,MACF;AAAA,IACF;AAEA,IAAA,KAAA;AAAA,MACE,oBAAA;AAAA,MACA,sCAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,KAAA;AAAA,MACE,cAAA;AAAA,MACA,yBAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,KAAA;AAAA,MACE,iBAAA;AAAA,MACA,+BAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,KAAA;AAAA,MACE,qBAAA;AAAA,MACA,6BAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,KAAA;AAAA,MACE,kBAAA;AAAA,MACA,0BAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,KAAA;AAAA,MACE,YAAA;AAAA,MACA,0BAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,KAAA;AAAA,MACE,kBAAA;AAAA,MACA,0BAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,KAAA;AAAA,MACE,UAAA;AAAA,MACA,wBAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,KAAA;AAAA,MACE,qBAAA;AAAA,MACA,kCAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF;AAAA,EAEA,OAAe,6BACbC,QAAA,EAC4B;AAC5B,IAAA,MAAM,qBAAA,GAAwB,4BAAA;AAE9B,IAAA,IAAI,CAACA,QAAA,CAAO,GAAA,CAAI,qBAAqB,CAAA,EAAG;AACtC,MAAA,OAAOC,sCAAA,CAA+B;AAAA,QACpC,UAAA,EAAY,GAAA;AAAA,QACZ,UAAA,EAAY;AAAA,OACb,CAAA;AAAA,IACH;AAEA,IAAA,IAAI,CAAC,OAAA,CAAQD,QAAA,CAAO,GAAA,CAAI,4BAA4B,CAAC,CAAA,EAAG;AACtD,MAAA,OAAO,MAAM;AACX,QAAA,MAAM,IAAI,KAAA;AAAA,UACR;AAAA,SACF;AAAA,MACF,CAAA;AAAA,IACF;AAEA,IAAA,MAAM,QAAA,GAAWE,8BAAuBF,QAAA,EAAQ;AAAA,MAC9C,GAAA,EAAK;AAAA,KACN,CAAA;AAED,IAAA,MAAM,UAAU,IAAA,CAAK,GAAA;AAAA,MACnB,CAAA;AAAA,MACA,IAAA,CAAK,KAAA,CAAMG,4BAAA,CAAuB,QAAQ,IAAI,GAAI;AAAA,KACpD;AAEA,IAAA,OAAOF,sCAAA,CAA+B;AAAA,MACpC,UAAA,EAAY,OAAA;AAAA,MACZ,YAAY,OAAA,GAAU;AAAA,KACvB,CAAA;AAAA,EACH;AACF;;;;"}
+{"version":3,"file":"CatalogBuilder.cjs.js","sources":["../../src/service/CatalogBuilder.ts"],"sourcesContent":["/*\n * Copyright 2020 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  DefaultNamespaceEntityPolicy,\n  Entity,\n  EntityPolicies,\n  EntityPolicy,\n  FieldFormatEntityPolicy,\n  makeValidator,\n  NoForeignRootFieldsEntityPolicy,\n  SchemaValidEntityPolicy,\n  Validators,\n} from '@backstage/catalog-model';\nimport { ScmIntegrations } from '@backstage/integration';\nimport { createHash } from 'node:crypto';\nimport { Router } from 'express';\nimport lodash from 'lodash';\n\nimport {\n  AuditorService,\n  AuthService,\n  DatabaseService,\n  HttpAuthService,\n  LoggerService,\n  PermissionsRegistryService,\n  PermissionsService,\n  RootConfigService,\n  SchedulerService,\n  UrlReaderService,\n} from '@backstage/backend-plugin-api';\nimport { Config, readDurationFromConfig } from '@backstage/config';\nimport { catalogPermissions } from '@backstage/plugin-catalog-common/alpha';\nimport {\n  CatalogProcessor,\n  CatalogProcessorParser,\n  LocationAnalyzer,\n  PlaceholderResolver,\n  ScmLocationAnalyzer,\n} from '@backstage/plugin-catalog-node';\nimport { EventsService } from '@backstage/plugin-events-node';\nimport { createConditionTransformer } from '@backstage/plugin-permission-node';\nimport { durationToMilliseconds } from '@backstage/types';\nimport { DefaultCatalogDatabase } from '../database/DefaultCatalogDatabase';\nimport { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase';\nimport { DefaultProviderDatabase } from '../database/DefaultProviderDatabase';\nimport { applyDatabaseMigrations } from '../database/migrations';\nimport { DefaultCatalogRulesEnforcer } from '../ingestion/CatalogRules';\nimport { RepoLocationAnalyzer } from '../ingestion/LocationAnalyzer';\nimport { permissionRules as catalogPermissionRules } from '../permissions/rules';\nimport { CatalogProcessingEngine } from '../processing/types';\nimport {\n  createRandomProcessingInterval,\n  ProcessingIntervalFunction,\n} from '../processing/refresh';\nimport {\n  connectEntityProviders,\n  EntityProviderEntry,\n} from '../processing/connectEntityProviders';\nimport { evictEntitiesFromOrphanedProviders } from '../processing/evictEntitiesFromOrphanedProviders';\nimport { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine';\nimport { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator';\nimport {\n  AnnotateLocationEntityProcessor,\n  BuiltinKindsEntityProcessor,\n  FileReaderProcessor,\n  PlaceholderProcessor,\n  UrlReaderProcessor,\n} from '../processors';\nimport {\n  jsonPlaceholderResolver,\n  textPlaceholderResolver,\n  yamlPlaceholderResolver,\n} from '../processors/PlaceholderProcessor';\nimport { ConfigLocationEntityProvider } from '../providers/ConfigLocationEntityProvider';\nimport { DefaultLocationStore } from '../providers/DefaultLocationStore';\nimport { DefaultStitcher } from '../stitching/DefaultStitcher';\nimport { defaultEntityDataParser } from '../util/parse';\nimport { AuthorizedEntitiesCatalog } from './AuthorizedEntitiesCatalog';\nimport { AuthorizedLocationAnalyzer } from './AuthorizedLocationAnalyzer';\nimport { AuthorizedLocationService } from './AuthorizedLocationService';\nimport { AuthorizedRefreshService } from './AuthorizedRefreshService';\nimport { createRouter } from './createRouter';\nimport { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog';\nimport { DefaultLocationService } from './DefaultLocationService';\nimport { DefaultRefreshService } from './DefaultRefreshService';\nimport { entitiesResponseToObjects } from './response';\nimport {\n  catalogEntityPermissionResourceRef,\n  CatalogScmEventsService,\n} from '@backstage/plugin-catalog-node/alpha';\nimport { filterAndSortProcessors, filterProviders } from './util';\nimport { GenericScmEventRefreshProvider } from '../providers/GenericScmEventRefreshProvider';\nimport { readScmEventHandlingConfig } from '../util/readScmEventHandlingConfig';\nimport { MetricsService } from '@backstage/backend-plugin-api/alpha';\nimport { ModelProcessor } from '../processors/ModelProcessor';\nimport { ModelHolder } from '../model/ModelHolder';\n\nexport type CatalogEnvironment = {\n  logger: LoggerService;\n  modelHolder?: ModelHolder;\n  database: DatabaseService;\n  config: RootConfigService;\n  reader: UrlReaderService;\n  permissions: PermissionsService;\n  permissionsRegistry: PermissionsRegistryService;\n  scheduler: SchedulerService;\n  auth: AuthService;\n  httpAuth: HttpAuthService;\n  auditor: AuditorService;\n  events: EventsService;\n  catalogScmEvents: CatalogScmEventsService;\n  metrics: MetricsService;\n};\n\n/**\n * A builder that helps wire up all of the component parts of the catalog.\n *\n * The touch points where you can replace or extend behavior are as follows:\n *\n * - Entity policies can be added or replaced. These are automatically run\n *   after the processors' pre-processing steps. All policies are given the\n *   chance to inspect the entity, and all of them have to pass in order for\n *   the entity to be considered valid from an overall point of view.\n * - Location analyzers can be added. These are responsible for analyzing\n *   repositories when onboarding them into the catalog, by finding\n *   catalog-info.yaml files and other artifacts that can help automatically\n *   register or create catalog data on the user's behalf.\n * - Placeholder resolvers can be replaced or added. These run on the raw\n *   structured data between the parsing and pre-processing steps, to replace\n *   dollar-prefixed entries with their actual values (like $file).\n * - Field format validators can be replaced. These check the format of\n *   individual core fields such as metadata.name, to ensure that they adhere\n *   to certain rules.\n * - Processors can be added or replaced. These implement the functionality of\n *   reading, parsing, validating, and processing the entity data before it is\n *   persisted in the catalog.\n *\n * @internal\n */\nexport class CatalogBuilder {\n  private readonly env: CatalogEnvironment;\n  private entityPolicies: EntityPolicy[];\n  private entityPoliciesReplace: boolean;\n  private placeholderResolvers: Record<string, PlaceholderResolver>;\n  private fieldFormatValidators: Partial<Validators>;\n  private entityProviders: EntityProviderEntry[];\n  private processors: CatalogProcessor[];\n  private locationAnalyzers: ScmLocationAnalyzer[];\n  private processorsReplace: boolean;\n  private parser: CatalogProcessorParser | undefined;\n  private onProcessingError?: (event: {\n    unprocessedEntity: Entity;\n    errors: Error[];\n  }) => Promise<void> | void;\n  private processingInterval: ProcessingIntervalFunction;\n  private locationAnalyzer: LocationAnalyzer | undefined = undefined;\n  private allowedLocationType: string[];\n\n  /**\n   * Creates a catalog builder.\n   */\n  static create(env: CatalogEnvironment): CatalogBuilder {\n    return new CatalogBuilder(env);\n  }\n\n  private constructor(env: CatalogEnvironment) {\n    this.env = env;\n    this.entityPolicies = [];\n    this.entityPoliciesReplace = false;\n    this.placeholderResolvers = {};\n    this.fieldFormatValidators = {};\n    this.entityProviders = [];\n    this.processors = [];\n    this.locationAnalyzers = [];\n    this.processorsReplace = false;\n    this.parser = undefined;\n    this.allowedLocationType = ['url'];\n\n    this.processingInterval = CatalogBuilder.getDefaultProcessingInterval(\n      env.config,\n    );\n  }\n\n  /**\n   * Adds policies that are used to validate entities between the pre-\n   * processing and post-processing stages. All such policies must pass for the\n   * entity to be considered valid.\n   *\n   * If what you want to do is to replace the rules for what format is allowed\n   * in various core entity fields (such as metadata.name), you may want to use\n   * {@link CatalogBuilder#setFieldFormatValidators} instead.\n   *\n   * @param policies - One or more policies\n   */\n  addEntityPolicy(\n    ...policies: Array<EntityPolicy | Array<EntityPolicy>>\n  ): CatalogBuilder {\n    this.entityPolicies.push(...policies.flat());\n    return this;\n  }\n\n  /**\n   * Overwrites the default location analyzer.\n   */\n  setLocationAnalyzer(locationAnalyzer: LocationAnalyzer): CatalogBuilder {\n    this.locationAnalyzer = locationAnalyzer;\n    return this;\n  }\n\n  /**\n   * Sets what policies to use for validation of entities between the pre-\n   * processing and post-processing stages. All such policies must pass for the\n   * entity to be considered valid.\n   *\n   * If what you want to do is to replace the rules for what format is allowed\n   * in various core entity fields (such as metadata.name), you may want to use\n   * {@link CatalogBuilder#setFieldFormatValidators} instead.\n   *\n   * This function replaces the default set of policies; use with care.\n   *\n   * @param policies - One or more policies\n   */\n  replaceEntityPolicies(policies: EntityPolicy[]): CatalogBuilder {\n    this.entityPolicies = [...policies];\n    this.entityPoliciesReplace = true;\n    return this;\n  }\n\n  /**\n   * Adds, or overwrites, a handler for placeholders (e.g. $file) in entity\n   * definition files.\n   *\n   * @param key - The key that identifies the placeholder, e.g. \"file\"\n   * @param resolver - The resolver that gets values for this placeholder\n   */\n  setPlaceholderResolver(\n    key: string,\n    resolver: PlaceholderResolver,\n  ): CatalogBuilder {\n    this.placeholderResolvers[key] = resolver;\n    return this;\n  }\n\n  /**\n   * Sets the validator function to use for one or more special fields of an\n   * entity. This is useful if the default rules for formatting of fields are\n   * not sufficient.\n   *\n   * This function has no effect if used together with\n   * {@link CatalogBuilder#replaceEntityPolicies}.\n   *\n   * @param validators - The (subset of) validators to set\n   */\n  setFieldFormatValidators(validators: Partial<Validators>): CatalogBuilder {\n    lodash.merge(this.fieldFormatValidators, validators);\n    return this;\n  }\n\n  /**\n   * Adds or replaces entity providers. These are responsible for bootstrapping\n   * the list of entities out of original data sources. For example, there is\n   * one entity source for the config locations, and one for the database\n   * stored locations. If you ingest entities out of a third party system, you\n   * may want to implement that in terms of an entity provider as well.\n   *\n   * @param providers - One or more entity providers\n   */\n  addEntityProvider(\n    ...providers: Array<EntityProviderEntry | Array<EntityProviderEntry>>\n  ): CatalogBuilder {\n    this.entityProviders.push(...providers.flat());\n    return this;\n  }\n\n  /**\n   * Adds entity processors. These are responsible for reading, parsing, and\n   * processing entities before they are persisted in the catalog.\n   *\n   * @param processors - One or more processors\n   */\n  addProcessor(\n    ...processors: Array<CatalogProcessor | Array<CatalogProcessor>>\n  ): CatalogBuilder {\n    this.processors.push(...processors.flat());\n    return this;\n  }\n\n  /**\n   * Sets what entity processors to use. These are responsible for reading,\n   * parsing, and processing entities before they are persisted in the catalog.\n   *\n   * This function replaces the default set of processors, consider using with\n   * {@link CatalogBuilder#getDefaultProcessors}; use with care.\n   *\n   * @param processors - One or more processors\n   */\n  replaceProcessors(processors: CatalogProcessor[]): CatalogBuilder {\n    this.processors = [...processors];\n    this.processorsReplace = true;\n    return this;\n  }\n\n  /**\n   * Returns the default list of entity processors. These are responsible for reading,\n   * parsing, and processing entities before they are persisted in the catalog. Changing\n   * the order of processing can give more control to custom processors.\n   *\n   * Consider using with {@link CatalogBuilder#replaceProcessors}\n   *\n   */\n  getDefaultProcessors(): CatalogProcessor[] {\n    const { config, logger, reader } = this.env;\n    const integrations = ScmIntegrations.fromConfig(config);\n\n    return [\n      new FileReaderProcessor(),\n      new UrlReaderProcessor({ reader, logger }),\n      new AnnotateLocationEntityProcessor({ integrations }),\n    ];\n  }\n\n  /**\n   * Adds Location Analyzers. These are responsible for analyzing\n   * repositories when onboarding them into the catalog, by finding\n   * catalog-info.yaml files and other artifacts that can help automatically\n   * register or create catalog data on the user's behalf.\n   *\n   * @param locationAnalyzers - One or more location analyzers\n   */\n  addLocationAnalyzers(\n    ...analyzers: Array<ScmLocationAnalyzer | Array<ScmLocationAnalyzer>>\n  ): CatalogBuilder {\n    this.locationAnalyzers.push(...analyzers.flat());\n    return this;\n  }\n\n  /**\n   * Sets up the catalog to use a custom parser for entity data.\n   *\n   * This is the function that gets called immediately after some raw entity\n   * specification data has been read from a remote source, and needs to be\n   * parsed and emitted as structured data.\n   *\n   * @param parser - The custom parser\n   */\n  setEntityDataParser(parser: CatalogProcessorParser): CatalogBuilder {\n    this.parser = parser;\n    return this;\n  }\n\n  /**\n   * Sets up the allowed location types from being registered via the location service.\n   *\n   * @param allowedLocationTypes - the allowed location types\n   */\n  setAllowedLocationTypes(allowedLocationTypes: string[]): CatalogBuilder {\n    this.allowedLocationType = allowedLocationTypes;\n    return this;\n  }\n\n  /**\n   * Wires up and returns all of the component parts of the catalog\n   */\n  async build(): Promise<{\n    processingEngine: CatalogProcessingEngine;\n    router: Router;\n  }> {\n    const {\n      config,\n      database,\n      logger,\n      permissions,\n      scheduler,\n      permissionsRegistry,\n      auditor,\n      auth,\n      httpAuth,\n      events,\n      catalogScmEvents,\n      metrics,\n    } = this.env;\n\n    const policy = this.buildEntityPolicy();\n    const processors = this.buildProcessors();\n    const parser = this.parser || defaultEntityDataParser;\n\n    const dbClient = await database.getClient();\n    if (!database.migrations?.skip) {\n      logger.info('Performing database migration');\n      await applyDatabaseMigrations(dbClient);\n    }\n\n    const stitcher = DefaultStitcher.fromConfig(config, {\n      knex: dbClient,\n      logger,\n      metrics,\n    });\n\n    const processingDatabase = new DefaultProcessingDatabase({\n      database: dbClient,\n      logger,\n      events,\n      refreshInterval: this.processingInterval,\n      metrics,\n    });\n    const providerDatabase = new DefaultProviderDatabase({\n      database: dbClient,\n      logger,\n    });\n    const catalogDatabase = new DefaultCatalogDatabase({\n      database: dbClient,\n      logger,\n    });\n    const integrations = ScmIntegrations.fromConfig(config);\n    const rulesEnforcer = DefaultCatalogRulesEnforcer.fromConfig(config);\n\n    const unauthorizedEntitiesCatalog = new DefaultEntitiesCatalog({\n      database: dbClient,\n      logger,\n    });\n\n    const orchestrator = new DefaultCatalogProcessingOrchestrator({\n      processors,\n      integrations,\n      rulesEnforcer,\n      logger,\n      parser,\n      policy,\n      allowedLocationTypes: this.allowedLocationType,\n    });\n\n    const entitiesCatalog = new AuthorizedEntitiesCatalog(\n      unauthorizedEntitiesCatalog,\n      permissions,\n      createConditionTransformer(\n        permissionsRegistry.getPermissionRuleset(\n          catalogEntityPermissionResourceRef,\n        ),\n      ),\n    );\n\n    const getResources = async (resourceRefs: string[]) => {\n      const { items } = await unauthorizedEntitiesCatalog.entitiesBatch({\n        credentials: await auth.getOwnServiceCredentials(),\n        entityRefs: resourceRefs,\n      });\n\n      return entitiesResponseToObjects(items).map(e => e || undefined);\n    };\n\n    permissionsRegistry.addResourceType({\n      resourceRef: catalogEntityPermissionResourceRef,\n      getResources,\n      permissions: [...catalogPermissions],\n      rules: Object.values(catalogPermissionRules),\n    });\n\n    const scmEventHandlingConfig = readScmEventHandlingConfig(config);\n    const locationStore = new DefaultLocationStore(\n      dbClient,\n      catalogScmEvents,\n      scmEventHandlingConfig,\n    );\n    const configLocationProvider = new ConfigLocationEntityProvider(config);\n    const scmEvents = new GenericScmEventRefreshProvider(\n      dbClient,\n      catalogScmEvents,\n      scmEventHandlingConfig,\n    );\n\n    const entityProviderEntries = lodash.uniqBy(\n      [\n        ...this.entityProviders,\n        { provider: locationStore },\n        { provider: configLocationProvider },\n        { provider: scmEvents },\n      ],\n      entry => entry.provider.getProviderName(),\n    );\n    const enabledProviderEntries = filterProviders(\n      entityProviderEntries,\n      config,\n    );\n    const enabledProviders = enabledProviderEntries.map(\n      entry => entry.provider,\n    );\n\n    const processingEngine = new DefaultCatalogProcessingEngine({\n      config,\n      scheduler,\n      logger,\n      knex: dbClient,\n      processingDatabase,\n      orchestrator,\n      createHash: () => createHash('sha1'),\n      pollingIntervalMs: 1000,\n      onProcessingError: event => {\n        this.onProcessingError?.(event);\n      },\n      events,\n      metrics,\n    });\n\n    const locationAnalyzer =\n      this.locationAnalyzer ??\n      new AuthorizedLocationAnalyzer(\n        new RepoLocationAnalyzer(logger, integrations, this.locationAnalyzers),\n        permissions,\n      );\n    const locationService = new AuthorizedLocationService(\n      new DefaultLocationService(locationStore, orchestrator, {\n        allowedLocationTypes: this.allowedLocationType,\n        defaultLocationConflictStrategy:\n          (config.getOptionalString(\n            'catalog.defaultLocationConflictStrategy',\n          ) as 'refresh' | 'reject') || 'reject',\n      }),\n      permissions,\n    );\n    const refreshService = new AuthorizedRefreshService(\n      new DefaultRefreshService({ database: catalogDatabase }),\n      permissions,\n    );\n\n    const router = await createRouter({\n      entitiesCatalog,\n      locationAnalyzer,\n      locationService,\n      orchestrator,\n      refreshService,\n      logger,\n      config,\n      auth,\n      httpAuth,\n      permissionsService: permissions,\n      auditor,\n    });\n\n    await connectEntityProviders(providerDatabase, enabledProviderEntries);\n\n    return {\n      processingEngine: {\n        async start() {\n          if (\n            config.getOptionalString('catalog.orphanProviderStrategy') !==\n            'keep'\n          ) {\n            await evictEntitiesFromOrphanedProviders({\n              db: providerDatabase,\n              providers: enabledProviders,\n              logger,\n            });\n          }\n          await processingEngine.start();\n          await stitcher.start();\n        },\n        async stop() {\n          await processingEngine.stop();\n          await stitcher.stop();\n        },\n      },\n      router,\n    };\n  }\n\n  subscribe(options: {\n    onProcessingError: (event: {\n      unprocessedEntity: Entity;\n      errors: Error[];\n    }) => Promise<void> | void;\n  }) {\n    this.onProcessingError = options.onProcessingError;\n  }\n\n  private buildEntityPolicy(): EntityPolicy {\n    const entityPolicies: EntityPolicy[] = this.entityPoliciesReplace\n      ? [new SchemaValidEntityPolicy(), ...this.entityPolicies]\n      : [\n          new SchemaValidEntityPolicy(),\n          new DefaultNamespaceEntityPolicy(),\n          new NoForeignRootFieldsEntityPolicy(),\n          new FieldFormatEntityPolicy(\n            makeValidator(this.fieldFormatValidators),\n          ),\n          ...this.entityPolicies,\n        ];\n\n    return EntityPolicies.allOf(entityPolicies);\n  }\n\n  private buildProcessors(): CatalogProcessor[] {\n    const { config, reader } = this.env;\n    const integrations = ScmIntegrations.fromConfig(config);\n\n    this.checkDeprecatedReaderProcessors();\n\n    const placeholderResolvers: Record<string, PlaceholderResolver> = {\n      json: jsonPlaceholderResolver,\n      yaml: yamlPlaceholderResolver,\n      text: textPlaceholderResolver,\n      ...this.placeholderResolvers,\n    };\n\n    // The placeholder is always there no matter what\n    const processors: CatalogProcessor[] = [\n      new PlaceholderProcessor({\n        resolvers: placeholderResolvers,\n        reader,\n        integrations,\n      }),\n    ];\n\n    const builtinKindsEntityProcessor = new BuiltinKindsEntityProcessor();\n    // If the user adds a processor named 'BuiltinKindsEntityProcessor', skip\n    // inclusion of the catalog-backend version. Same if there's a model\n    // registered - then we are using the new model flow.\n    if (\n      !this.env.modelHolder &&\n      !this.processors.some(\n        processor =>\n          processor.getProcessorName() ===\n          builtinKindsEntityProcessor.getProcessorName(),\n      )\n    ) {\n      processors.push(builtinKindsEntityProcessor);\n    }\n    if (this.env.modelHolder) {\n      processors.push(new ModelProcessor(this.env.modelHolder));\n    }\n\n    const disableDefaultProcessors = config.getOptionalBoolean(\n      'catalog.disableDefaultProcessors',\n    );\n\n    // Add default processors if:\n    // - processors have NOT been explicitly replaced\n    // - and default processors are NOT disabled via config\n    if (!this.processorsReplace && !disableDefaultProcessors) {\n      processors.push(...this.getDefaultProcessors());\n    }\n\n    // Add the ones (if any) that the user added\n    processors.push(...this.processors);\n\n    this.checkMissingExternalProcessors(processors);\n\n    return filterAndSortProcessors(processors, config);\n  }\n\n  // TODO(Rugvip): These old processors are removed, for a while we'll be throwing\n  //               errors here to make sure people know where to move the config\n  private checkDeprecatedReaderProcessors() {\n    const pc = this.env.config.getOptionalConfig('catalog.processors');\n    if (pc?.has('github')) {\n      throw new Error(\n        `Using deprecated configuration for catalog.processors.github, move to using integrations.github instead`,\n      );\n    }\n    if (pc?.has('gitlabApi')) {\n      throw new Error(\n        `Using deprecated configuration for catalog.processors.gitlabApi, move to using integrations.gitlab instead`,\n      );\n    }\n    if (pc?.has('bitbucketApi')) {\n      throw new Error(\n        `Using deprecated configuration for catalog.processors.bitbucketApi, move to using integrations.bitbucket instead`,\n      );\n    }\n    if (pc?.has('azureApi')) {\n      throw new Error(\n        `Using deprecated configuration for catalog.processors.azureApi, move to using integrations.azure instead`,\n      );\n    }\n  }\n\n  // TODO(freben): This can be removed no sooner than June 2022, after adopters have had some time to adapt to the new package structure\n  private checkMissingExternalProcessors(processors: CatalogProcessor[]) {\n    const skipCheckVarName = 'BACKSTAGE_CATALOG_SKIP_MISSING_PROCESSORS_CHECK';\n    if (process.env[skipCheckVarName]) {\n      return;\n    }\n\n    const locationTypes = new Set(\n      this.env.config\n        .getOptionalConfigArray('catalog.locations')\n        ?.map(l => l.getString('type')) ?? [],\n    );\n    const processorNames = new Set(processors.map(p => p.getProcessorName()));\n\n    function check(\n      locationType: string,\n      processorName: string,\n      installationUrl: string,\n    ) {\n      if (\n        locationTypes.has(locationType) &&\n        !processorNames.has(processorName)\n      ) {\n        throw new Error(\n          [\n            `Your config contains a \"catalog.locations\" entry of type ${locationType},`,\n            `but does not have the corresponding catalog processor ${processorName} installed.`,\n            `This processor used to be built into the catalog itself, but is now moved to an`,\n            `external module that has to be installed manually. Please follow the installation`,\n            `instructions at ${installationUrl} if you are using this ability, or remove the`,\n            `location from your app config if you do not. You can also silence this check entirely`,\n            `by setting the environment variable ${skipCheckVarName} to 'true'.`,\n          ].join(' '),\n        );\n      }\n    }\n\n    check(\n      'aws-cloud-accounts',\n      'AwsOrganizationCloudAccountProcessor',\n      'https://backstage.io/docs/integrations',\n    );\n    check(\n      's3-discovery',\n      'AwsS3DiscoveryProcessor',\n      'https://backstage.io/docs/integrations/aws-s3/discovery',\n    );\n    check(\n      'azure-discovery',\n      'AzureDevOpsDiscoveryProcessor',\n      'https://backstage.io/docs/integrations/azure/discovery',\n    );\n    check(\n      'bitbucket-discovery',\n      'BitbucketDiscoveryProcessor',\n      'https://backstage.io/docs/integrations/bitbucket/discovery',\n    );\n    check(\n      'github-discovery',\n      'GithubDiscoveryProcessor',\n      'https://backstage.io/docs/integrations/github/discovery',\n    );\n    check(\n      'github-org',\n      'GithubOrgReaderProcessor',\n      'https://backstage.io/docs/integrations/github/org',\n    );\n    check(\n      'gitlab-discovery',\n      'GitLabDiscoveryProcessor',\n      'https://backstage.io/docs/integrations/gitlab/discovery',\n    );\n    check(\n      'ldap-org',\n      'LdapOrgReaderProcessor',\n      'https://backstage.io/docs/integrations/ldap/org',\n    );\n    check(\n      'microsoft-graph-org',\n      'MicrosoftGraphOrgReaderProcessor',\n      'https://backstage.io/docs/integrations/azure/org',\n    );\n  }\n\n  private static getDefaultProcessingInterval(\n    config: Config,\n  ): ProcessingIntervalFunction {\n    const processingIntervalKey = 'catalog.processingInterval';\n\n    if (!config.has(processingIntervalKey)) {\n      return createRandomProcessingInterval({\n        minSeconds: 100,\n        maxSeconds: 150,\n      });\n    }\n\n    if (!Boolean(config.get('catalog.processingInterval'))) {\n      return () => {\n        throw new Error(\n          'catalog.processingInterval is set to false, processing is disabled.',\n        );\n      };\n    }\n\n    const duration = readDurationFromConfig(config, {\n      key: processingIntervalKey,\n    });\n\n    const seconds = Math.max(\n      1,\n      Math.round(durationToMilliseconds(duration) / 1000),\n    );\n\n    return createRandomProcessingInterval({\n      minSeconds: seconds,\n      maxSeconds: seconds * 1.5,\n    });\n  }\n}\n"],"names":["lodash","ScmIntegrations","FileReaderProcessor","UrlReaderProcessor","AnnotateLocationEntityProcessor","defaultEntityDataParser","applyDatabaseMigrations","DefaultStitcher","DefaultProcessingDatabase","DefaultProviderDatabase","DefaultCatalogDatabase","DefaultCatalogRulesEnforcer","DefaultEntitiesCatalog","DefaultCatalogProcessingOrchestrator","AuthorizedEntitiesCatalog","createConditionTransformer","catalogEntityPermissionResourceRef","entitiesResponseToObjects","catalogPermissions","catalogPermissionRules","readScmEventHandlingConfig","DefaultLocationStore","ConfigLocationEntityProvider","GenericScmEventRefreshProvider","filterProviders","DefaultCatalogProcessingEngine","createHash","AuthorizedLocationAnalyzer","RepoLocationAnalyzer","AuthorizedLocationService","DefaultLocationService","AuthorizedRefreshService","DefaultRefreshService","createRouter","connectEntityProviders","evictEntitiesFromOrphanedProviders","SchemaValidEntityPolicy","DefaultNamespaceEntityPolicy","NoForeignRootFieldsEntityPolicy","FieldFormatEntityPolicy","makeValidator","EntityPolicies","jsonPlaceholderResolver","yamlPlaceholderResolver","textPlaceholderResolver","PlaceholderProcessor","BuiltinKindsEntityProcessor","ModelProcessor","filterAndSortProcessors","config","createRandomProcessingInterval","readDurationFromConfig","durationToMilliseconds"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyJO,MAAM,cAAA,CAAe;AAAA,EACT,GAAA;AAAA,EACT,cAAA;AAAA,EACA,qBAAA;AAAA,EACA,oBAAA;AAAA,EACA,qBAAA;AAAA,EACA,eAAA;AAAA,EACA,UAAA;AAAA,EACA,iBAAA;AAAA,EACA,iBAAA;AAAA,EACA,MAAA;AAAA,EACA,iBAAA;AAAA,EAIA,kBAAA;AAAA,EACA,gBAAA,GAAiD,MAAA;AAAA,EACjD,mBAAA;AAAA;AAAA;AAAA;AAAA,EAKR,OAAO,OAAO,GAAA,EAAyC;AACrD,IAAA,OAAO,IAAI,eAAe,GAAG,CAAA;AAAA,EAC/B;AAAA,EAEQ,YAAY,GAAA,EAAyB;AAC3C,IAAA,IAAA,CAAK,GAAA,GAAM,GAAA;AACX,IAAA,IAAA,CAAK,iBAAiB,EAAC;AACvB,IAAA,IAAA,CAAK,qBAAA,GAAwB,KAAA;AAC7B,IAAA,IAAA,CAAK,uBAAuB,EAAC;AAC7B,IAAA,IAAA,CAAK,wBAAwB,EAAC;AAC9B,IAAA,IAAA,CAAK,kBAAkB,EAAC;AACxB,IAAA,IAAA,CAAK,aAAa,EAAC;AACnB,IAAA,IAAA,CAAK,oBAAoB,EAAC;AAC1B,IAAA,IAAA,CAAK,iBAAA,GAAoB,KAAA;AACzB,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AACd,IAAA,IAAA,CAAK,mBAAA,GAAsB,CAAC,KAAK,CAAA;AAEjC,IAAA,IAAA,CAAK,qBAAqB,cAAA,CAAe,4BAAA;AAAA,MACvC,GAAA,CAAI;AAAA,KACN;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAaA,mBACK,QAAA,EACa;AAChB,IAAA,IAAA,CAAK,cAAA,CAAe,IAAA,CAAK,GAAG,QAAA,CAAS,MAAM,CAAA;AAC3C,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,oBAAoB,gBAAA,EAAoD;AACtE,IAAA,IAAA,CAAK,gBAAA,GAAmB,gBAAA;AACxB,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAeA,sBAAsB,QAAA,EAA0C;AAC9D,IAAA,IAAA,CAAK,cAAA,GAAiB,CAAC,GAAG,QAAQ,CAAA;AAClC,IAAA,IAAA,CAAK,qBAAA,GAAwB,IAAA;AAC7B,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,sBAAA,CACE,KACA,QAAA,EACgB;AAChB,IAAA,IAAA,CAAK,oBAAA,CAAqB,GAAG,CAAA,GAAI,QAAA;AACjC,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,yBAAyB,UAAA,EAAiD;AACxE,IAAAA,uBAAA,CAAO,KAAA,CAAM,IAAA,CAAK,qBAAA,EAAuB,UAAU,CAAA;AACnD,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,qBACK,SAAA,EACa;AAChB,IAAA,IAAA,CAAK,eAAA,CAAgB,IAAA,CAAK,GAAG,SAAA,CAAU,MAAM,CAAA;AAC7C,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,gBACK,UAAA,EACa;AAChB,IAAA,IAAA,CAAK,UAAA,CAAW,IAAA,CAAK,GAAG,UAAA,CAAW,MAAM,CAAA;AACzC,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,kBAAkB,UAAA,EAAgD;AAChE,IAAA,IAAA,CAAK,UAAA,GAAa,CAAC,GAAG,UAAU,CAAA;AAChC,IAAA,IAAA,CAAK,iBAAA,GAAoB,IAAA;AACzB,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,oBAAA,GAA2C;AACzC,IAAA,MAAM,EAAE,MAAA,EAAQ,MAAA,EAAQ,MAAA,KAAW,IAAA,CAAK,GAAA;AACxC,IAAA,MAAM,YAAA,GAAeC,2BAAA,CAAgB,UAAA,CAAW,MAAM,CAAA;AAEtD,IAAA,OAAO;AAAA,MACL,IAAIC,uCAAA,EAAoB;AAAA,MACxB,IAAIC,qCAAA,CAAmB,EAAE,MAAA,EAAQ,QAAQ,CAAA;AAAA,MACzC,IAAIC,+DAAA,CAAgC,EAAE,YAAA,EAAc;AAAA,KACtD;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,wBACK,SAAA,EACa;AAChB,IAAA,IAAA,CAAK,iBAAA,CAAkB,IAAA,CAAK,GAAG,SAAA,CAAU,MAAM,CAAA;AAC/C,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,oBAAoB,MAAA,EAAgD;AAClE,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AACd,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,wBAAwB,oBAAA,EAAgD;AACtE,IAAA,IAAA,CAAK,mBAAA,GAAsB,oBAAA;AAC3B,IAAA,OAAO,IAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,KAAA,GAGH;AACD,IAAA,MAAM;AAAA,MACJ,MAAA;AAAA,MACA,QAAA;AAAA,MACA,MAAA;AAAA,MACA,WAAA;AAAA,MACA,SAAA;AAAA,MACA,mBAAA;AAAA,MACA,OAAA;AAAA,MACA,IAAA;AAAA,MACA,QAAA;AAAA,MACA,MAAA;AAAA,MACA,gBAAA;AAAA,MACA;AAAA,QACE,IAAA,CAAK,GAAA;AAET,IAAA,MAAM,MAAA,GAAS,KAAK,iBAAA,EAAkB;AACtC,IAAA,MAAM,UAAA,GAAa,KAAK,eAAA,EAAgB;AACxC,IAAA,MAAM,MAAA,GAAS,KAAK,MAAA,IAAUC,6BAAA;AAE9B,IAAA,MAAM,QAAA,GAAW,MAAM,QAAA,CAAS,SAAA,EAAU;AAC1C,IAAA,IAAI,CAAC,QAAA,CAAS,UAAA,EAAY,IAAA,EAAM;AAC9B,MAAA,MAAA,CAAO,KAAK,+BAA+B,CAAA;AAC3C,MAAA,MAAMC,mCAAwB,QAAQ,CAAA;AAAA,IACxC;AAEA,IAAA,MAAM,QAAA,GAAWC,+BAAA,CAAgB,UAAA,CAAW,MAAA,EAAQ;AAAA,MAClD,IAAA,EAAM,QAAA;AAAA,MACN,MAAA;AAAA,MACA;AAAA,KACD,CAAA;AAED,IAAA,MAAM,kBAAA,GAAqB,IAAIC,mDAAA,CAA0B;AAAA,MACvD,QAAA,EAAU,QAAA;AAAA,MACV,MAAA;AAAA,MACA,MAAA;AAAA,MACA,iBAAiB,IAAA,CAAK,kBAAA;AAAA,MACtB;AAAA,KACD,CAAA;AACD,IAAA,MAAM,gBAAA,GAAmB,IAAIC,+CAAA,CAAwB;AAAA,MACnD,QAAA,EAAU,QAAA;AAAA,MACV;AAAA,KACD,CAAA;AACD,IAAA,MAAM,eAAA,GAAkB,IAAIC,6CAAA,CAAuB;AAAA,MACjD,QAAA,EAAU,QAAA;AAAA,MACV;AAAA,KACD,CAAA;AACD,IAAA,MAAM,YAAA,GAAeT,2BAAA,CAAgB,UAAA,CAAW,MAAM,CAAA;AACtD,IAAA,MAAM,aAAA,GAAgBU,wCAAA,CAA4B,UAAA,CAAW,MAAM,CAAA;AAEnE,IAAA,MAAM,2BAAA,GAA8B,IAAIC,6CAAA,CAAuB;AAAA,MAC7D,QAAA,EAAU,QAAA;AAAA,MACV;AAAA,KACD,CAAA;AAED,IAAA,MAAM,YAAA,GAAe,IAAIC,yEAAA,CAAqC;AAAA,MAC5D,UAAA;AAAA,MACA,YAAA;AAAA,MACA,aAAA;AAAA,MACA,MAAA;AAAA,MACA,MAAA;AAAA,MACA,MAAA;AAAA,MACA,sBAAsB,IAAA,CAAK;AAAA,KAC5B,CAAA;AAED,IAAA,MAAM,kBAAkB,IAAIC,mDAAA;AAAA,MAC1B,2BAAA;AAAA,MACA,WAAA;AAAA,MACAC,+CAAA;AAAA,QACE,mBAAA,CAAoB,oBAAA;AAAA,UAClBC;AAAA;AACF;AACF,KACF;AAEA,IAAA,MAAM,YAAA,GAAe,OAAO,YAAA,KAA2B;AACrD,MAAA,MAAM,EAAE,KAAA,EAAM,GAAI,MAAM,4BAA4B,aAAA,CAAc;AAAA,QAChE,WAAA,EAAa,MAAM,IAAA,CAAK,wBAAA,EAAyB;AAAA,QACjD,UAAA,EAAY;AAAA,OACb,CAAA;AAED,MAAA,OAAOC,oCAA0B,KAAK,CAAA,CAAE,GAAA,CAAI,CAAA,CAAA,KAAK,KAAK,MAAS,CAAA;AAAA,IACjE,CAAA;AAEA,IAAA,mBAAA,CAAoB,eAAA,CAAgB;AAAA,MAClC,WAAA,EAAaD,wCAAA;AAAA,MACb,YAAA;AAAA,MACA,WAAA,EAAa,CAAC,GAAGE,0BAAkB,CAAA;AAAA,MACnC,KAAA,EAAO,MAAA,CAAO,MAAA,CAAOC,qBAAsB;AAAA,KAC5C,CAAA;AAED,IAAA,MAAM,sBAAA,GAAyBC,sDAA2B,MAAM,CAAA;AAChE,IAAA,MAAM,gBAAgB,IAAIC,yCAAA;AAAA,MACxB,QAAA;AAAA,MACA,gBAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,MAAM,sBAAA,GAAyB,IAAIC,yDAAA,CAA6B,MAAM,CAAA;AACtE,IAAA,MAAM,YAAY,IAAIC,6DAAA;AAAA,MACpB,QAAA;AAAA,MACA,gBAAA;AAAA,MACA;AAAA,KACF;AAEA,IAAA,MAAM,wBAAwBvB,uBAAA,CAAO,MAAA;AAAA,MACnC;AAAA,QACE,GAAG,IAAA,CAAK,eAAA;AAAA,QACR,EAAE,UAAU,aAAA,EAAc;AAAA,QAC1B,EAAE,UAAU,sBAAA,EAAuB;AAAA,QACnC,EAAE,UAAU,SAAA;AAAU,OACxB;AAAA,MACA,CAAA,KAAA,KAAS,KAAA,CAAM,QAAA,CAAS,eAAA;AAAgB,KAC1C;AACA,IAAA,MAAM,sBAAA,GAAyBwB,oBAAA;AAAA,MAC7B,qBAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,MAAM,mBAAmB,sBAAA,CAAuB,GAAA;AAAA,MAC9C,WAAS,KAAA,CAAM;AAAA,KACjB;AAEA,IAAA,MAAM,gBAAA,GAAmB,IAAIC,6DAAA,CAA+B;AAAA,MAC1D,MAAA;AAAA,MACA,SAAA;AAAA,MACA,MAAA;AAAA,MACA,IAAA,EAAM,QAAA;AAAA,MACN,kBAAA;AAAA,MACA,YAAA;AAAA,MACA,UAAA,EAAY,MAAMC,sBAAA,CAAW,MAAM,CAAA;AAAA,MACnC,iBAAA,EAAmB,GAAA;AAAA,MACnB,mBAAmB,CAAA,KAAA,KAAS;AAC1B,QAAA,IAAA,CAAK,oBAAoB,KAAK,CAAA;AAAA,MAChC,CAAA;AAAA,MACA,MAAA;AAAA,MACA;AAAA,KACD,CAAA;AAED,IAAA,MAAM,gBAAA,GACJ,IAAA,CAAK,gBAAA,IACL,IAAIC,qDAAA;AAAA,MACF,IAAIC,qCAAA,CAAqB,MAAA,EAAQ,YAAA,EAAc,KAAK,iBAAiB,CAAA;AAAA,MACrE;AAAA,KACF;AACF,IAAA,MAAM,kBAAkB,IAAIC,mDAAA;AAAA,MAC1B,IAAIC,6CAAA,CAAuB,aAAA,EAAe,YAAA,EAAc;AAAA,QACtD,sBAAsB,IAAA,CAAK,mBAAA;AAAA,QAC3B,iCACG,MAAA,CAAO,iBAAA;AAAA,UACN;AAAA,SACF,IAA8B;AAAA,OACjC,CAAA;AAAA,MACD;AAAA,KACF;AACA,IAAA,MAAM,iBAAiB,IAAIC,iDAAA;AAAA,MACzB,IAAIC,2CAAA,CAAsB,EAAE,QAAA,EAAU,iBAAiB,CAAA;AAAA,MACvD;AAAA,KACF;AAEA,IAAA,MAAM,MAAA,GAAS,MAAMC,yBAAA,CAAa;AAAA,MAChC,eAAA;AAAA,MACA,gBAAA;AAAA,MACA,eAAA;AAAA,MACA,YAAA;AAAA,MACA,cAAA;AAAA,MACA,MAAA;AAAA,MACA,MAAA;AAAA,MACA,IAAA;AAAA,MACA,QAAA;AAAA,MACA,kBAAA,EAAoB,WAAA;AAAA,MACpB;AAAA,KACD,CAAA;AAED,IAAA,MAAMC,6CAAA,CAAuB,kBAAkB,sBAAsB,CAAA;AAErE,IAAA,OAAO;AAAA,MACL,gBAAA,EAAkB;AAAA,QAChB,MAAM,KAAA,GAAQ;AACZ,UAAA,IACE,MAAA,CAAO,iBAAA,CAAkB,gCAAgC,CAAA,KACzD,MAAA,EACA;AACA,YAAA,MAAMC,qEAAA,CAAmC;AAAA,cACvC,EAAA,EAAI,gBAAA;AAAA,cACJ,SAAA,EAAW,gBAAA;AAAA,cACX;AAAA,aACD,CAAA;AAAA,UACH;AACA,UAAA,MAAM,iBAAiB,KAAA,EAAM;AAC7B,UAAA,MAAM,SAAS,KAAA,EAAM;AAAA,QACvB,CAAA;AAAA,QACA,MAAM,IAAA,GAAO;AACX,UAAA,MAAM,iBAAiB,IAAA,EAAK;AAC5B,UAAA,MAAM,SAAS,IAAA,EAAK;AAAA,QACtB;AAAA,OACF;AAAA,MACA;AAAA,KACF;AAAA,EACF;AAAA,EAEA,UAAU,OAAA,EAKP;AACD,IAAA,IAAA,CAAK,oBAAoB,OAAA,CAAQ,iBAAA;AAAA,EACnC;AAAA,EAEQ,iBAAA,GAAkC;AACxC,IAAA,MAAM,cAAA,GAAiC,IAAA,CAAK,qBAAA,GACxC,CAAC,IAAIC,sCAAwB,EAAG,GAAG,IAAA,CAAK,cAAc,CAAA,GACtD;AAAA,MACE,IAAIA,oCAAA,EAAwB;AAAA,MAC5B,IAAIC,yCAAA,EAA6B;AAAA,MACjC,IAAIC,4CAAA,EAAgC;AAAA,MACpC,IAAIC,oCAAA;AAAA,QACFC,0BAAA,CAAc,KAAK,qBAAqB;AAAA,OAC1C;AAAA,MACA,GAAG,IAAA,CAAK;AAAA,KACV;AAEJ,IAAA,OAAOC,2BAAA,CAAe,MAAM,cAAc,CAAA;AAAA,EAC5C;AAAA,EAEQ,eAAA,GAAsC;AAC5C,IAAA,MAAM,EAAE,MAAA,EAAQ,MAAA,EAAO,GAAI,IAAA,CAAK,GAAA;AAChC,IAAA,MAAM,YAAA,GAAexC,2BAAA,CAAgB,UAAA,CAAW,MAAM,CAAA;AAEtD,IAAA,IAAA,CAAK,+BAAA,EAAgC;AAErC,IAAA,MAAM,oBAAA,GAA4D;AAAA,MAChE,IAAA,EAAMyC,4CAAA;AAAA,MACN,IAAA,EAAMC,4CAAA;AAAA,MACN,IAAA,EAAMC,4CAAA;AAAA,MACN,GAAG,IAAA,CAAK;AAAA,KACV;AAGA,IAAA,MAAM,UAAA,GAAiC;AAAA,MACrC,IAAIC,yCAAA,CAAqB;AAAA,QACvB,SAAA,EAAW,oBAAA;AAAA,QACX,MAAA;AAAA,QACA;AAAA,OACD;AAAA,KACH;AAEA,IAAA,MAAM,2BAAA,GAA8B,IAAIC,uDAAA,EAA4B;AAIpE,IAAA,IACE,CAAC,IAAA,CAAK,GAAA,CAAI,WAAA,IACV,CAAC,KAAK,UAAA,CAAW,IAAA;AAAA,MACf,CAAA,SAAA,KACE,SAAA,CAAU,gBAAA,EAAiB,KAC3B,4BAA4B,gBAAA;AAAiB,KACjD,EACA;AACA,MAAA,UAAA,CAAW,KAAK,2BAA2B,CAAA;AAAA,IAC7C;AACA,IAAA,IAAI,IAAA,CAAK,IAAI,WAAA,EAAa;AACxB,MAAA,UAAA,CAAW,KAAK,IAAIC,6BAAA,CAAe,IAAA,CAAK,GAAA,CAAI,WAAW,CAAC,CAAA;AAAA,IAC1D;AAEA,IAAA,MAAM,2BAA2B,MAAA,CAAO,kBAAA;AAAA,MACtC;AAAA,KACF;AAKA,IAAA,IAAI,CAAC,IAAA,CAAK,iBAAA,IAAqB,CAAC,wBAAA,EAA0B;AACxD,MAAA,UAAA,CAAW,IAAA,CAAK,GAAG,IAAA,CAAK,oBAAA,EAAsB,CAAA;AAAA,IAChD;AAGA,IAAA,UAAA,CAAW,IAAA,CAAK,GAAG,IAAA,CAAK,UAAU,CAAA;AAElC,IAAA,IAAA,CAAK,+BAA+B,UAAU,CAAA;AAE9C,IAAA,OAAOC,4BAAA,CAAwB,YAAY,MAAM,CAAA;AAAA,EACnD;AAAA;AAAA;AAAA,EAIQ,+BAAA,GAAkC;AACxC,IAAA,MAAM,EAAA,GAAK,IAAA,CAAK,GAAA,CAAI,MAAA,CAAO,kBAAkB,oBAAoB,CAAA;AACjE,IAAA,IAAI,EAAA,EAAI,GAAA,CAAI,QAAQ,CAAA,EAAG;AACrB,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,CAAA,uGAAA;AAAA,OACF;AAAA,IACF;AACA,IAAA,IAAI,EAAA,EAAI,GAAA,CAAI,WAAW,CAAA,EAAG;AACxB,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,CAAA,0GAAA;AAAA,OACF;AAAA,IACF;AACA,IAAA,IAAI,EAAA,EAAI,GAAA,CAAI,cAAc,CAAA,EAAG;AAC3B,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,CAAA,gHAAA;AAAA,OACF;AAAA,IACF;AACA,IAAA,IAAI,EAAA,EAAI,GAAA,CAAI,UAAU,CAAA,EAAG;AACvB,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,CAAA,wGAAA;AAAA,OACF;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGQ,+BAA+B,UAAA,EAAgC;AACrE,IAAA,MAAM,gBAAA,GAAmB,iDAAA;AACzB,IAAA,IAAI,OAAA,CAAQ,GAAA,CAAI,gBAAgB,CAAA,EAAG;AACjC,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,gBAAgB,IAAI,GAAA;AAAA,MACxB,IAAA,CAAK,GAAA,CAAI,MAAA,CACN,sBAAA,CAAuB,mBAAmB,CAAA,EACzC,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,SAAA,CAAU,MAAM,CAAC,KAAK;AAAC,KACxC;AACA,IAAA,MAAM,cAAA,GAAiB,IAAI,GAAA,CAAI,UAAA,CAAW,IAAI,CAAA,CAAA,KAAK,CAAA,CAAE,gBAAA,EAAkB,CAAC,CAAA;AAExE,IAAA,SAAS,KAAA,CACP,YAAA,EACA,aAAA,EACA,eAAA,EACA;AACA,MAAA,IACE,aAAA,CAAc,IAAI,YAAY,CAAA,IAC9B,CAAC,cAAA,CAAe,GAAA,CAAI,aAAa,CAAA,EACjC;AACA,QAAA,MAAM,IAAI,KAAA;AAAA,UACR;AAAA,YACE,4DAA4D,YAAY,CAAA,CAAA,CAAA;AAAA,YACxE,yDAAyD,aAAa,CAAA,WAAA,CAAA;AAAA,YACtE,CAAA,+EAAA,CAAA;AAAA,YACA,CAAA,iFAAA,CAAA;AAAA,YACA,mBAAmB,eAAe,CAAA,6CAAA,CAAA;AAAA,YAClC,CAAA,qFAAA,CAAA;AAAA,YACA,uCAAuC,gBAAgB,CAAA,WAAA;AAAA,WACzD,CAAE,KAAK,GAAG;AAAA,SACZ;AAAA,MACF;AAAA,IACF;AAEA,IAAA,KAAA;AAAA,MACE,oBAAA;AAAA,MACA,sCAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,KAAA;AAAA,MACE,cAAA;AAAA,MACA,yBAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,KAAA;AAAA,MACE,iBAAA;AAAA,MACA,+BAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,KAAA;AAAA,MACE,qBAAA;AAAA,MACA,6BAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,KAAA;AAAA,MACE,kBAAA;AAAA,MACA,0BAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,KAAA;AAAA,MACE,YAAA;AAAA,MACA,0BAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,KAAA;AAAA,MACE,kBAAA;AAAA,MACA,0BAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,KAAA;AAAA,MACE,UAAA;AAAA,MACA,wBAAA;AAAA,MACA;AAAA,KACF;AACA,IAAA,KAAA;AAAA,MACE,qBAAA;AAAA,MACA,kCAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF;AAAA,EAEA,OAAe,6BACbC,QAAA,EAC4B;AAC5B,IAAA,MAAM,qBAAA,GAAwB,4BAAA;AAE9B,IAAA,IAAI,CAACA,QAAA,CAAO,GAAA,CAAI,qBAAqB,CAAA,EAAG;AACtC,MAAA,OAAOC,sCAAA,CAA+B;AAAA,QACpC,UAAA,EAAY,GAAA;AAAA,QACZ,UAAA,EAAY;AAAA,OACb,CAAA;AAAA,IACH;AAEA,IAAA,IAAI,CAAC,OAAA,CAAQD,QAAA,CAAO,GAAA,CAAI,4BAA4B,CAAC,CAAA,EAAG;AACtD,MAAA,OAAO,MAAM;AACX,QAAA,MAAM,IAAI,KAAA;AAAA,UACR;AAAA,SACF;AAAA,MACF,CAAA;AAAA,IACF;AAEA,IAAA,MAAM,QAAA,GAAWE,8BAAuBF,QAAA,EAAQ;AAAA,MAC9C,GAAA,EAAK;AAAA,KACN,CAAA;AAED,IAAA,MAAM,UAAU,IAAA,CAAK,GAAA;AAAA,MACnB,CAAA;AAAA,MACA,IAAA,CAAK,KAAA,CAAMG,4BAAA,CAAuB,QAAQ,IAAI,GAAI;AAAA,KACpD;AAEA,IAAA,OAAOF,sCAAA,CAA+B;AAAA,MACpC,UAAA,EAAY,OAAA;AAAA,MACZ,YAAY,OAAA,GAAU;AAAA,KACvB,CAAA;AAAA,EACH;AACF;;;;"}

package/dist/service/DefaultEntitiesCatalog.cjs.js

@@ -49,13 +49,9 @@ function stringifyPagination(input) {
 class DefaultEntitiesCatalog {
   database;
   logger;
-  enableRelationsCompatibility;
   constructor(options) {
     this.database = options.database;
     this.logger = options.logger;
-    this.enableRelationsCompatibility = Boolean(
-      options.enableRelationsCompatibility
-    );
   }
   async entities(request) {
     const db = this.database;
@@ -138,13 +134,7 @@ class DefaultEntitiesCatalog {
     return {
       entities: process.processRawEntitiesResult(
         rows.map((r) => r.final_entity),
-        this.enableRelationsCompatibility ? (e) => {
-          util.expandLegacyCompoundRelationsInEntity(e);
-          if (request?.fields) {
-            return request.fields(e);
-          }
-          return e;
-        } : request?.fields
+        request?.fields
       ),
       pageInfo
     };