@backstage/plugin-catalog-backend 1.28.0-next.3 → 1.28.1-next.0

package/CHANGELOG.md

@@ -1,5 +1,70 @@
 # @backstage/plugin-catalog-backend
 
+## 1.28.1-next.0
+
+### Patch Changes
+
+- f159b25: Compute deltas more efficiently, which generally leads to less wasted processing cycles
+- 56511ba: Be more aggressive in dequeueing entities for stitching
+- 71152e3: Correctly report stitching queue length
+- 3ab57c6: Support changing location keys on existing entities, in delta mutations
+- 2924ffe: Compute some metrics using search table facet aggregations instead of reading the full refresh state
+- Updated dependencies
+  - @backstage/integration@1.16.0-next.0
+  - @backstage/backend-plugin-api@1.0.3-next.0
+  - @backstage/plugin-search-backend-module-catalog@0.2.6-next.0
+  - @backstage/plugin-events-node@0.4.6-next.0
+  - @backstage/catalog-client@1.8.1-next.0
+  - @backstage/backend-openapi-utils@0.3.1-next.0
+  - @backstage/catalog-model@1.7.1
+  - @backstage/config@1.3.0
+  - @backstage/errors@1.2.5
+  - @backstage/types@1.2.0
+  - @backstage/plugin-catalog-common@1.1.1
+  - @backstage/plugin-catalog-node@1.14.1-next.0
+  - @backstage/plugin-permission-common@0.8.2
+  - @backstage/plugin-permission-node@0.8.6-next.0
+
+## 1.28.0
+
+### Minor Changes
+
+- 39fd704: Internal update to use the new generated server types from `backstage-cli package schema openapi generate --server`.
+- 76857da: Added `entity_ref` column to `final_entities` in order to move `refresh_state` away from the read path
+- 34d4360: Drop redundant indices from the database.
+
+  The following redundant indices are removed in this version:
+
+  - `final_entities_entity_id_idx` - overlaps with `final_entities_pkey`
+  - `refresh_state_entity_id_idx` - overlaps with `refresh_state_pkey`
+  - `refresh_state_entity_ref_idx` - overlaps with `refresh_state_entity_ref_uniq`
+  - `search_key_idx` and `search_value_idx` - these were replaced by the composite index `search_key_value_idx` in #22594
+
+  No negative end user impact is expected, but rather that performance should increase due to less index churn.
+
+### Patch Changes
+
+- d52d7f9: Support ISO and ms string forms of durations in config too
+- b89834b: Fixed an issue where entities would not be marked for restitching if only the target of a relationship changed.
+- 1bf02cc: Fixed bug when searching an entity by `spec.profile.displayName` in the catalog on the frontend. Text filter fields were not applied correctly to the database query resulting in empty results.
+- 4e58bc7: Upgrade to uuid v11 internally
+- 5efde17: Internal refactor to slightly speed up the processing loop
+- Updated dependencies
+  - @backstage/catalog-client@1.8.0
+  - @backstage/config@1.3.0
+  - @backstage/plugin-events-node@0.4.5
+  - @backstage/types@1.2.0
+  - @backstage/plugin-search-backend-module-catalog@0.2.5
+  - @backstage/plugin-catalog-node@1.14.0
+  - @backstage/backend-plugin-api@1.0.2
+  - @backstage/backend-openapi-utils@0.3.0
+  - @backstage/plugin-permission-common@0.8.2
+  - @backstage/catalog-model@1.7.1
+  - @backstage/errors@1.2.5
+  - @backstage/integration@1.15.2
+  - @backstage/plugin-catalog-common@1.1.1
+  - @backstage/plugin-permission-node@0.8.5
+
 ## 1.28.0-next.3
 
 ### Minor Changes

package/config.d.ts

@@ -157,9 +157,9 @@ export interface Config {
           /** Defer stitching to be performed asynchronously */
           mode: 'deferred';
           /** Polling interval for tasks in seconds */
-          pollingInterval?: HumanDuration;
+          pollingInterval?: HumanDuration | string;
           /** How long to wait for a stitch to complete before giving up in seconds */
-          stitchTimeout?: HumanDuration;
+          stitchTimeout?: HumanDuration | string;
         };
 
     /**

package/dist/database/DefaultProcessingDatabase.cjs.js

@@ -104,24 +104,32 @@ class DefaultProcessingDatabase {
     const { id, state } = options;
     await tx("refresh_state").update({ cache: JSON.stringify(state ?? {}) }).where("entity_id", id);
   }
-  async getProcessableEntities(txOpaque, request) {
-    const tx = txOpaque;
-    let itemsQuery = tx("refresh_state").select();
-    if (["mysql", "mysql2", "pg"].includes(tx.client.config.client)) {
+  async getProcessableEntities(maybeTx, request) {
+    const knex = maybeTx;
+    let itemsQuery = knex("refresh_state").select([
+      "entity_id",
+      "entity_ref",
+      "unprocessed_entity",
+      "result_hash",
+      "cache",
+      "errors",
+      "location_key",
+      "next_update_at"
+    ]);
+    if (["mysql", "mysql2", "pg"].includes(knex.client.config.client)) {
       itemsQuery = itemsQuery.forUpdate().skipLocked();
     }
-    const items = await itemsQuery.where("next_update_at", "<=", tx.fn.now()).limit(request.processBatchSize).orderBy("next_update_at", "asc");
+    const items = await itemsQuery.where("next_update_at", "<=", knex.fn.now()).limit(request.processBatchSize).orderBy("next_update_at", "asc");
     const interval = this.options.refreshInterval();
     const nextUpdateAt = (refreshInterval) => {
-      if (tx.client.config.client.includes("sqlite3")) {
-        return tx.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);
-      }
-      if (tx.client.config.client.includes("mysql")) {
-        return tx.raw(`now() + interval ${refreshInterval} second`);
+      if (knex.client.config.client.includes("sqlite3")) {
+        return knex.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);
+      } else if (knex.client.config.client.includes("mysql")) {
+        return knex.raw(`now() + interval ${refreshInterval} second`);
       }
-      return tx.raw(`now() + interval '${refreshInterval} seconds'`);
+      return knex.raw(`now() + interval '${refreshInterval} seconds'`);
     };
-    await tx("refresh_state").whereIn(
+    await knex("refresh_state").whereIn(
       "entity_ref",
       items.map((i) => i.entity_ref)
     ).update({
@@ -133,10 +141,8 @@ class DefaultProcessingDatabase {
           id: i.entity_id,
           entityRef: i.entity_ref,
           unprocessedEntity: JSON.parse(i.unprocessed_entity),
-          processedEntity: i.processed_entity ? JSON.parse(i.processed_entity) : void 0,
           resultHash: i.result_hash || "",
           nextUpdateAt: conversion.timestampToDateTime(i.next_update_at),
-          lastDiscoveryAt: conversion.timestampToDateTime(i.last_discovery_at),
           state: i.cache ? JSON.parse(i.cache) : void 0,
           errors: i.errors,
           locationKey: i.location_key

package/dist/database/DefaultProcessingDatabase.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultProcessingDatabase.cjs.js","sources":["../../src/database/DefaultProcessingDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError } from '@backstage/errors';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { ProcessingIntervalFunction } from '../processing';\nimport { rethrowError, timestampToDateTime } from './conversion';\nimport { initDatabaseMetrics } from './metrics';\nimport {\n  DbRefreshKeysRow,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n} from './tables';\nimport {\n  GetProcessableEntitiesResult,\n  ListParentsOptions,\n  ListParentsResult,\n  ProcessingDatabase,\n  RefreshStateItem,\n  Transaction,\n  UpdateEntityCacheOptions,\n  UpdateProcessedEntityOptions,\n} from './types';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { generateStableHash, generateTargetKey } from './util';\nimport {\n  EventBroker,\n  EventParams,\n  EventsService,\n} from '@backstage/plugin-events-node';\nimport { DateTime } from 'luxon';\nimport { CATALOG_CONFLICTS_TOPIC } from '../constants';\nimport { CatalogConflictEventPayload } from '../catalog/types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProcessingDatabase implements ProcessingDatabase {\n  constructor(\n    private readonly options: {\n      database: Knex;\n      logger: LoggerService;\n      refreshInterval: ProcessingIntervalFunction;\n      eventBroker?: EventBroker | EventsService;\n    },\n  ) {\n    initDatabaseMetrics(options.database);\n  }\n\n  async updateProcessedEntity(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<{ previous: { relations: DbRelationsRow[] } }> {\n    const tx = txOpaque as Knex.Transaction;\n    const {\n      id,\n      processedEntity,\n      resultHash,\n      errors,\n      relations,\n      deferredEntities,\n      refreshKeys,\n      locationKey,\n    } = options;\n    const configClient = tx.client.config.client;\n    const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        processed_entity: JSON.stringify(processedEntity),\n        result_hash: resultHash,\n        errors,\n        location_key: locationKey,\n      })\n      .where('entity_id', id)\n      .andWhere(inner => {\n        if (!locationKey) {\n          return inner.whereNull('location_key');\n        }\n        return inner\n          .where('location_key', locationKey)\n          .orWhereNull('location_key');\n      });\n    if (refreshResult === 0) {\n      throw new ConflictError(\n        `Conflicting write of processing result for ${id} with location key '${locationKey}'`,\n      );\n    }\n    const sourceEntityRef = stringifyEntityRef(processedEntity);\n\n    // Schedule all deferred entities for future processing.\n    await this.addUnprocessedEntities(tx, {\n      entities: deferredEntities,\n      sourceEntityRef,\n    });\n\n    // Delete old relations\n    // NOTE(freben): knex implemented support for returning() on update queries for sqlite, but at the current time of writing (Sep 2022) not for delete() queries.\n    let previousRelationRows: DbRelationsRow[];\n    if (configClient.includes('sqlite3') || configClient.includes('mysql')) {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .select('*')\n        .where({ originating_entity_id: id });\n      await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete();\n    } else {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete()\n        .returning('*');\n    }\n\n    // Batch insert new relations\n    const relationRows: DbRelationsRow[] = relations.map(\n      ({ source, target, type }) => ({\n        originating_entity_id: id,\n        source_entity_ref: stringifyEntityRef(source),\n        target_entity_ref: stringifyEntityRef(target),\n        type,\n      }),\n    );\n\n    await tx.batchInsert(\n      'relations',\n      this.deduplicateRelations(relationRows),\n      BATCH_SIZE,\n    );\n\n    // Delete old refresh keys\n    await tx<DbRefreshKeysRow>('refresh_keys')\n      .where({ entity_id: id })\n      .delete();\n\n    // Insert the refresh keys for the processed entity\n    await tx.batchInsert(\n      'refresh_keys',\n      refreshKeys.map(k => ({\n        entity_id: id,\n        key: generateTargetKey(k.key),\n      })),\n      BATCH_SIZE,\n    );\n\n    return {\n      previous: {\n        relations: previousRelationRows,\n      },\n    };\n  }\n\n  async updateProcessedEntityErrors(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, errors, resultHash } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        errors,\n        result_hash: resultHash,\n      })\n      .where('entity_id', id);\n  }\n\n  async updateEntityCache(\n    txOpaque: Transaction,\n    options: UpdateEntityCacheOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, state } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({ cache: JSON.stringify(state ?? {}) })\n      .where('entity_id', id);\n  }\n\n  async getProcessableEntities(\n    txOpaque: Transaction,\n    request: { processBatchSize: number },\n  ): Promise<GetProcessableEntitiesResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    let itemsQuery = tx<DbRefreshStateRow>('refresh_state').select();\n\n    // This avoids duplication of work because of race conditions and is\n    // also fast because locked rows are ignored rather than blocking.\n    // It's only available in MySQL and PostgreSQL\n    if (['mysql', 'mysql2', 'pg'].includes(tx.client.config.client)) {\n      itemsQuery = itemsQuery.forUpdate().skipLocked();\n    }\n\n    const items = await itemsQuery\n      .where('next_update_at', '<=', tx.fn.now())\n      .limit(request.processBatchSize)\n      .orderBy('next_update_at', 'asc');\n\n    const interval = this.options.refreshInterval();\n\n    const nextUpdateAt = (refreshInterval: number) => {\n      if (tx.client.config.client.includes('sqlite3')) {\n        return tx.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);\n      }\n\n      if (tx.client.config.client.includes('mysql')) {\n        return tx.raw(`now() + interval ${refreshInterval} second`);\n      }\n\n      return tx.raw(`now() + interval '${refreshInterval} seconds'`);\n    };\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .whereIn(\n        'entity_ref',\n        items.map(i => i.entity_ref),\n      )\n      .update({\n        next_update_at: nextUpdateAt(interval),\n      });\n\n    return {\n      items: items.map(\n        i =>\n          ({\n            id: i.entity_id,\n            entityRef: i.entity_ref,\n            unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity,\n            processedEntity: i.processed_entity\n              ? (JSON.parse(i.processed_entity) as Entity)\n              : undefined,\n            resultHash: i.result_hash || '',\n            nextUpdateAt: timestampToDateTime(i.next_update_at),\n            lastDiscoveryAt: timestampToDateTime(i.last_discovery_at),\n            state: i.cache ? JSON.parse(i.cache) : undefined,\n            errors: i.errors,\n            locationKey: i.location_key,\n          } as RefreshStateItem),\n      ),\n    };\n  }\n\n  async listParents(\n    txOpaque: Transaction,\n    options: ListParentsOptions,\n  ): Promise<ListParentsResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .where({ target_entity_ref: options.entityRef })\n      .select();\n\n    const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);\n\n    return { entityRefs };\n  }\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] {\n    return lodash.uniqBy(\n      rows,\n      r => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`,\n    );\n  }\n\n  /**\n   * Add a set of deferred entities for processing.\n   * The entities will be added at the front of the processing queue.\n   */\n  private async addUnprocessedEntities(\n    txOpaque: Transaction,\n    options: {\n      sourceEntityRef: string;\n      entities: DeferredEntity[];\n    },\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n\n    // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards\n    const stateReferences = new Array<string>();\n\n    // Upsert all of the unprocessed entities into the refresh_state table, by\n    // their entity ref.\n    for (const { entity, locationKey } of options.entities) {\n      const entityRef = stringifyEntityRef(entity);\n      const hash = generateStableHash(entity);\n\n      const updated = await updateUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n      });\n      if (updated) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      const inserted = await insertUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n        logger: this.options.logger,\n      });\n      if (inserted) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      // If the row can't be inserted, we have a conflict, but it could be either\n      // because of a conflicting locationKey or a race with another instance, so check\n      // whether the conflicting entity has the same entityRef but a different locationKey\n      const conflictingKey = await checkLocationKeyConflict({\n        tx,\n        entityRef,\n        locationKey,\n      });\n      if (conflictingKey) {\n        this.options.logger.warn(\n          `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n        );\n        if (this.options.eventBroker && locationKey) {\n          const eventParams: EventParams<CatalogConflictEventPayload> = {\n            topic: CATALOG_CONFLICTS_TOPIC,\n            eventPayload: {\n              unprocessedEntity: entity,\n              entityRef,\n              newLocationKey: locationKey,\n              existingLocationKey: conflictingKey,\n              lastConflictAt: DateTime.now().toISO()!,\n            },\n          };\n          await this.options.eventBroker?.publish(eventParams);\n        }\n      }\n    }\n\n    // Replace all references for the originating entity or source and then create new ones\n    await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n      .andWhere({ source_entity_ref: options.sourceEntityRef })\n      .delete();\n    await tx.batchInsert(\n      'refresh_state_references',\n      stateReferences.map(entityRef => ({\n        source_entity_ref: options.sourceEntityRef,\n        target_entity_ref: entityRef,\n      })),\n      BATCH_SIZE,\n    );\n  }\n}\n"],"names":["initDatabaseMetrics","errors","ConflictError","stringifyEntityRef","generateTargetKey","timestampToDateTime","rethrowError","lodash","generateStableHash","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","CATALOG_CONFLICTS_TOPIC","DateTime"],"mappings":";;;;;;;;;;;;;;;;;;AA0DA,MAAM,UAAa,GAAA,EAAA;AAEZ,MAAM,yBAAwD,CAAA;AAAA,EACnE,YACmB,OAMjB,EAAA;AANiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAOjB,IAAAA,2BAAA,CAAoB,QAAQ,QAAQ,CAAA;AAAA;AACtC,EAEA,MAAM,qBACJ,CAAA,QAAA,EACA,OACwD,EAAA;AACxD,IAAA,MAAM,EAAK,GAAA,QAAA;AACX,IAAM,MAAA;AAAA,MACJ,EAAA;AAAA,MACA,eAAA;AAAA,MACA,UAAA;AAAA,cACAC,QAAA;AAAA,MACA,SAAA;AAAA,MACA,gBAAA;AAAA,MACA,WAAA;AAAA,MACA;AAAA,KACE,GAAA,OAAA;AACJ,IAAM,MAAA,YAAA,GAAe,EAAG,CAAA,MAAA,CAAO,MAAO,CAAA,MAAA;AACtC,IAAA,MAAM,aAAgB,GAAA,MAAM,EAAsB,CAAA,eAAe,EAC9D,MAAO,CAAA;AAAA,MACN,gBAAA,EAAkB,IAAK,CAAA,SAAA,CAAU,eAAe,CAAA;AAAA,MAChD,WAAa,EAAA,UAAA;AAAA,cACbA,QAAA;AAAA,MACA,YAAc,EAAA;AAAA,KACf,CACA,CAAA,KAAA,CAAM,aAAa,EAAE,CAAA,CACrB,SAAS,CAAS,KAAA,KAAA;AACjB,MAAA,IAAI,CAAC,WAAa,EAAA;AAChB,QAAO,OAAA,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA;AAEvC,MAAA,OAAO,MACJ,KAAM,CAAA,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,KAC9B,CAAA;AACH,IAAA,IAAI,kBAAkB,CAAG,EAAA;AACvB,MAAA,MAAM,IAAIC,oBAAA;AAAA,QACR,CAAA,2CAAA,EAA8C,EAAE,CAAA,oBAAA,EAAuB,WAAW,CAAA,CAAA;AAAA,OACpF;AAAA;AAEF,IAAM,MAAA,eAAA,GAAkBC,gCAAmB,eAAe,CAAA;AAG1D,IAAM,MAAA,IAAA,CAAK,uBAAuB,EAAI,EAAA;AAAA,MACpC,QAAU,EAAA,gBAAA;AAAA,MACV;AAAA,KACD,CAAA;AAID,IAAI,IAAA,oBAAA;AACJ,IAAA,IAAI,aAAa,QAAS,CAAA,SAAS,KAAK,YAAa,CAAA,QAAA,CAAS,OAAO,CAAG,EAAA;AACtE,MAAuB,oBAAA,GAAA,MAAM,EAAmB,CAAA,WAAW,CACxD,CAAA,MAAA,CAAO,GAAG,CAAA,CACV,KAAM,CAAA,EAAE,qBAAuB,EAAA,EAAA,EAAI,CAAA;AACtC,MAAM,MAAA,EAAA,CAAmB,WAAW,CACjC,CAAA,KAAA,CAAM,EAAE,qBAAuB,EAAA,EAAA,EAAI,CAAA,CACnC,MAAO,EAAA;AAAA,KACL,MAAA;AACL,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,KAAM,CAAA,EAAE,qBAAuB,EAAA,EAAA,EAAI,CAAA,CACnC,MAAO,EAAA,CACP,UAAU,GAAG,CAAA;AAAA;AAIlB,IAAA,MAAM,eAAiC,SAAU,CAAA,GAAA;AAAA,MAC/C,CAAC,EAAE,MAAQ,EAAA,MAAA,EAAQ,MAAY,MAAA;AAAA,QAC7B,qBAAuB,EAAA,EAAA;AAAA,QACvB,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C;AAAA,OACF;AAAA,KACF;AAEA,IAAA,MAAM,EAAG,CAAA,WAAA;AAAA,MACP,WAAA;AAAA,MACA,IAAA,CAAK,qBAAqB,YAAY,CAAA;AAAA,MACtC;AAAA,KACF;AAGA,IAAM,MAAA,EAAA,CAAqB,cAAc,CACtC,CAAA,KAAA,CAAM,EAAE,SAAW,EAAA,EAAA,EAAI,CAAA,CACvB,MAAO,EAAA;AAGV,IAAA,MAAM,EAAG,CAAA,WAAA;AAAA,MACP,cAAA;AAAA,MACA,WAAA,CAAY,IAAI,CAAM,CAAA,MAAA;AAAA,QACpB,SAAW,EAAA,EAAA;AAAA,QACX,GAAA,EAAKC,sBAAkB,CAAA,CAAA,CAAE,GAAG;AAAA,OAC5B,CAAA,CAAA;AAAA,MACF;AAAA,KACF;AAEA,IAAO,OAAA;AAAA,MACL,QAAU,EAAA;AAAA,QACR,SAAW,EAAA;AAAA;AACb,KACF;AAAA;AACF,EAEA,MAAM,2BACJ,CAAA,QAAA,EACA,OACe,EAAA;AACf,IAAA,MAAM,EAAK,GAAA,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,MAAQ,EAAA,UAAA,EAAe,GAAA,OAAA;AAEnC,IAAM,MAAA,EAAA,CAAsB,eAAe,CAAA,CACxC,MAAO,CAAA;AAAA,MACN,MAAA;AAAA,MACA,WAAa,EAAA;AAAA,KACd,CAAA,CACA,KAAM,CAAA,WAAA,EAAa,EAAE,CAAA;AAAA;AAC1B,EAEA,MAAM,iBACJ,CAAA,QAAA,EACA,OACe,EAAA;AACf,IAAA,MAAM,EAAK,GAAA,QAAA;AACX,IAAM,MAAA,EAAE,EAAI,EAAA,KAAA,EAAU,GAAA,OAAA;AAEtB,IAAA,MAAM,GAAsB,eAAe,CAAA,CACxC,MAAO,CAAA,EAAE,OAAO,IAAK,CAAA,SAAA,CAAU,KAAS,IAAA,EAAE,CAAE,EAAC,CAC7C,CAAA,KAAA,CAAM,aAAa,EAAE,CAAA;AAAA;AAC1B,EAEA,MAAM,sBACJ,CAAA,QAAA,EACA,OACuC,EAAA;AACvC,IAAA,MAAM,EAAK,GAAA,QAAA;AAEX,IAAA,IAAI,UAAa,GAAA,EAAA,CAAsB,eAAe,CAAA,CAAE,MAAO,EAAA;AAK/D,IAAI,IAAA,CAAC,OAAS,EAAA,QAAA,EAAU,IAAI,CAAA,CAAE,SAAS,EAAG,CAAA,MAAA,CAAO,MAAO,CAAA,MAAM,CAAG,EAAA;AAC/D,MAAa,UAAA,GAAA,UAAA,CAAW,SAAU,EAAA,CAAE,UAAW,EAAA;AAAA;AAGjD,IAAA,MAAM,QAAQ,MAAM,UAAA,CACjB,KAAM,CAAA,gBAAA,EAAkB,MAAM,EAAG,CAAA,EAAA,CAAG,GAAI,EAAC,EACzC,KAAM,CAAA,OAAA,CAAQ,gBAAgB,CAC9B,CAAA,OAAA,CAAQ,kBAAkB,KAAK,CAAA;AAElC,IAAM,MAAA,QAAA,GAAW,IAAK,CAAA,OAAA,CAAQ,eAAgB,EAAA;AAE9C,IAAM,MAAA,YAAA,GAAe,CAAC,eAA4B,KAAA;AAChD,MAAA,IAAI,GAAG,MAAO,CAAA,MAAA,CAAO,MAAO,CAAA,QAAA,CAAS,SAAS,CAAG,EAAA;AAC/C,QAAA,OAAO,GAAG,GAAI,CAAA,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAG,EAAA,eAAe,UAAU,CAAC,CAAA;AAAA;AAGpE,MAAA,IAAI,GAAG,MAAO,CAAA,MAAA,CAAO,MAAO,CAAA,QAAA,CAAS,OAAO,CAAG,EAAA;AAC7C,QAAA,OAAO,EAAG,CAAA,GAAA,CAAI,CAAoB,iBAAA,EAAA,eAAe,CAAS,OAAA,CAAA,CAAA;AAAA;AAG5D,MAAA,OAAO,EAAG,CAAA,GAAA,CAAI,CAAqB,kBAAA,EAAA,eAAe,CAAW,SAAA,CAAA,CAAA;AAAA,KAC/D;AAEA,IAAM,MAAA,EAAA,CAAsB,eAAe,CACxC,CAAA,OAAA;AAAA,MACC,YAAA;AAAA,MACA,KAAM,CAAA,GAAA,CAAI,CAAK,CAAA,KAAA,CAAA,CAAE,UAAU;AAAA,MAE5B,MAAO,CAAA;AAAA,MACN,cAAA,EAAgB,aAAa,QAAQ;AAAA,KACtC,CAAA;AAEH,IAAO,OAAA;AAAA,MACL,OAAO,KAAM,CAAA,GAAA;AAAA,QACX,CACG,CAAA,MAAA;AAAA,UACC,IAAI,CAAE,CAAA,SAAA;AAAA,UACN,WAAW,CAAE,CAAA,UAAA;AAAA,UACb,iBAAmB,EAAA,IAAA,CAAK,KAAM,CAAA,CAAA,CAAE,kBAAkB,CAAA;AAAA,UAClD,iBAAiB,CAAE,CAAA,gBAAA,GACd,KAAK,KAAM,CAAA,CAAA,CAAE,gBAAgB,CAC9B,GAAA,KAAA,CAAA;AAAA,UACJ,UAAA,EAAY,EAAE,WAAe,IAAA,EAAA;AAAA,UAC7B,YAAA,EAAcC,8BAAoB,CAAA,CAAA,CAAE,cAAc,CAAA;AAAA,UAClD,eAAA,EAAiBA,8BAAoB,CAAA,CAAA,CAAE,iBAAiB,CAAA;AAAA,UACxD,OAAO,CAAE,CAAA,KAAA,GAAQ,KAAK,KAAM,CAAA,CAAA,CAAE,KAAK,CAAI,GAAA,KAAA,CAAA;AAAA,UACvC,QAAQ,CAAE,CAAA,MAAA;AAAA,UACV,aAAa,CAAE,CAAA;AAAA,SACjB;AAAA;AACJ,KACF;AAAA;AACF,EAEA,MAAM,WACJ,CAAA,QAAA,EACA,OAC4B,EAAA;AAC5B,IAAA,MAAM,EAAK,GAAA,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,KACF,CACG,MAAM,EAAE,iBAAA,EAAmB,QAAQ,SAAU,EAAC,EAC9C,MAAO,EAAA;AAEV,IAAM,MAAA,UAAA,GAAa,KAAK,GAAI,CAAA,CAAA,CAAA,KAAK,EAAE,iBAAkB,CAAA,CAAE,OAAO,OAAO,CAAA;AAErE,IAAA,OAAO,EAAE,UAAW,EAAA;AAAA;AACtB,EAEA,MAAM,YAAe,EAAiD,EAAA;AACpE,IAAI,IAAA;AACF,MAAA,IAAI,MAAwB,GAAA,KAAA,CAAA;AAE5B,MAAM,MAAA,IAAA,CAAK,QAAQ,QAAS,CAAA,WAAA;AAAA,QAC1B,OAAM,EAAM,KAAA;AAGV,UAAS,MAAA,GAAA,MAAM,GAAG,EAAE,CAAA;AAAA,SACtB;AAAA,QACA;AAAA;AAAA,UAEE,qBAAuB,EAAA;AAAA;AACzB,OACF;AAEA,MAAO,OAAA,MAAA;AAAA,aACA,CAAG,EAAA;AACV,MAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,CAAO,KAAM,CAAA,CAAA,0BAAA,EAA6B,CAAC,CAAE,CAAA,CAAA;AAC1D,MAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA;AACtB;AACF,EAEQ,qBAAqB,IAA0C,EAAA;AACrE,IAAA,OAAOC,uBAAO,CAAA,MAAA;AAAA,MACZ,IAAA;AAAA,MACA,CAAA,CAAA,KAAK,GAAG,CAAE,CAAA,iBAAiB,IAAI,CAAE,CAAA,iBAAiB,CAAI,CAAA,EAAA,CAAA,CAAE,IAAI,CAAA;AAAA,KAC9D;AAAA;AACF;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,sBACZ,CAAA,QAAA,EACA,OAIe,EAAA;AACf,IAAA,MAAM,EAAK,GAAA,QAAA;AAGX,IAAM,MAAA,eAAA,GAAkB,IAAI,KAAc,EAAA;AAI1C,IAAA,KAAA,MAAW,EAAE,MAAA,EAAQ,WAAY,EAAA,IAAK,QAAQ,QAAU,EAAA;AACtD,MAAM,MAAA,SAAA,GAAYJ,gCAAmB,MAAM,CAAA;AAC3C,MAAM,MAAA,IAAA,GAAOK,wBAAmB,MAAM,CAAA;AAEtC,MAAM,MAAA,OAAA,GAAU,MAAMC,+CAAwB,CAAA;AAAA,QAC5C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,OAAS,EAAA;AACX,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA;AAGF,MAAM,MAAA,QAAA,GAAW,MAAMC,+CAAwB,CAAA;AAAA,QAC7C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA,WAAA;AAAA,QACA,MAAA,EAAQ,KAAK,OAAQ,CAAA;AAAA,OACtB,CAAA;AACD,MAAA,IAAI,QAAU,EAAA;AACZ,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA;AAMF,MAAM,MAAA,cAAA,GAAiB,MAAMC,iDAAyB,CAAA;AAAA,QACpD,EAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,cAAgB,EAAA;AAClB,QAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,UAClB,CAAkC,+BAAA,EAAA,SAAS,CAA0B,uBAAA,EAAA,cAAc,iBAAiB,WAAW,CAAA;AAAA,SACjH;AACA,QAAI,IAAA,IAAA,CAAK,OAAQ,CAAA,WAAA,IAAe,WAAa,EAAA;AAC3C,UAAA,MAAM,WAAwD,GAAA;AAAA,YAC5D,KAAO,EAAAC,iCAAA;AAAA,YACP,YAAc,EAAA;AAAA,cACZ,iBAAmB,EAAA,MAAA;AAAA,cACnB,SAAA;AAAA,cACA,cAAgB,EAAA,WAAA;AAAA,cAChB,mBAAqB,EAAA,cAAA;AAAA,cACrB,cAAgB,EAAAC,cAAA,CAAS,GAAI,EAAA,CAAE,KAAM;AAAA;AACvC,WACF;AACA,UAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,WAAa,EAAA,OAAA,CAAQ,WAAW,CAAA;AAAA;AACrD;AACF;AAIF,IAAM,MAAA,EAAA,CAAgC,0BAA0B,CAAA,CAC7D,QAAS,CAAA,EAAE,mBAAmB,OAAQ,CAAA,eAAA,EAAiB,CAAA,CACvD,MAAO,EAAA;AACV,IAAA,MAAM,EAAG,CAAA,WAAA;AAAA,MACP,0BAAA;AAAA,MACA,eAAA,CAAgB,IAAI,CAAc,SAAA,MAAA;AAAA,QAChC,mBAAmB,OAAQ,CAAA,eAAA;AAAA,QAC3B,iBAAmB,EAAA;AAAA,OACnB,CAAA,CAAA;AAAA,MACF;AAAA,KACF;AAAA;AAEJ;;;;"}
+{"version":3,"file":"DefaultProcessingDatabase.cjs.js","sources":["../../src/database/DefaultProcessingDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError } from '@backstage/errors';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { ProcessingIntervalFunction } from '../processing';\nimport { rethrowError, timestampToDateTime } from './conversion';\nimport { initDatabaseMetrics } from './metrics';\nimport {\n  DbRefreshKeysRow,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n} from './tables';\nimport {\n  GetProcessableEntitiesResult,\n  ListParentsOptions,\n  ListParentsResult,\n  ProcessingDatabase,\n  RefreshStateItem,\n  Transaction,\n  UpdateEntityCacheOptions,\n  UpdateProcessedEntityOptions,\n} from './types';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { generateStableHash, generateTargetKey } from './util';\nimport {\n  EventBroker,\n  EventParams,\n  EventsService,\n} from '@backstage/plugin-events-node';\nimport { DateTime } from 'luxon';\nimport { CATALOG_CONFLICTS_TOPIC } from '../constants';\nimport { CatalogConflictEventPayload } from '../catalog/types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProcessingDatabase implements ProcessingDatabase {\n  constructor(\n    private readonly options: {\n      database: Knex;\n      logger: LoggerService;\n      refreshInterval: ProcessingIntervalFunction;\n      eventBroker?: EventBroker | EventsService;\n    },\n  ) {\n    initDatabaseMetrics(options.database);\n  }\n\n  async updateProcessedEntity(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<{ previous: { relations: DbRelationsRow[] } }> {\n    const tx = txOpaque as Knex.Transaction;\n    const {\n      id,\n      processedEntity,\n      resultHash,\n      errors,\n      relations,\n      deferredEntities,\n      refreshKeys,\n      locationKey,\n    } = options;\n    const configClient = tx.client.config.client;\n    const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        processed_entity: JSON.stringify(processedEntity),\n        result_hash: resultHash,\n        errors,\n        location_key: locationKey,\n      })\n      .where('entity_id', id)\n      .andWhere(inner => {\n        if (!locationKey) {\n          return inner.whereNull('location_key');\n        }\n        return inner\n          .where('location_key', locationKey)\n          .orWhereNull('location_key');\n      });\n    if (refreshResult === 0) {\n      throw new ConflictError(\n        `Conflicting write of processing result for ${id} with location key '${locationKey}'`,\n      );\n    }\n    const sourceEntityRef = stringifyEntityRef(processedEntity);\n\n    // Schedule all deferred entities for future processing.\n    await this.addUnprocessedEntities(tx, {\n      entities: deferredEntities,\n      sourceEntityRef,\n    });\n\n    // Delete old relations\n    // NOTE(freben): knex implemented support for returning() on update queries for sqlite, but at the current time of writing (Sep 2022) not for delete() queries.\n    let previousRelationRows: DbRelationsRow[];\n    if (configClient.includes('sqlite3') || configClient.includes('mysql')) {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .select('*')\n        .where({ originating_entity_id: id });\n      await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete();\n    } else {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete()\n        .returning('*');\n    }\n\n    // Batch insert new relations\n    const relationRows: DbRelationsRow[] = relations.map(\n      ({ source, target, type }) => ({\n        originating_entity_id: id,\n        source_entity_ref: stringifyEntityRef(source),\n        target_entity_ref: stringifyEntityRef(target),\n        type,\n      }),\n    );\n\n    await tx.batchInsert(\n      'relations',\n      this.deduplicateRelations(relationRows),\n      BATCH_SIZE,\n    );\n\n    // Delete old refresh keys\n    await tx<DbRefreshKeysRow>('refresh_keys')\n      .where({ entity_id: id })\n      .delete();\n\n    // Insert the refresh keys for the processed entity\n    await tx.batchInsert(\n      'refresh_keys',\n      refreshKeys.map(k => ({\n        entity_id: id,\n        key: generateTargetKey(k.key),\n      })),\n      BATCH_SIZE,\n    );\n\n    return {\n      previous: {\n        relations: previousRelationRows,\n      },\n    };\n  }\n\n  async updateProcessedEntityErrors(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, errors, resultHash } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        errors,\n        result_hash: resultHash,\n      })\n      .where('entity_id', id);\n  }\n\n  async updateEntityCache(\n    txOpaque: Transaction,\n    options: UpdateEntityCacheOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, state } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({ cache: JSON.stringify(state ?? {}) })\n      .where('entity_id', id);\n  }\n\n  async getProcessableEntities(\n    maybeTx: Transaction | Knex,\n    request: { processBatchSize: number },\n  ): Promise<GetProcessableEntitiesResult> {\n    const knex = maybeTx as Knex.Transaction | Knex;\n\n    let itemsQuery = knex<DbRefreshStateRow>('refresh_state').select([\n      'entity_id',\n      'entity_ref',\n      'unprocessed_entity',\n      'result_hash',\n      'cache',\n      'errors',\n      'location_key',\n      'next_update_at',\n    ]);\n\n    // This avoids duplication of work because of race conditions and is\n    // also fast because locked rows are ignored rather than blocking.\n    // It's only available in MySQL and PostgreSQL\n    if (['mysql', 'mysql2', 'pg'].includes(knex.client.config.client)) {\n      itemsQuery = itemsQuery.forUpdate().skipLocked();\n    }\n\n    const items = await itemsQuery\n      .where('next_update_at', '<=', knex.fn.now())\n      .limit(request.processBatchSize)\n      .orderBy('next_update_at', 'asc');\n\n    const interval = this.options.refreshInterval();\n\n    const nextUpdateAt = (refreshInterval: number) => {\n      if (knex.client.config.client.includes('sqlite3')) {\n        return knex.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);\n      } else if (knex.client.config.client.includes('mysql')) {\n        return knex.raw(`now() + interval ${refreshInterval} second`);\n      }\n      return knex.raw(`now() + interval '${refreshInterval} seconds'`);\n    };\n\n    await knex<DbRefreshStateRow>('refresh_state')\n      .whereIn(\n        'entity_ref',\n        items.map(i => i.entity_ref),\n      )\n      .update({\n        next_update_at: nextUpdateAt(interval),\n      });\n\n    return {\n      items: items.map(\n        i =>\n          ({\n            id: i.entity_id,\n            entityRef: i.entity_ref,\n            unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity,\n            resultHash: i.result_hash || '',\n            nextUpdateAt: timestampToDateTime(i.next_update_at),\n            state: i.cache ? JSON.parse(i.cache) : undefined,\n            errors: i.errors,\n            locationKey: i.location_key,\n          } satisfies RefreshStateItem),\n      ),\n    };\n  }\n\n  async listParents(\n    txOpaque: Transaction,\n    options: ListParentsOptions,\n  ): Promise<ListParentsResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .where({ target_entity_ref: options.entityRef })\n      .select();\n\n    const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);\n\n    return { entityRefs };\n  }\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] {\n    return lodash.uniqBy(\n      rows,\n      r => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`,\n    );\n  }\n\n  /**\n   * Add a set of deferred entities for processing.\n   * The entities will be added at the front of the processing queue.\n   */\n  private async addUnprocessedEntities(\n    txOpaque: Transaction,\n    options: {\n      sourceEntityRef: string;\n      entities: DeferredEntity[];\n    },\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n\n    // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards\n    const stateReferences = new Array<string>();\n\n    // Upsert all of the unprocessed entities into the refresh_state table, by\n    // their entity ref.\n    for (const { entity, locationKey } of options.entities) {\n      const entityRef = stringifyEntityRef(entity);\n      const hash = generateStableHash(entity);\n\n      const updated = await updateUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n      });\n      if (updated) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      const inserted = await insertUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n        logger: this.options.logger,\n      });\n      if (inserted) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      // If the row can't be inserted, we have a conflict, but it could be either\n      // because of a conflicting locationKey or a race with another instance, so check\n      // whether the conflicting entity has the same entityRef but a different locationKey\n      const conflictingKey = await checkLocationKeyConflict({\n        tx,\n        entityRef,\n        locationKey,\n      });\n      if (conflictingKey) {\n        this.options.logger.warn(\n          `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n        );\n        if (this.options.eventBroker && locationKey) {\n          const eventParams: EventParams<CatalogConflictEventPayload> = {\n            topic: CATALOG_CONFLICTS_TOPIC,\n            eventPayload: {\n              unprocessedEntity: entity,\n              entityRef,\n              newLocationKey: locationKey,\n              existingLocationKey: conflictingKey,\n              lastConflictAt: DateTime.now().toISO()!,\n            },\n          };\n          await this.options.eventBroker?.publish(eventParams);\n        }\n      }\n    }\n\n    // Replace all references for the originating entity or source and then create new ones\n    await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n      .andWhere({ source_entity_ref: options.sourceEntityRef })\n      .delete();\n    await tx.batchInsert(\n      'refresh_state_references',\n      stateReferences.map(entityRef => ({\n        source_entity_ref: options.sourceEntityRef,\n        target_entity_ref: entityRef,\n      })),\n      BATCH_SIZE,\n    );\n  }\n}\n"],"names":["initDatabaseMetrics","errors","ConflictError","stringifyEntityRef","generateTargetKey","timestampToDateTime","rethrowError","lodash","generateStableHash","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","CATALOG_CONFLICTS_TOPIC","DateTime"],"mappings":";;;;;;;;;;;;;;;;;;AA0DA,MAAM,UAAa,GAAA,EAAA;AAEZ,MAAM,yBAAwD,CAAA;AAAA,EACnE,YACmB,OAMjB,EAAA;AANiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAOjB,IAAAA,2BAAA,CAAoB,QAAQ,QAAQ,CAAA;AAAA;AACtC,EAEA,MAAM,qBACJ,CAAA,QAAA,EACA,OACwD,EAAA;AACxD,IAAA,MAAM,EAAK,GAAA,QAAA;AACX,IAAM,MAAA;AAAA,MACJ,EAAA;AAAA,MACA,eAAA;AAAA,MACA,UAAA;AAAA,cACAC,QAAA;AAAA,MACA,SAAA;AAAA,MACA,gBAAA;AAAA,MACA,WAAA;AAAA,MACA;AAAA,KACE,GAAA,OAAA;AACJ,IAAM,MAAA,YAAA,GAAe,EAAG,CAAA,MAAA,CAAO,MAAO,CAAA,MAAA;AACtC,IAAA,MAAM,aAAgB,GAAA,MAAM,EAAsB,CAAA,eAAe,EAC9D,MAAO,CAAA;AAAA,MACN,gBAAA,EAAkB,IAAK,CAAA,SAAA,CAAU,eAAe,CAAA;AAAA,MAChD,WAAa,EAAA,UAAA;AAAA,cACbA,QAAA;AAAA,MACA,YAAc,EAAA;AAAA,KACf,CACA,CAAA,KAAA,CAAM,aAAa,EAAE,CAAA,CACrB,SAAS,CAAS,KAAA,KAAA;AACjB,MAAA,IAAI,CAAC,WAAa,EAAA;AAChB,QAAO,OAAA,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA;AAEvC,MAAA,OAAO,MACJ,KAAM,CAAA,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,KAC9B,CAAA;AACH,IAAA,IAAI,kBAAkB,CAAG,EAAA;AACvB,MAAA,MAAM,IAAIC,oBAAA;AAAA,QACR,CAAA,2CAAA,EAA8C,EAAE,CAAA,oBAAA,EAAuB,WAAW,CAAA,CAAA;AAAA,OACpF;AAAA;AAEF,IAAM,MAAA,eAAA,GAAkBC,gCAAmB,eAAe,CAAA;AAG1D,IAAM,MAAA,IAAA,CAAK,uBAAuB,EAAI,EAAA;AAAA,MACpC,QAAU,EAAA,gBAAA;AAAA,MACV;AAAA,KACD,CAAA;AAID,IAAI,IAAA,oBAAA;AACJ,IAAA,IAAI,aAAa,QAAS,CAAA,SAAS,KAAK,YAAa,CAAA,QAAA,CAAS,OAAO,CAAG,EAAA;AACtE,MAAuB,oBAAA,GAAA,MAAM,EAAmB,CAAA,WAAW,CACxD,CAAA,MAAA,CAAO,GAAG,CAAA,CACV,KAAM,CAAA,EAAE,qBAAuB,EAAA,EAAA,EAAI,CAAA;AACtC,MAAM,MAAA,EAAA,CAAmB,WAAW,CACjC,CAAA,KAAA,CAAM,EAAE,qBAAuB,EAAA,EAAA,EAAI,CAAA,CACnC,MAAO,EAAA;AAAA,KACL,MAAA;AACL,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,KAAM,CAAA,EAAE,qBAAuB,EAAA,EAAA,EAAI,CAAA,CACnC,MAAO,EAAA,CACP,UAAU,GAAG,CAAA;AAAA;AAIlB,IAAA,MAAM,eAAiC,SAAU,CAAA,GAAA;AAAA,MAC/C,CAAC,EAAE,MAAQ,EAAA,MAAA,EAAQ,MAAY,MAAA;AAAA,QAC7B,qBAAuB,EAAA,EAAA;AAAA,QACvB,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C;AAAA,OACF;AAAA,KACF;AAEA,IAAA,MAAM,EAAG,CAAA,WAAA;AAAA,MACP,WAAA;AAAA,MACA,IAAA,CAAK,qBAAqB,YAAY,CAAA;AAAA,MACtC;AAAA,KACF;AAGA,IAAM,MAAA,EAAA,CAAqB,cAAc,CACtC,CAAA,KAAA,CAAM,EAAE,SAAW,EAAA,EAAA,EAAI,CAAA,CACvB,MAAO,EAAA;AAGV,IAAA,MAAM,EAAG,CAAA,WAAA;AAAA,MACP,cAAA;AAAA,MACA,WAAA,CAAY,IAAI,CAAM,CAAA,MAAA;AAAA,QACpB,SAAW,EAAA,EAAA;AAAA,QACX,GAAA,EAAKC,sBAAkB,CAAA,CAAA,CAAE,GAAG;AAAA,OAC5B,CAAA,CAAA;AAAA,MACF;AAAA,KACF;AAEA,IAAO,OAAA;AAAA,MACL,QAAU,EAAA;AAAA,QACR,SAAW,EAAA;AAAA;AACb,KACF;AAAA;AACF,EAEA,MAAM,2BACJ,CAAA,QAAA,EACA,OACe,EAAA;AACf,IAAA,MAAM,EAAK,GAAA,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,MAAQ,EAAA,UAAA,EAAe,GAAA,OAAA;AAEnC,IAAM,MAAA,EAAA,CAAsB,eAAe,CAAA,CACxC,MAAO,CAAA;AAAA,MACN,MAAA;AAAA,MACA,WAAa,EAAA;AAAA,KACd,CAAA,CACA,KAAM,CAAA,WAAA,EAAa,EAAE,CAAA;AAAA;AAC1B,EAEA,MAAM,iBACJ,CAAA,QAAA,EACA,OACe,EAAA;AACf,IAAA,MAAM,EAAK,GAAA,QAAA;AACX,IAAM,MAAA,EAAE,EAAI,EAAA,KAAA,EAAU,GAAA,OAAA;AAEtB,IAAA,MAAM,GAAsB,eAAe,CAAA,CACxC,MAAO,CAAA,EAAE,OAAO,IAAK,CAAA,SAAA,CAAU,KAAS,IAAA,EAAE,CAAE,EAAC,CAC7C,CAAA,KAAA,CAAM,aAAa,EAAE,CAAA;AAAA;AAC1B,EAEA,MAAM,sBACJ,CAAA,OAAA,EACA,OACuC,EAAA;AACvC,IAAA,MAAM,IAAO,GAAA,OAAA;AAEb,IAAA,IAAI,UAAa,GAAA,IAAA,CAAwB,eAAe,CAAA,CAAE,MAAO,CAAA;AAAA,MAC/D,WAAA;AAAA,MACA,YAAA;AAAA,MACA,oBAAA;AAAA,MACA,aAAA;AAAA,MACA,OAAA;AAAA,MACA,QAAA;AAAA,MACA,cAAA;AAAA,MACA;AAAA,KACD,CAAA;AAKD,IAAI,IAAA,CAAC,OAAS,EAAA,QAAA,EAAU,IAAI,CAAA,CAAE,SAAS,IAAK,CAAA,MAAA,CAAO,MAAO,CAAA,MAAM,CAAG,EAAA;AACjE,MAAa,UAAA,GAAA,UAAA,CAAW,SAAU,EAAA,CAAE,UAAW,EAAA;AAAA;AAGjD,IAAA,MAAM,QAAQ,MAAM,UAAA,CACjB,KAAM,CAAA,gBAAA,EAAkB,MAAM,IAAK,CAAA,EAAA,CAAG,GAAI,EAAC,EAC3C,KAAM,CAAA,OAAA,CAAQ,gBAAgB,CAC9B,CAAA,OAAA,CAAQ,kBAAkB,KAAK,CAAA;AAElC,IAAM,MAAA,QAAA,GAAW,IAAK,CAAA,OAAA,CAAQ,eAAgB,EAAA;AAE9C,IAAM,MAAA,YAAA,GAAe,CAAC,eAA4B,KAAA;AAChD,MAAA,IAAI,KAAK,MAAO,CAAA,MAAA,CAAO,MAAO,CAAA,QAAA,CAAS,SAAS,CAAG,EAAA;AACjD,QAAA,OAAO,KAAK,GAAI,CAAA,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAG,EAAA,eAAe,UAAU,CAAC,CAAA;AAAA,iBAC3D,IAAK,CAAA,MAAA,CAAO,OAAO,MAAO,CAAA,QAAA,CAAS,OAAO,CAAG,EAAA;AACtD,QAAA,OAAO,IAAK,CAAA,GAAA,CAAI,CAAoB,iBAAA,EAAA,eAAe,CAAS,OAAA,CAAA,CAAA;AAAA;AAE9D,MAAA,OAAO,IAAK,CAAA,GAAA,CAAI,CAAqB,kBAAA,EAAA,eAAe,CAAW,SAAA,CAAA,CAAA;AAAA,KACjE;AAEA,IAAM,MAAA,IAAA,CAAwB,eAAe,CAC1C,CAAA,OAAA;AAAA,MACC,YAAA;AAAA,MACA,KAAM,CAAA,GAAA,CAAI,CAAK,CAAA,KAAA,CAAA,CAAE,UAAU;AAAA,MAE5B,MAAO,CAAA;AAAA,MACN,cAAA,EAAgB,aAAa,QAAQ;AAAA,KACtC,CAAA;AAEH,IAAO,OAAA;AAAA,MACL,OAAO,KAAM,CAAA,GAAA;AAAA,QACX,CACG,CAAA,MAAA;AAAA,UACC,IAAI,CAAE,CAAA,SAAA;AAAA,UACN,WAAW,CAAE,CAAA,UAAA;AAAA,UACb,iBAAmB,EAAA,IAAA,CAAK,KAAM,CAAA,CAAA,CAAE,kBAAkB,CAAA;AAAA,UAClD,UAAA,EAAY,EAAE,WAAe,IAAA,EAAA;AAAA,UAC7B,YAAA,EAAcC,8BAAoB,CAAA,CAAA,CAAE,cAAc,CAAA;AAAA,UAClD,OAAO,CAAE,CAAA,KAAA,GAAQ,KAAK,KAAM,CAAA,CAAA,CAAE,KAAK,CAAI,GAAA,KAAA,CAAA;AAAA,UACvC,QAAQ,CAAE,CAAA,MAAA;AAAA,UACV,aAAa,CAAE,CAAA;AAAA,SACjB;AAAA;AACJ,KACF;AAAA;AACF,EAEA,MAAM,WACJ,CAAA,QAAA,EACA,OAC4B,EAAA;AAC5B,IAAA,MAAM,EAAK,GAAA,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,KACF,CACG,MAAM,EAAE,iBAAA,EAAmB,QAAQ,SAAU,EAAC,EAC9C,MAAO,EAAA;AAEV,IAAM,MAAA,UAAA,GAAa,KAAK,GAAI,CAAA,CAAA,CAAA,KAAK,EAAE,iBAAkB,CAAA,CAAE,OAAO,OAAO,CAAA;AAErE,IAAA,OAAO,EAAE,UAAW,EAAA;AAAA;AACtB,EAEA,MAAM,YAAe,EAAiD,EAAA;AACpE,IAAI,IAAA;AACF,MAAA,IAAI,MAAwB,GAAA,KAAA,CAAA;AAE5B,MAAM,MAAA,IAAA,CAAK,QAAQ,QAAS,CAAA,WAAA;AAAA,QAC1B,OAAM,EAAM,KAAA;AAGV,UAAS,MAAA,GAAA,MAAM,GAAG,EAAE,CAAA;AAAA,SACtB;AAAA,QACA;AAAA;AAAA,UAEE,qBAAuB,EAAA;AAAA;AACzB,OACF;AAEA,MAAO,OAAA,MAAA;AAAA,aACA,CAAG,EAAA;AACV,MAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,CAAO,KAAM,CAAA,CAAA,0BAAA,EAA6B,CAAC,CAAE,CAAA,CAAA;AAC1D,MAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA;AACtB;AACF,EAEQ,qBAAqB,IAA0C,EAAA;AACrE,IAAA,OAAOC,uBAAO,CAAA,MAAA;AAAA,MACZ,IAAA;AAAA,MACA,CAAA,CAAA,KAAK,GAAG,CAAE,CAAA,iBAAiB,IAAI,CAAE,CAAA,iBAAiB,CAAI,CAAA,EAAA,CAAA,CAAE,IAAI,CAAA;AAAA,KAC9D;AAAA;AACF;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,sBACZ,CAAA,QAAA,EACA,OAIe,EAAA;AACf,IAAA,MAAM,EAAK,GAAA,QAAA;AAGX,IAAM,MAAA,eAAA,GAAkB,IAAI,KAAc,EAAA;AAI1C,IAAA,KAAA,MAAW,EAAE,MAAA,EAAQ,WAAY,EAAA,IAAK,QAAQ,QAAU,EAAA;AACtD,MAAM,MAAA,SAAA,GAAYJ,gCAAmB,MAAM,CAAA;AAC3C,MAAM,MAAA,IAAA,GAAOK,wBAAmB,MAAM,CAAA;AAEtC,MAAM,MAAA,OAAA,GAAU,MAAMC,+CAAwB,CAAA;AAAA,QAC5C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,OAAS,EAAA;AACX,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA;AAGF,MAAM,MAAA,QAAA,GAAW,MAAMC,+CAAwB,CAAA;AAAA,QAC7C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA,WAAA;AAAA,QACA,MAAA,EAAQ,KAAK,OAAQ,CAAA;AAAA,OACtB,CAAA;AACD,MAAA,IAAI,QAAU,EAAA;AACZ,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA;AAMF,MAAM,MAAA,cAAA,GAAiB,MAAMC,iDAAyB,CAAA;AAAA,QACpD,EAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,cAAgB,EAAA;AAClB,QAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,UAClB,CAAkC,+BAAA,EAAA,SAAS,CAA0B,uBAAA,EAAA,cAAc,iBAAiB,WAAW,CAAA;AAAA,SACjH;AACA,QAAI,IAAA,IAAA,CAAK,OAAQ,CAAA,WAAA,IAAe,WAAa,EAAA;AAC3C,UAAA,MAAM,WAAwD,GAAA;AAAA,YAC5D,KAAO,EAAAC,iCAAA;AAAA,YACP,YAAc,EAAA;AAAA,cACZ,iBAAmB,EAAA,MAAA;AAAA,cACnB,SAAA;AAAA,cACA,cAAgB,EAAA,WAAA;AAAA,cAChB,mBAAqB,EAAA,cAAA;AAAA,cACrB,cAAgB,EAAAC,cAAA,CAAS,GAAI,EAAA,CAAE,KAAM;AAAA;AACvC,WACF;AACA,UAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,WAAa,EAAA,OAAA,CAAQ,WAAW,CAAA;AAAA;AACrD;AACF;AAIF,IAAM,MAAA,EAAA,CAAgC,0BAA0B,CAAA,CAC7D,QAAS,CAAA,EAAE,mBAAmB,OAAQ,CAAA,eAAA,EAAiB,CAAA,CACvD,MAAO,EAAA;AACV,IAAA,MAAM,EAAG,CAAA,WAAA;AAAA,MACP,0BAAA;AAAA,MACA,eAAA,CAAgB,IAAI,CAAc,SAAA,MAAA;AAAA,QAChC,mBAAmB,OAAQ,CAAA,eAAA;AAAA,QAC3B,iBAAmB,EAAA;AAAA,OACnB,CAAA,CAAA;AAAA,MACF;AAAA,KACF;AAAA;AAEJ;;;;"}

package/dist/database/DefaultProviderDatabase.cjs.js

@@ -145,14 +145,36 @@ class DefaultProviderDatabase {
   }
   async createDelta(tx, options) {
     if (options.type === "delta") {
-      return {
-        toAdd: [],
-        toUpsert: options.added.map((e) => ({
-          deferred: e,
-          hash: util.generateStableHash(e.entity)
-        })),
-        toRemove: options.removed.map((e) => e.entityRef)
-      };
+      const toAdd2 = new Array();
+      const toUpsert2 = new Array();
+      const toRemove2 = options.removed.map((e) => e.entityRef);
+      for (const chunk of lodash__default.default.chunk(options.added, 1e3)) {
+        const entityRefs = chunk.map((e) => catalogModel.stringifyEntityRef(e.entity));
+        const rows = await tx("refresh_state").select(["entity_ref", "unprocessed_hash", "location_key"]).whereIn("entity_ref", entityRefs);
+        const oldStates = new Map(
+          rows.map((row) => [
+            row.entity_ref,
+            {
+              unprocessed_hash: row.unprocessed_hash,
+              location_key: row.location_key
+            }
+          ])
+        );
+        chunk.forEach((deferred, i) => {
+          const entityRef = entityRefs[i];
+          const newHash = util.generateStableHash(deferred.entity);
+          const oldState = oldStates.get(entityRef);
+          if (oldState === void 0) {
+            toAdd2.push({ deferred, hash: newHash });
+          } else if ((deferred.locationKey ?? null) !== (oldState.location_key ?? null)) {
+            toRemove2.push(entityRef);
+            toAdd2.push({ deferred, hash: newHash });
+          } else if (newHash !== oldState.unprocessed_hash) {
+            toUpsert2.push({ deferred, hash: newHash });
+          }
+        });
+      }
+      return { toAdd: toAdd2, toUpsert: toUpsert2, toRemove: toRemove2 };
     }
     const oldRefs = await tx(
       "refresh_state_references"
@@ -186,7 +208,7 @@ class DefaultProviderDatabase {
       const upsertItem = { deferred: item.deferred, hash: item.hash };
       if (!oldRef) {
         toAdd.push(upsertItem);
-      } else if ((oldRef?.locationKey ?? void 0) !== (item.deferred.locationKey ?? void 0)) {
+      } else if ((oldRef.locationKey ?? void 0) !== (item.deferred.locationKey ?? void 0)) {
         toRemove.push(item.ref);
         toAdd.push(upsertItem);
       } else if (oldRef.oldEntityHash !== item.hash) {

package/dist/database/DefaultProviderDatabase.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultProviderDatabase.cjs.js","sources":["../../src/database/DefaultProviderDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { stringifyEntityRef } from '@backstage/catalog-model';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { v4 as uuid } from 'uuid';\nimport { rethrowError } from './conversion';\nimport { deleteWithEagerPruningOfChildren } from './operations/provider/deleteWithEagerPruningOfChildren';\nimport { refreshByRefreshKeys } from './operations/provider/refreshByRefreshKeys';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { DbRefreshStateReferencesRow, DbRefreshStateRow } from './tables';\nimport {\n  ProviderDatabase,\n  RefreshByKeyOptions,\n  ReplaceUnprocessedEntitiesOptions,\n  Transaction,\n} from './types';\nimport { generateStableHash } from './util';\nimport {\n  LoggerService,\n  isDatabaseConflictError,\n} from '@backstage/backend-plugin-api';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProviderDatabase implements ProviderDatabase {\n  constructor(\n    private readonly options: {\n      database: Knex;\n      logger: LoggerService;\n    },\n  ) {}\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the\n          // transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  async replaceUnprocessedEntities(\n    txOpaque: Transaction,\n    options: ReplaceUnprocessedEntitiesOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { toAdd, toUpsert, toRemove } = await this.createDelta(tx, options);\n\n    if (toRemove.length) {\n      const removedCount = await deleteWithEagerPruningOfChildren({\n        knex: tx,\n        entityRefs: toRemove,\n        sourceKey: options.sourceKey,\n      });\n      this.options.logger.debug(\n        `removed, ${removedCount} entities: ${JSON.stringify(toRemove)}`,\n      );\n    }\n\n    if (toAdd.length) {\n      // The reason for this chunking, rather than just massively batch\n      // inserting the entire payload, is that we fall back to the individual\n      // upsert mechanism below on conflicts. That path is massively slower than\n      // the fast batch path, so we don't want to end up accidentally having to\n      // for example item-by-item upsert tens of thousands of entities in a\n      // large initial delivery dump. The implication is that the size of these\n      // chunks needs to weigh the benefit of fast successful inserts, against\n      // the drawback of super slow but more rare fallbacks. There's quickly\n      // diminishing returns though with turning up this value way high.\n      for (const chunk of lodash.chunk(toAdd, 50)) {\n        try {\n          await tx.batchInsert(\n            'refresh_state',\n            chunk.map(item => ({\n              entity_id: uuid(),\n              entity_ref: stringifyEntityRef(item.deferred.entity),\n              unprocessed_entity: JSON.stringify(item.deferred.entity),\n              unprocessed_hash: item.hash,\n              errors: '',\n              location_key: item.deferred.locationKey,\n              next_update_at: tx.fn.now(),\n              last_discovery_at: tx.fn.now(),\n            })),\n            BATCH_SIZE,\n          );\n          await tx.batchInsert(\n            'refresh_state_references',\n            chunk.map(item => ({\n              source_key: options.sourceKey,\n              target_entity_ref: stringifyEntityRef(item.deferred.entity),\n            })),\n            BATCH_SIZE,\n          );\n        } catch (error) {\n          if (!isDatabaseConflictError(error)) {\n            throw error;\n          } else {\n            this.options.logger.debug(\n              `Fast insert path failed, falling back to slow path, ${error}`,\n            );\n            toUpsert.push(...chunk);\n          }\n        }\n      }\n    }\n\n    if (toUpsert.length) {\n      for (const {\n        deferred: { entity, locationKey },\n        hash,\n      } of toUpsert) {\n        const entityRef = stringifyEntityRef(entity);\n\n        try {\n          let ok = await updateUnprocessedEntity({\n            tx,\n            entity,\n            hash,\n            locationKey,\n          });\n          if (!ok) {\n            ok = await insertUnprocessedEntity({\n              tx,\n              entity,\n              hash,\n              locationKey,\n              logger: this.options.logger,\n            });\n          }\n\n          await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n            .where('target_entity_ref', entityRef)\n            .andWhere({ source_key: options.sourceKey })\n            .delete();\n\n          if (ok) {\n            await tx<DbRefreshStateReferencesRow>(\n              'refresh_state_references',\n            ).insert({\n              source_key: options.sourceKey,\n              target_entity_ref: entityRef,\n            });\n          } else {\n            const conflictingKey = await checkLocationKeyConflict({\n              tx,\n              entityRef,\n              locationKey,\n            });\n            if (conflictingKey) {\n              this.options.logger.warn(\n                `Source ${options.sourceKey} detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n              );\n            }\n          }\n        } catch (error) {\n          this.options.logger.error(\n            `Failed to add '${entityRef}' from source '${options.sourceKey}', ${error}`,\n          );\n        }\n      }\n    }\n  }\n\n  async refreshByRefreshKeys(\n    txOpaque: Transaction,\n    options: RefreshByKeyOptions,\n  ) {\n    const tx = txOpaque as Knex.Transaction;\n    await refreshByRefreshKeys({ tx, keys: options.keys });\n  }\n\n  private async createDelta(\n    tx: Knex.Transaction,\n    options: ReplaceUnprocessedEntitiesOptions,\n  ): Promise<{\n    toAdd: { deferred: DeferredEntity; hash: string }[];\n    toUpsert: { deferred: DeferredEntity; hash: string }[];\n    toRemove: string[];\n  }> {\n    if (options.type === 'delta') {\n      return {\n        toAdd: [],\n        toUpsert: options.added.map(e => ({\n          deferred: e,\n          hash: generateStableHash(e.entity),\n        })),\n        toRemove: options.removed.map(e => e.entityRef),\n      };\n    }\n\n    // Grab all of the existing references from the same source, and their locationKeys as well\n    const oldRefs = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .leftJoin<DbRefreshStateRow>('refresh_state', {\n        target_entity_ref: 'entity_ref',\n      })\n      .where({ source_key: options.sourceKey })\n      .select({\n        target_entity_ref: 'refresh_state_references.target_entity_ref',\n        location_key: 'refresh_state.location_key',\n        unprocessed_hash: 'refresh_state.unprocessed_hash',\n      });\n\n    const items = options.items.map(deferred => ({\n      deferred,\n      ref: stringifyEntityRef(deferred.entity),\n      hash: generateStableHash(deferred.entity),\n    }));\n\n    const oldRefsSet = new Map(\n      oldRefs.map(r => [\n        r.target_entity_ref,\n        {\n          locationKey: r.location_key,\n          oldEntityHash: r.unprocessed_hash,\n        },\n      ]),\n    );\n    const newRefsSet = new Set(items.map(item => item.ref));\n\n    const toAdd = new Array<{ deferred: DeferredEntity; hash: string }>();\n    const toUpsert = new Array<{ deferred: DeferredEntity; hash: string }>();\n    const toRemove = oldRefs\n      .map(row => row.target_entity_ref)\n      .filter(ref => !newRefsSet.has(ref));\n\n    for (const item of items) {\n      const oldRef = oldRefsSet.get(item.ref);\n      const upsertItem = { deferred: item.deferred, hash: item.hash };\n      if (!oldRef) {\n        // Add any entity that does not exist in the database\n        toAdd.push(upsertItem);\n      } else if (\n        (oldRef?.locationKey ?? undefined) !==\n        (item.deferred.locationKey ?? undefined)\n      ) {\n        // Remove and then re-add any entity that exists, but with a different location key\n        toRemove.push(item.ref);\n        toAdd.push(upsertItem);\n      } else if (oldRef.oldEntityHash !== item.hash) {\n        // Entities with modifications should be pushed through too\n        toUpsert.push(upsertItem);\n      }\n    }\n\n    return { toAdd, toUpsert, toRemove };\n  }\n}\n"],"names":["rethrowError","deleteWithEagerPruningOfChildren","lodash","uuid","stringifyEntityRef","isDatabaseConflictError","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","refreshByRefreshKeys","generateStableHash"],"mappings":";;;;;;;;;;;;;;;;;;AA4CA,MAAM,UAAa,GAAA,EAAA;AAEZ,MAAM,uBAAoD,CAAA;AAAA,EAC/D,YACmB,OAIjB,EAAA;AAJiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA;AAIhB,EAEH,MAAM,YAAe,EAAiD,EAAA;AACpE,IAAI,IAAA;AACF,MAAA,IAAI,MAAwB,GAAA,KAAA,CAAA;AAC5B,MAAM,MAAA,IAAA,CAAK,QAAQ,QAAS,CAAA,WAAA;AAAA,QAC1B,OAAM,EAAM,KAAA;AAIV,UAAS,MAAA,GAAA,MAAM,GAAG,EAAE,CAAA;AAAA,SACtB;AAAA,QACA;AAAA;AAAA,UAEE,qBAAuB,EAAA;AAAA;AACzB,OACF;AACA,MAAO,OAAA,MAAA;AAAA,aACA,CAAG,EAAA;AACV,MAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,CAAO,KAAM,CAAA,CAAA,0BAAA,EAA6B,CAAC,CAAE,CAAA,CAAA;AAC1D,MAAA,MAAMA,wBAAa,CAAC,CAAA;AAAA;AACtB;AACF,EAEA,MAAM,0BACJ,CAAA,QAAA,EACA,OACe,EAAA;AACf,IAAA,MAAM,EAAK,GAAA,QAAA;AACX,IAAM,MAAA,EAAE,OAAO,QAAU,EAAA,QAAA,KAAa,MAAM,IAAA,CAAK,WAAY,CAAA,EAAA,EAAI,OAAO,CAAA;AAExE,IAAA,IAAI,SAAS,MAAQ,EAAA;AACnB,MAAM,MAAA,YAAA,GAAe,MAAMC,iEAAiC,CAAA;AAAA,QAC1D,IAAM,EAAA,EAAA;AAAA,QACN,UAAY,EAAA,QAAA;AAAA,QACZ,WAAW,OAAQ,CAAA;AAAA,OACpB,CAAA;AACD,MAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,QAClB,YAAY,YAAY,CAAA,WAAA,EAAc,IAAK,CAAA,SAAA,CAAU,QAAQ,CAAC,CAAA;AAAA,OAChE;AAAA;AAGF,IAAA,IAAI,MAAM,MAAQ,EAAA;AAUhB,MAAA,KAAA,MAAW,KAAS,IAAAC,uBAAA,CAAO,KAAM,CAAA,KAAA,EAAO,EAAE,CAAG,EAAA;AAC3C,QAAI,IAAA;AACF,UAAA,MAAM,EAAG,CAAA,WAAA;AAAA,YACP,eAAA;AAAA,YACA,KAAA,CAAM,IAAI,CAAS,IAAA,MAAA;AAAA,cACjB,WAAWC,OAAK,EAAA;AAAA,cAChB,UAAY,EAAAC,+BAAA,CAAmB,IAAK,CAAA,QAAA,CAAS,MAAM,CAAA;AAAA,cACnD,kBAAoB,EAAA,IAAA,CAAK,SAAU,CAAA,IAAA,CAAK,SAAS,MAAM,CAAA;AAAA,cACvD,kBAAkB,IAAK,CAAA,IAAA;AAAA,cACvB,MAAQ,EAAA,EAAA;AAAA,cACR,YAAA,EAAc,KAAK,QAAS,CAAA,WAAA;AAAA,cAC5B,cAAA,EAAgB,EAAG,CAAA,EAAA,CAAG,GAAI,EAAA;AAAA,cAC1B,iBAAA,EAAmB,EAAG,CAAA,EAAA,CAAG,GAAI;AAAA,aAC7B,CAAA,CAAA;AAAA,YACF;AAAA,WACF;AACA,UAAA,MAAM,EAAG,CAAA,WAAA;AAAA,YACP,0BAAA;AAAA,YACA,KAAA,CAAM,IAAI,CAAS,IAAA,MAAA;AAAA,cACjB,YAAY,OAAQ,CAAA,SAAA;AAAA,cACpB,iBAAmB,EAAAA,+BAAA,CAAmB,IAAK,CAAA,QAAA,CAAS,MAAM;AAAA,aAC1D,CAAA,CAAA;AAAA,YACF;AAAA,WACF;AAAA,iBACO,KAAO,EAAA;AACd,UAAI,IAAA,CAACC,wCAAwB,CAAA,KAAK,CAAG,EAAA;AACnC,YAAM,MAAA,KAAA;AAAA,WACD,MAAA;AACL,YAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,cAClB,uDAAuD,KAAK,CAAA;AAAA,aAC9D;AACA,YAAS,QAAA,CAAA,IAAA,CAAK,GAAG,KAAK,CAAA;AAAA;AACxB;AACF;AACF;AAGF,IAAA,IAAI,SAAS,MAAQ,EAAA;AACnB,MAAW,KAAA,MAAA;AAAA,QACT,QAAA,EAAU,EAAE,MAAA,EAAQ,WAAY,EAAA;AAAA,QAChC;AAAA,WACG,QAAU,EAAA;AACb,QAAM,MAAA,SAAA,GAAYD,gCAAmB,MAAM,CAAA;AAE3C,QAAI,IAAA;AACF,UAAI,IAAA,EAAA,GAAK,MAAME,+CAAwB,CAAA;AAAA,YACrC,EAAA;AAAA,YACA,MAAA;AAAA,YACA,IAAA;AAAA,YACA;AAAA,WACD,CAAA;AACD,UAAA,IAAI,CAAC,EAAI,EAAA;AACP,YAAA,EAAA,GAAK,MAAMC,+CAAwB,CAAA;AAAA,cACjC,EAAA;AAAA,cACA,MAAA;AAAA,cACA,IAAA;AAAA,cACA,WAAA;AAAA,cACA,MAAA,EAAQ,KAAK,OAAQ,CAAA;AAAA,aACtB,CAAA;AAAA;AAGH,UAAA,MAAM,EAAgC,CAAA,0BAA0B,CAC7D,CAAA,KAAA,CAAM,qBAAqB,SAAS,CAAA,CACpC,QAAS,CAAA,EAAE,UAAY,EAAA,OAAA,CAAQ,SAAU,EAAC,EAC1C,MAAO,EAAA;AAEV,UAAA,IAAI,EAAI,EAAA;AACN,YAAM,MAAA,EAAA;AAAA,cACJ;AAAA,cACA,MAAO,CAAA;AAAA,cACP,YAAY,OAAQ,CAAA,SAAA;AAAA,cACpB,iBAAmB,EAAA;AAAA,aACpB,CAAA;AAAA,WACI,MAAA;AACL,YAAM,MAAA,cAAA,GAAiB,MAAMC,iDAAyB,CAAA;AAAA,cACpD,EAAA;AAAA,cACA,SAAA;AAAA,cACA;AAAA,aACD,CAAA;AACD,YAAA,IAAI,cAAgB,EAAA;AAClB,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,gBAClB,CAAA,OAAA,EAAU,QAAQ,SAAS,CAAA,gCAAA,EAAmC,SAAS,CAA0B,uBAAA,EAAA,cAAc,iBAAiB,WAAW,CAAA;AAAA,eAC7I;AAAA;AACF;AACF,iBACO,KAAO,EAAA;AACd,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,YAClB,kBAAkB,SAAS,CAAA,eAAA,EAAkB,OAAQ,CAAA,SAAS,MAAM,KAAK,CAAA;AAAA,WAC3E;AAAA;AACF;AACF;AACF;AACF,EAEA,MAAM,oBACJ,CAAA,QAAA,EACA,OACA,EAAA;AACA,IAAA,MAAM,EAAK,GAAA,QAAA;AACX,IAAA,MAAMC,0CAAqB,EAAE,EAAA,EAAI,IAAM,EAAA,OAAA,CAAQ,MAAM,CAAA;AAAA;AACvD,EAEA,MAAc,WACZ,CAAA,EAAA,EACA,OAKC,EAAA;AACD,IAAI,IAAA,OAAA,CAAQ,SAAS,OAAS,EAAA;AAC5B,MAAO,OAAA;AAAA,QACL,OAAO,EAAC;AAAA,QACR,QAAU,EAAA,OAAA,CAAQ,KAAM,CAAA,GAAA,CAAI,CAAM,CAAA,MAAA;AAAA,UAChC,QAAU,EAAA,CAAA;AAAA,UACV,IAAA,EAAMC,uBAAmB,CAAA,CAAA,CAAE,MAAM;AAAA,SACjC,CAAA,CAAA;AAAA,QACF,UAAU,OAAQ,CAAA,OAAA,CAAQ,GAAI,CAAA,CAAA,CAAA,KAAK,EAAE,SAAS;AAAA,OAChD;AAAA;AAIF,IAAA,MAAM,UAAU,MAAM,EAAA;AAAA,MACpB;AAAA,KACF,CACG,SAA4B,eAAiB,EAAA;AAAA,MAC5C,iBAAmB,EAAA;AAAA,KACpB,EACA,KAAM,CAAA,EAAE,YAAY,OAAQ,CAAA,SAAA,EAAW,CAAA,CACvC,MAAO,CAAA;AAAA,MACN,iBAAmB,EAAA,4CAAA;AAAA,MACnB,YAAc,EAAA,4BAAA;AAAA,MACd,gBAAkB,EAAA;AAAA,KACnB,CAAA;AAEH,IAAA,MAAM,KAAQ,GAAA,OAAA,CAAQ,KAAM,CAAA,GAAA,CAAI,CAAa,QAAA,MAAA;AAAA,MAC3C,QAAA;AAAA,MACA,GAAA,EAAKN,+BAAmB,CAAA,QAAA,CAAS,MAAM,CAAA;AAAA,MACvC,IAAA,EAAMM,uBAAmB,CAAA,QAAA,CAAS,MAAM;AAAA,KACxC,CAAA,CAAA;AAEF,IAAA,MAAM,aAAa,IAAI,GAAA;AAAA,MACrB,OAAA,CAAQ,IAAI,CAAK,CAAA,KAAA;AAAA,QACf,CAAE,CAAA,iBAAA;AAAA,QACF;AAAA,UACE,aAAa,CAAE,CAAA,YAAA;AAAA,UACf,eAAe,CAAE,CAAA;AAAA;AACnB,OACD;AAAA,KACH;AACA,IAAM,MAAA,UAAA,GAAa,IAAI,GAAI,CAAA,KAAA,CAAM,IAAI,CAAQ,IAAA,KAAA,IAAA,CAAK,GAAG,CAAC,CAAA;AAEtD,IAAM,MAAA,KAAA,GAAQ,IAAI,KAAkD,EAAA;AACpE,IAAM,MAAA,QAAA,GAAW,IAAI,KAAkD,EAAA;AACvE,IAAA,MAAM,QAAW,GAAA,OAAA,CACd,GAAI,CAAA,CAAA,GAAA,KAAO,GAAI,CAAA,iBAAiB,CAChC,CAAA,MAAA,CAAO,CAAO,GAAA,KAAA,CAAC,UAAW,CAAA,GAAA,CAAI,GAAG,CAAC,CAAA;AAErC,IAAA,KAAA,MAAW,QAAQ,KAAO,EAAA;AACxB,MAAA,MAAM,MAAS,GAAA,UAAA,CAAW,GAAI,CAAA,IAAA,CAAK,GAAG,CAAA;AACtC,MAAA,MAAM,aAAa,EAAE,QAAA,EAAU,KAAK,QAAU,EAAA,IAAA,EAAM,KAAK,IAAK,EAAA;AAC9D,MAAA,IAAI,CAAC,MAAQ,EAAA;AAEX,QAAA,KAAA,CAAM,KAAK,UAAU,CAAA;AAAA,kBAEpB,MAAQ,EAAA,WAAA,IAAe,aACvB,IAAK,CAAA,QAAA,CAAS,eAAe,KAC9B,CAAA,CAAA,EAAA;AAEA,QAAS,QAAA,CAAA,IAAA,CAAK,KAAK,GAAG,CAAA;AACtB,QAAA,KAAA,CAAM,KAAK,UAAU,CAAA;AAAA,OACZ,MAAA,IAAA,MAAA,CAAO,aAAkB,KAAA,IAAA,CAAK,IAAM,EAAA;AAE7C,QAAA,QAAA,CAAS,KAAK,UAAU,CAAA;AAAA;AAC1B;AAGF,IAAO,OAAA,EAAE,KAAO,EAAA,QAAA,EAAU,QAAS,EAAA;AAAA;AAEvC;;;;"}
+{"version":3,"file":"DefaultProviderDatabase.cjs.js","sources":["../../src/database/DefaultProviderDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { stringifyEntityRef } from '@backstage/catalog-model';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { v4 as uuid } from 'uuid';\nimport { rethrowError } from './conversion';\nimport { deleteWithEagerPruningOfChildren } from './operations/provider/deleteWithEagerPruningOfChildren';\nimport { refreshByRefreshKeys } from './operations/provider/refreshByRefreshKeys';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { DbRefreshStateReferencesRow, DbRefreshStateRow } from './tables';\nimport {\n  ProviderDatabase,\n  RefreshByKeyOptions,\n  ReplaceUnprocessedEntitiesOptions,\n  Transaction,\n} from './types';\nimport { generateStableHash } from './util';\nimport {\n  LoggerService,\n  isDatabaseConflictError,\n} from '@backstage/backend-plugin-api';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProviderDatabase implements ProviderDatabase {\n  constructor(\n    private readonly options: {\n      database: Knex;\n      logger: LoggerService;\n    },\n  ) {}\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the\n          // transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  async replaceUnprocessedEntities(\n    txOpaque: Knex | Transaction,\n    options: ReplaceUnprocessedEntitiesOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex | Knex.Transaction;\n    const { toAdd, toUpsert, toRemove } = await this.createDelta(tx, options);\n\n    if (toRemove.length) {\n      const removedCount = await deleteWithEagerPruningOfChildren({\n        knex: tx,\n        entityRefs: toRemove,\n        sourceKey: options.sourceKey,\n      });\n      this.options.logger.debug(\n        `removed, ${removedCount} entities: ${JSON.stringify(toRemove)}`,\n      );\n    }\n\n    if (toAdd.length) {\n      // The reason for this chunking, rather than just massively batch\n      // inserting the entire payload, is that we fall back to the individual\n      // upsert mechanism below on conflicts. That path is massively slower than\n      // the fast batch path, so we don't want to end up accidentally having to\n      // for example item-by-item upsert tens of thousands of entities in a\n      // large initial delivery dump. The implication is that the size of these\n      // chunks needs to weigh the benefit of fast successful inserts, against\n      // the drawback of super slow but more rare fallbacks. There's quickly\n      // diminishing returns though with turning up this value way high.\n      for (const chunk of lodash.chunk(toAdd, 50)) {\n        try {\n          await tx.batchInsert(\n            'refresh_state',\n            chunk.map(item => ({\n              entity_id: uuid(),\n              entity_ref: stringifyEntityRef(item.deferred.entity),\n              unprocessed_entity: JSON.stringify(item.deferred.entity),\n              unprocessed_hash: item.hash,\n              errors: '',\n              location_key: item.deferred.locationKey,\n              next_update_at: tx.fn.now(),\n              last_discovery_at: tx.fn.now(),\n            })),\n            BATCH_SIZE,\n          );\n          await tx.batchInsert(\n            'refresh_state_references',\n            chunk.map(item => ({\n              source_key: options.sourceKey,\n              target_entity_ref: stringifyEntityRef(item.deferred.entity),\n            })),\n            BATCH_SIZE,\n          );\n        } catch (error) {\n          if (!isDatabaseConflictError(error)) {\n            throw error;\n          } else {\n            this.options.logger.debug(\n              `Fast insert path failed, falling back to slow path, ${error}`,\n            );\n            toUpsert.push(...chunk);\n          }\n        }\n      }\n    }\n\n    if (toUpsert.length) {\n      for (const {\n        deferred: { entity, locationKey },\n        hash,\n      } of toUpsert) {\n        const entityRef = stringifyEntityRef(entity);\n\n        try {\n          let ok = await updateUnprocessedEntity({\n            tx,\n            entity,\n            hash,\n            locationKey,\n          });\n          if (!ok) {\n            ok = await insertUnprocessedEntity({\n              tx,\n              entity,\n              hash,\n              locationKey,\n              logger: this.options.logger,\n            });\n          }\n\n          await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n            .where('target_entity_ref', entityRef)\n            .andWhere({ source_key: options.sourceKey })\n            .delete();\n\n          if (ok) {\n            await tx<DbRefreshStateReferencesRow>(\n              'refresh_state_references',\n            ).insert({\n              source_key: options.sourceKey,\n              target_entity_ref: entityRef,\n            });\n          } else {\n            const conflictingKey = await checkLocationKeyConflict({\n              tx,\n              entityRef,\n              locationKey,\n            });\n            if (conflictingKey) {\n              this.options.logger.warn(\n                `Source ${options.sourceKey} detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n              );\n            }\n          }\n        } catch (error) {\n          this.options.logger.error(\n            `Failed to add '${entityRef}' from source '${options.sourceKey}', ${error}`,\n          );\n        }\n      }\n    }\n  }\n\n  async refreshByRefreshKeys(\n    txOpaque: Transaction,\n    options: RefreshByKeyOptions,\n  ) {\n    const tx = txOpaque as Knex.Transaction;\n    await refreshByRefreshKeys({ tx, keys: options.keys });\n  }\n\n  private async createDelta(\n    tx: Knex | Knex.Transaction,\n    options: ReplaceUnprocessedEntitiesOptions,\n  ): Promise<{\n    toAdd: { deferred: DeferredEntity; hash: string }[];\n    toUpsert: { deferred: DeferredEntity; hash: string }[];\n    toRemove: string[];\n  }> {\n    if (options.type === 'delta') {\n      const toAdd = new Array<{ deferred: DeferredEntity; hash: string }>();\n      const toUpsert = new Array<{ deferred: DeferredEntity; hash: string }>();\n      const toRemove = options.removed.map(e => e.entityRef);\n\n      for (const chunk of lodash.chunk(options.added, 1000)) {\n        const entityRefs = chunk.map(e => stringifyEntityRef(e.entity));\n        const rows = await tx<DbRefreshStateRow>('refresh_state')\n          .select(['entity_ref', 'unprocessed_hash', 'location_key'])\n          .whereIn('entity_ref', entityRefs);\n        const oldStates = new Map(\n          rows.map(row => [\n            row.entity_ref,\n            {\n              unprocessed_hash: row.unprocessed_hash,\n              location_key: row.location_key,\n            },\n          ]),\n        );\n\n        chunk.forEach((deferred, i) => {\n          const entityRef = entityRefs[i];\n          const newHash = generateStableHash(deferred.entity);\n          const oldState = oldStates.get(entityRef);\n          if (oldState === undefined) {\n            // Add any entity that does not exist in the database\n            toAdd.push({ deferred, hash: newHash });\n          } else if (\n            (deferred.locationKey ?? null) !== (oldState.location_key ?? null)\n          ) {\n            // Remove and then re-add any entity that exists, but with a different location key\n            toRemove.push(entityRef);\n            toAdd.push({ deferred, hash: newHash });\n          } else if (newHash !== oldState.unprocessed_hash) {\n            // Entities with modifications should be pushed through too\n            toUpsert.push({ deferred, hash: newHash });\n          }\n        });\n      }\n\n      return { toAdd, toUpsert, toRemove };\n    }\n\n    // Grab all of the existing references from the same source, and their locationKeys as well\n    const oldRefs = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .leftJoin<DbRefreshStateRow>('refresh_state', {\n        target_entity_ref: 'entity_ref',\n      })\n      .where({ source_key: options.sourceKey })\n      .select({\n        target_entity_ref: 'refresh_state_references.target_entity_ref',\n        location_key: 'refresh_state.location_key',\n        unprocessed_hash: 'refresh_state.unprocessed_hash',\n      });\n\n    const items = options.items.map(deferred => ({\n      deferred,\n      ref: stringifyEntityRef(deferred.entity),\n      hash: generateStableHash(deferred.entity),\n    }));\n\n    const oldRefsSet = new Map(\n      oldRefs.map(r => [\n        r.target_entity_ref,\n        {\n          locationKey: r.location_key,\n          oldEntityHash: r.unprocessed_hash,\n        },\n      ]),\n    );\n    const newRefsSet = new Set(items.map(item => item.ref));\n\n    const toAdd = new Array<{ deferred: DeferredEntity; hash: string }>();\n    const toUpsert = new Array<{ deferred: DeferredEntity; hash: string }>();\n    const toRemove = oldRefs\n      .map(row => row.target_entity_ref)\n      .filter(ref => !newRefsSet.has(ref));\n\n    for (const item of items) {\n      const oldRef = oldRefsSet.get(item.ref);\n      const upsertItem = { deferred: item.deferred, hash: item.hash };\n      if (!oldRef) {\n        // Add any entity that does not exist in the database\n        toAdd.push(upsertItem);\n      } else if (\n        (oldRef.locationKey ?? undefined) !==\n        (item.deferred.locationKey ?? undefined)\n      ) {\n        // Remove and then re-add any entity that exists, but with a different location key\n        toRemove.push(item.ref);\n        toAdd.push(upsertItem);\n      } else if (oldRef.oldEntityHash !== item.hash) {\n        // Entities with modifications should be pushed through too\n        toUpsert.push(upsertItem);\n      }\n    }\n\n    return { toAdd, toUpsert, toRemove };\n  }\n}\n"],"names":["rethrowError","deleteWithEagerPruningOfChildren","lodash","uuid","stringifyEntityRef","isDatabaseConflictError","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","refreshByRefreshKeys","toAdd","toUpsert","toRemove","generateStableHash"],"mappings":";;;;;;;;;;;;;;;;;;AA4CA,MAAM,UAAa,GAAA,EAAA;AAEZ,MAAM,uBAAoD,CAAA;AAAA,EAC/D,YACmB,OAIjB,EAAA;AAJiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA;AAIhB,EAEH,MAAM,YAAe,EAAiD,EAAA;AACpE,IAAI,IAAA;AACF,MAAA,IAAI,MAAwB,GAAA,KAAA,CAAA;AAC5B,MAAM,MAAA,IAAA,CAAK,QAAQ,QAAS,CAAA,WAAA;AAAA,QAC1B,OAAM,EAAM,KAAA;AAIV,UAAS,MAAA,GAAA,MAAM,GAAG,EAAE,CAAA;AAAA,SACtB;AAAA,QACA;AAAA;AAAA,UAEE,qBAAuB,EAAA;AAAA;AACzB,OACF;AACA,MAAO,OAAA,MAAA;AAAA,aACA,CAAG,EAAA;AACV,MAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,CAAO,KAAM,CAAA,CAAA,0BAAA,EAA6B,CAAC,CAAE,CAAA,CAAA;AAC1D,MAAA,MAAMA,wBAAa,CAAC,CAAA;AAAA;AACtB;AACF,EAEA,MAAM,0BACJ,CAAA,QAAA,EACA,OACe,EAAA;AACf,IAAA,MAAM,EAAK,GAAA,QAAA;AACX,IAAM,MAAA,EAAE,OAAO,QAAU,EAAA,QAAA,KAAa,MAAM,IAAA,CAAK,WAAY,CAAA,EAAA,EAAI,OAAO,CAAA;AAExE,IAAA,IAAI,SAAS,MAAQ,EAAA;AACnB,MAAM,MAAA,YAAA,GAAe,MAAMC,iEAAiC,CAAA;AAAA,QAC1D,IAAM,EAAA,EAAA;AAAA,QACN,UAAY,EAAA,QAAA;AAAA,QACZ,WAAW,OAAQ,CAAA;AAAA,OACpB,CAAA;AACD,MAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,QAClB,YAAY,YAAY,CAAA,WAAA,EAAc,IAAK,CAAA,SAAA,CAAU,QAAQ,CAAC,CAAA;AAAA,OAChE;AAAA;AAGF,IAAA,IAAI,MAAM,MAAQ,EAAA;AAUhB,MAAA,KAAA,MAAW,KAAS,IAAAC,uBAAA,CAAO,KAAM,CAAA,KAAA,EAAO,EAAE,CAAG,EAAA;AAC3C,QAAI,IAAA;AACF,UAAA,MAAM,EAAG,CAAA,WAAA;AAAA,YACP,eAAA;AAAA,YACA,KAAA,CAAM,IAAI,CAAS,IAAA,MAAA;AAAA,cACjB,WAAWC,OAAK,EAAA;AAAA,cAChB,UAAY,EAAAC,+BAAA,CAAmB,IAAK,CAAA,QAAA,CAAS,MAAM,CAAA;AAAA,cACnD,kBAAoB,EAAA,IAAA,CAAK,SAAU,CAAA,IAAA,CAAK,SAAS,MAAM,CAAA;AAAA,cACvD,kBAAkB,IAAK,CAAA,IAAA;AAAA,cACvB,MAAQ,EAAA,EAAA;AAAA,cACR,YAAA,EAAc,KAAK,QAAS,CAAA,WAAA;AAAA,cAC5B,cAAA,EAAgB,EAAG,CAAA,EAAA,CAAG,GAAI,EAAA;AAAA,cAC1B,iBAAA,EAAmB,EAAG,CAAA,EAAA,CAAG,GAAI;AAAA,aAC7B,CAAA,CAAA;AAAA,YACF;AAAA,WACF;AACA,UAAA,MAAM,EAAG,CAAA,WAAA;AAAA,YACP,0BAAA;AAAA,YACA,KAAA,CAAM,IAAI,CAAS,IAAA,MAAA;AAAA,cACjB,YAAY,OAAQ,CAAA,SAAA;AAAA,cACpB,iBAAmB,EAAAA,+BAAA,CAAmB,IAAK,CAAA,QAAA,CAAS,MAAM;AAAA,aAC1D,CAAA,CAAA;AAAA,YACF;AAAA,WACF;AAAA,iBACO,KAAO,EAAA;AACd,UAAI,IAAA,CAACC,wCAAwB,CAAA,KAAK,CAAG,EAAA;AACnC,YAAM,MAAA,KAAA;AAAA,WACD,MAAA;AACL,YAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,cAClB,uDAAuD,KAAK,CAAA;AAAA,aAC9D;AACA,YAAS,QAAA,CAAA,IAAA,CAAK,GAAG,KAAK,CAAA;AAAA;AACxB;AACF;AACF;AAGF,IAAA,IAAI,SAAS,MAAQ,EAAA;AACnB,MAAW,KAAA,MAAA;AAAA,QACT,QAAA,EAAU,EAAE,MAAA,EAAQ,WAAY,EAAA;AAAA,QAChC;AAAA,WACG,QAAU,EAAA;AACb,QAAM,MAAA,SAAA,GAAYD,gCAAmB,MAAM,CAAA;AAE3C,QAAI,IAAA;AACF,UAAI,IAAA,EAAA,GAAK,MAAME,+CAAwB,CAAA;AAAA,YACrC,EAAA;AAAA,YACA,MAAA;AAAA,YACA,IAAA;AAAA,YACA;AAAA,WACD,CAAA;AACD,UAAA,IAAI,CAAC,EAAI,EAAA;AACP,YAAA,EAAA,GAAK,MAAMC,+CAAwB,CAAA;AAAA,cACjC,EAAA;AAAA,cACA,MAAA;AAAA,cACA,IAAA;AAAA,cACA,WAAA;AAAA,cACA,MAAA,EAAQ,KAAK,OAAQ,CAAA;AAAA,aACtB,CAAA;AAAA;AAGH,UAAA,MAAM,EAAgC,CAAA,0BAA0B,CAC7D,CAAA,KAAA,CAAM,qBAAqB,SAAS,CAAA,CACpC,QAAS,CAAA,EAAE,UAAY,EAAA,OAAA,CAAQ,SAAU,EAAC,EAC1C,MAAO,EAAA;AAEV,UAAA,IAAI,EAAI,EAAA;AACN,YAAM,MAAA,EAAA;AAAA,cACJ;AAAA,cACA,MAAO,CAAA;AAAA,cACP,YAAY,OAAQ,CAAA,SAAA;AAAA,cACpB,iBAAmB,EAAA;AAAA,aACpB,CAAA;AAAA,WACI,MAAA;AACL,YAAM,MAAA,cAAA,GAAiB,MAAMC,iDAAyB,CAAA;AAAA,cACpD,EAAA;AAAA,cACA,SAAA;AAAA,cACA;AAAA,aACD,CAAA;AACD,YAAA,IAAI,cAAgB,EAAA;AAClB,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,gBAClB,CAAA,OAAA,EAAU,QAAQ,SAAS,CAAA,gCAAA,EAAmC,SAAS,CAA0B,uBAAA,EAAA,cAAc,iBAAiB,WAAW,CAAA;AAAA,eAC7I;AAAA;AACF;AACF,iBACO,KAAO,EAAA;AACd,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,YAClB,kBAAkB,SAAS,CAAA,eAAA,EAAkB,OAAQ,CAAA,SAAS,MAAM,KAAK,CAAA;AAAA,WAC3E;AAAA;AACF;AACF;AACF;AACF,EAEA,MAAM,oBACJ,CAAA,QAAA,EACA,OACA,EAAA;AACA,IAAA,MAAM,EAAK,GAAA,QAAA;AACX,IAAA,MAAMC,0CAAqB,EAAE,EAAA,EAAI,IAAM,EAAA,OAAA,CAAQ,MAAM,CAAA;AAAA;AACvD,EAEA,MAAc,WACZ,CAAA,EAAA,EACA,OAKC,EAAA;AACD,IAAI,IAAA,OAAA,CAAQ,SAAS,OAAS,EAAA;AAC5B,MAAMC,MAAAA,MAAAA,GAAQ,IAAI,KAAkD,EAAA;AACpE,MAAMC,MAAAA,SAAAA,GAAW,IAAI,KAAkD,EAAA;AACvE,MAAA,MAAMC,YAAW,OAAQ,CAAA,OAAA,CAAQ,GAAI,CAAA,CAAA,CAAA,KAAK,EAAE,SAAS,CAAA;AAErD,MAAA,KAAA,MAAW,SAASV,uBAAO,CAAA,KAAA,CAAM,OAAQ,CAAA,KAAA,EAAO,GAAI,CAAG,EAAA;AACrD,QAAA,MAAM,aAAa,KAAM,CAAA,GAAA,CAAI,OAAKE,+BAAmB,CAAA,CAAA,CAAE,MAAM,CAAC,CAAA;AAC9D,QAAA,MAAM,IAAO,GAAA,MAAM,EAAsB,CAAA,eAAe,EACrD,MAAO,CAAA,CAAC,YAAc,EAAA,kBAAA,EAAoB,cAAc,CAAC,CACzD,CAAA,OAAA,CAAQ,cAAc,UAAU,CAAA;AACnC,QAAA,MAAM,YAAY,IAAI,GAAA;AAAA,UACpB,IAAA,CAAK,IAAI,CAAO,GAAA,KAAA;AAAA,YACd,GAAI,CAAA,UAAA;AAAA,YACJ;AAAA,cACE,kBAAkB,GAAI,CAAA,gBAAA;AAAA,cACtB,cAAc,GAAI,CAAA;AAAA;AACpB,WACD;AAAA,SACH;AAEA,QAAM,KAAA,CAAA,OAAA,CAAQ,CAAC,QAAA,EAAU,CAAM,KAAA;AAC7B,UAAM,MAAA,SAAA,GAAY,WAAW,CAAC,CAAA;AAC9B,UAAM,MAAA,OAAA,GAAUS,uBAAmB,CAAA,QAAA,CAAS,MAAM,CAAA;AAClD,UAAM,MAAA,QAAA,GAAW,SAAU,CAAA,GAAA,CAAI,SAAS,CAAA;AACxC,UAAA,IAAI,aAAa,KAAW,CAAA,EAAA;AAE1B,YAAAH,OAAM,IAAK,CAAA,EAAE,QAAU,EAAA,IAAA,EAAM,SAAS,CAAA;AAAA,sBAErC,QAAS,CAAA,WAAA,IAAe,IAAW,OAAA,QAAA,CAAS,gBAAgB,IAC7D,CAAA,EAAA;AAEA,YAAAE,SAAAA,CAAS,KAAK,SAAS,CAAA;AACvB,YAAAF,OAAM,IAAK,CAAA,EAAE,QAAU,EAAA,IAAA,EAAM,SAAS,CAAA;AAAA,WACxC,MAAA,IAAW,OAAY,KAAA,QAAA,CAAS,gBAAkB,EAAA;AAEhD,YAAAC,UAAS,IAAK,CAAA,EAAE,QAAU,EAAA,IAAA,EAAM,SAAS,CAAA;AAAA;AAC3C,SACD,CAAA;AAAA;AAGH,MAAA,OAAO,EAAE,KAAAD,EAAAA,MAAAA,EAAO,QAAAC,EAAAA,SAAAA,EAAU,UAAAC,SAAS,EAAA;AAAA;AAIrC,IAAA,MAAM,UAAU,MAAM,EAAA;AAAA,MACpB;AAAA,KACF,CACG,SAA4B,eAAiB,EAAA;AAAA,MAC5C,iBAAmB,EAAA;AAAA,KACpB,EACA,KAAM,CAAA,EAAE,YAAY,OAAQ,CAAA,SAAA,EAAW,CAAA,CACvC,MAAO,CAAA;AAAA,MACN,iBAAmB,EAAA,4CAAA;AAAA,MACnB,YAAc,EAAA,4BAAA;AAAA,MACd,gBAAkB,EAAA;AAAA,KACnB,CAAA;AAEH,IAAA,MAAM,KAAQ,GAAA,OAAA,CAAQ,KAAM,CAAA,GAAA,CAAI,CAAa,QAAA,MAAA;AAAA,MAC3C,QAAA;AAAA,MACA,GAAA,EAAKR,+BAAmB,CAAA,QAAA,CAAS,MAAM,CAAA;AAAA,MACvC,IAAA,EAAMS,uBAAmB,CAAA,QAAA,CAAS,MAAM;AAAA,KACxC,CAAA,CAAA;AAEF,IAAA,MAAM,aAAa,IAAI,GAAA;AAAA,MACrB,OAAA,CAAQ,IAAI,CAAK,CAAA,KAAA;AAAA,QACf,CAAE,CAAA,iBAAA;AAAA,QACF;AAAA,UACE,aAAa,CAAE,CAAA,YAAA;AAAA,UACf,eAAe,CAAE,CAAA;AAAA;AACnB,OACD;AAAA,KACH;AACA,IAAM,MAAA,UAAA,GAAa,IAAI,GAAI,CAAA,KAAA,CAAM,IAAI,CAAQ,IAAA,KAAA,IAAA,CAAK,GAAG,CAAC,CAAA;AAEtD,IAAM,MAAA,KAAA,GAAQ,IAAI,KAAkD,EAAA;AACpE,IAAM,MAAA,QAAA,GAAW,IAAI,KAAkD,EAAA;AACvE,IAAA,MAAM,QAAW,GAAA,OAAA,CACd,GAAI,CAAA,CAAA,GAAA,KAAO,GAAI,CAAA,iBAAiB,CAChC,CAAA,MAAA,CAAO,CAAO,GAAA,KAAA,CAAC,UAAW,CAAA,GAAA,CAAI,GAAG,CAAC,CAAA;AAErC,IAAA,KAAA,MAAW,QAAQ,KAAO,EAAA;AACxB,MAAA,MAAM,MAAS,GAAA,UAAA,CAAW,GAAI,CAAA,IAAA,CAAK,GAAG,CAAA;AACtC,MAAA,MAAM,aAAa,EAAE,QAAA,EAAU,KAAK,QAAU,EAAA,IAAA,EAAM,KAAK,IAAK,EAAA;AAC9D,MAAA,IAAI,CAAC,MAAQ,EAAA;AAEX,QAAA,KAAA,CAAM,KAAK,UAAU,CAAA;AAAA,kBAEpB,MAAO,CAAA,WAAA,IAAe,aACtB,IAAK,CAAA,QAAA,CAAS,eAAe,KAC9B,CAAA,CAAA,EAAA;AAEA,QAAS,QAAA,CAAA,IAAA,CAAK,KAAK,GAAG,CAAA;AACtB,QAAA,KAAA,CAAM,KAAK,UAAU,CAAA;AAAA,OACZ,MAAA,IAAA,MAAA,CAAO,aAAkB,KAAA,IAAA,CAAK,IAAM,EAAA;AAE7C,QAAA,QAAA,CAAS,KAAK,UAAU,CAAA;AAAA;AAC1B;AAGF,IAAO,OAAA,EAAE,KAAO,EAAA,QAAA,EAAU,QAAS,EAAA;AAAA;AAEvC;;;;"}

package/dist/database/metrics.cjs.js

@@ -2,7 +2,6 @@
 
 var metrics = require('../util/metrics.cjs.js');
 var api = require('@opentelemetry/api');
-var catalogModel = require('@backstage/catalog-model');
 
 function initDatabaseMetrics(knex) {
   const seenProm = /* @__PURE__ */ new Set();
@@ -14,18 +13,15 @@ function initDatabaseMetrics(knex) {
       help: "Total amount of entities in the catalog. DEPRECATED: Please use opentelemetry metrics instead.",
       labelNames: ["kind"],
       async collect() {
-        const result = await knex("refresh_state").select(
-          "entity_ref"
-        );
-        const results = result.map((row) => row.entity_ref.split(":")[0]).reduce((acc, e) => acc.set(e, (acc.get(e) || 0) + 1), /* @__PURE__ */ new Map());
-        results.forEach((value, key) => {
-          seenProm.add(key);
-          this.set({ kind: key }, value);
+        const results = await knex("search").where("key", "=", "kind").whereNotNull("value").select({ kind: "value", count: knex.raw("count(*)") }).groupBy("value");
+        results.forEach(({ kind, count }) => {
+          seenProm.add(kind);
+          this.set({ kind }, Number(count));
         });
-        seenProm.forEach((key) => {
-          if (!results.has(key)) {
-            this.set({ kind: key }, 0);
-            seenProm.delete(key);
+        seenProm.forEach((kind) => {
+          if (!results.some((r) => r.kind === kind)) {
+            this.set({ kind }, 0);
+            seenProm.delete(kind);
           }
         });
       }
@@ -53,18 +49,15 @@ function initDatabaseMetrics(knex) {
     entities_count: meter.createObservableGauge("catalog_entities_count", {
       description: "Total amount of entities in the catalog"
     }).addCallback(async (gauge) => {
-      const result = await knex("refresh_state").select(
-        "entity_ref"
-      );
-      const results = result.map((row) => catalogModel.parseEntityRef(row.entity_ref).kind).reduce((acc, e) => acc.set(e, (acc.get(e) || 0) + 1), /* @__PURE__ */ new Map());
-      results.forEach((value, key) => {
-        seen.add(key);
-        gauge.observe(value, { kind: key });
+      const results = await knex("search").where("key", "=", "kind").whereNotNull("value").select({ kind: "value", count: knex.raw("count(*)") }).groupBy("value");
+      results.forEach(({ kind, count }) => {
+        seen.add(kind);
+        gauge.observe(Number(count), { kind });
       });
-      seen.forEach((key) => {
-        if (!results.has(key)) {
-          gauge.observe(0, { kind: key });
-          seen.delete(key);
+      seen.forEach((kind) => {
+        if (!results.some((r) => r.kind === kind)) {
+          gauge.observe(0, { kind });
+          seen.delete(kind);
         }
       });
     }),

package/dist/database/metrics.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"metrics.cjs.js","sources":["../../src/database/metrics.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport { createGaugeMetric } from '../util/metrics';\nimport { DbRefreshStateRow, DbRelationsRow, DbLocationsRow } from './tables';\nimport { metrics } from '@opentelemetry/api';\nimport { parseEntityRef } from '@backstage/catalog-model';\n\nexport function initDatabaseMetrics(knex: Knex) {\n  const seenProm = new Set<string>();\n  const seen = new Set<string>();\n  const meter = metrics.getMeter('default');\n  return {\n    entities_count_prom: createGaugeMetric({\n      name: 'catalog_entities_count',\n      help: 'Total amount of entities in the catalog. DEPRECATED: Please use opentelemetry metrics instead.',\n      labelNames: ['kind'],\n      async collect() {\n        const result = await knex<DbRefreshStateRow>('refresh_state').select(\n          'entity_ref',\n        );\n        const results = result\n          .map(row => row.entity_ref.split(':')[0])\n          .reduce((acc, e) => acc.set(e, (acc.get(e) || 0) + 1), new Map());\n\n        results.forEach((value, key) => {\n          seenProm.add(key);\n          this.set({ kind: key }, value);\n        });\n\n        // Set all the entities that were not seenProm to 0 and delete them from the seenProm set.\n        seenProm.forEach(key => {\n          if (!results.has(key)) {\n            this.set({ kind: key }, 0);\n            seenProm.delete(key);\n          }\n        });\n      },\n    }),\n    registered_locations_prom: createGaugeMetric({\n      name: 'catalog_registered_locations_count',\n      help: 'Total amount of registered locations in the catalog. DEPRECATED: Please use opentelemetry metrics instead.',\n      async collect() {\n        const total = await knex<DbLocationsRow>('locations').count({\n          count: '*',\n        });\n        this.set(Number(total[0].count));\n      },\n    }),\n    relations_prom: createGaugeMetric({\n      name: 'catalog_relations_count',\n      help: 'Total amount of relations between entities. DEPRECATED: Please use opentelemetry metrics instead.',\n      async collect() {\n        const total = await knex<DbRelationsRow>('relations').count({\n          count: '*',\n        });\n        this.set(Number(total[0].count));\n      },\n    }),\n    entities_count: meter\n      .createObservableGauge('catalog_entities_count', {\n        description: 'Total amount of entities in the catalog',\n      })\n      .addCallback(async gauge => {\n        const result = await knex<DbRefreshStateRow>('refresh_state').select(\n          'entity_ref',\n        );\n        const results = result\n          .map(row => parseEntityRef(row.entity_ref).kind)\n          .reduce((acc, e) => acc.set(e, (acc.get(e) || 0) + 1), new Map());\n\n        results.forEach((value, key) => {\n          seen.add(key);\n          gauge.observe(value, { kind: key });\n        });\n\n        // Set all the entities that were not seen to 0 and delete them from the seen set.\n        seen.forEach(key => {\n          if (!results.has(key)) {\n            gauge.observe(0, { kind: key });\n            seen.delete(key);\n          }\n        });\n      }),\n    registered_locations: meter\n      .createObservableGauge('catalog_registered_locations_count', {\n        description: 'Total amount of registered locations in the catalog',\n      })\n      .addCallback(async gauge => {\n        const total = await knex<DbLocationsRow>('locations').count({\n          count: '*',\n        });\n        gauge.observe(Number(total[0].count));\n      }),\n    relations: meter\n      .createObservableGauge('catalog_relations_count', {\n        description: 'Total amount of relations between entities',\n      })\n      .addCallback(async gauge => {\n        const total = await knex<DbRelationsRow>('relations').count({\n          count: '*',\n        });\n        gauge.observe(Number(total[0].count));\n      }),\n  };\n}\n"],"names":["metrics","createGaugeMetric","parseEntityRef"],"mappings":";;;;;;AAsBO,SAAS,oBAAoB,IAAY,EAAA;AAC9C,EAAM,MAAA,QAAA,uBAAe,GAAY,EAAA;AACjC,EAAM,MAAA,IAAA,uBAAW,GAAY,EAAA;AAC7B,EAAM,MAAA,KAAA,GAAQA,WAAQ,CAAA,QAAA,CAAS,SAAS,CAAA;AACxC,EAAO,OAAA;AAAA,IACL,qBAAqBC,yBAAkB,CAAA;AAAA,MACrC,IAAM,EAAA,wBAAA;AAAA,MACN,IAAM,EAAA,gGAAA;AAAA,MACN,UAAA,EAAY,CAAC,MAAM,CAAA;AAAA,MACnB,MAAM,OAAU,GAAA;AACd,QAAA,MAAM,MAAS,GAAA,MAAM,IAAwB,CAAA,eAAe,CAAE,CAAA,MAAA;AAAA,UAC5D;AAAA,SACF;AACA,QAAA,MAAM,OAAU,GAAA,MAAA,CACb,GAAI,CAAA,CAAA,GAAA,KAAO,GAAI,CAAA,UAAA,CAAW,KAAM,CAAA,GAAG,CAAE,CAAA,CAAC,CAAC,CAAA,CACvC,MAAO,CAAA,CAAC,GAAK,EAAA,CAAA,KAAM,GAAI,CAAA,GAAA,CAAI,CAAI,EAAA,CAAA,GAAA,CAAI,GAAI,CAAA,CAAC,CAAK,IAAA,CAAA,IAAK,CAAC,CAAA,kBAAO,IAAA,GAAA,EAAK,CAAA;AAElE,QAAQ,OAAA,CAAA,OAAA,CAAQ,CAAC,KAAA,EAAO,GAAQ,KAAA;AAC9B,UAAA,QAAA,CAAS,IAAI,GAAG,CAAA;AAChB,UAAA,IAAA,CAAK,GAAI,CAAA,EAAE,IAAM,EAAA,GAAA,IAAO,KAAK,CAAA;AAAA,SAC9B,CAAA;AAGD,QAAA,QAAA,CAAS,QAAQ,CAAO,GAAA,KAAA;AACtB,UAAA,IAAI,CAAC,OAAA,CAAQ,GAAI,CAAA,GAAG,CAAG,EAAA;AACrB,YAAA,IAAA,CAAK,GAAI,CAAA,EAAE,IAAM,EAAA,GAAA,IAAO,CAAC,CAAA;AACzB,YAAA,QAAA,CAAS,OAAO,GAAG,CAAA;AAAA;AACrB,SACD,CAAA;AAAA;AACH,KACD,CAAA;AAAA,IACD,2BAA2BA,yBAAkB,CAAA;AAAA,MAC3C,IAAM,EAAA,oCAAA;AAAA,MACN,IAAM,EAAA,4GAAA;AAAA,MACN,MAAM,OAAU,GAAA;AACd,QAAA,MAAM,KAAQ,GAAA,MAAM,IAAqB,CAAA,WAAW,EAAE,KAAM,CAAA;AAAA,UAC1D,KAAO,EAAA;AAAA,SACR,CAAA;AACD,QAAA,IAAA,CAAK,IAAI,MAAO,CAAA,KAAA,CAAM,CAAC,CAAA,CAAE,KAAK,CAAC,CAAA;AAAA;AACjC,KACD,CAAA;AAAA,IACD,gBAAgBA,yBAAkB,CAAA;AAAA,MAChC,IAAM,EAAA,yBAAA;AAAA,MACN,IAAM,EAAA,mGAAA;AAAA,MACN,MAAM,OAAU,GAAA;AACd,QAAA,MAAM,KAAQ,GAAA,MAAM,IAAqB,CAAA,WAAW,EAAE,KAAM,CAAA;AAAA,UAC1D,KAAO,EAAA;AAAA,SACR,CAAA;AACD,QAAA,IAAA,CAAK,IAAI,MAAO,CAAA,KAAA,CAAM,CAAC,CAAA,CAAE,KAAK,CAAC,CAAA;AAAA;AACjC,KACD,CAAA;AAAA,IACD,cAAA,EAAgB,KACb,CAAA,qBAAA,CAAsB,wBAA0B,EAAA;AAAA,MAC/C,WAAa,EAAA;AAAA,KACd,CAAA,CACA,WAAY,CAAA,OAAM,KAAS,KAAA;AAC1B,MAAA,MAAM,MAAS,GAAA,MAAM,IAAwB,CAAA,eAAe,CAAE,CAAA,MAAA;AAAA,QAC5D;AAAA,OACF;AACA,MAAM,MAAA,OAAA,GAAU,MACb,CAAA,GAAA,CAAI,CAAO,GAAA,KAAAC,2BAAA,CAAe,GAAI,CAAA,UAAU,CAAE,CAAA,IAAI,CAC9C,CAAA,MAAA,CAAO,CAAC,GAAA,EAAK,MAAM,GAAI,CAAA,GAAA,CAAI,CAAI,EAAA,CAAA,GAAA,CAAI,GAAI,CAAA,CAAC,CAAK,IAAA,CAAA,IAAK,CAAC,CAAA,kBAAO,IAAA,GAAA,EAAK,CAAA;AAElE,MAAQ,OAAA,CAAA,OAAA,CAAQ,CAAC,KAAA,EAAO,GAAQ,KAAA;AAC9B,QAAA,IAAA,CAAK,IAAI,GAAG,CAAA;AACZ,QAAA,KAAA,CAAM,OAAQ,CAAA,KAAA,EAAO,EAAE,IAAA,EAAM,KAAK,CAAA;AAAA,OACnC,CAAA;AAGD,MAAA,IAAA,CAAK,QAAQ,CAAO,GAAA,KAAA;AAClB,QAAA,IAAI,CAAC,OAAA,CAAQ,GAAI,CAAA,GAAG,CAAG,EAAA;AACrB,UAAA,KAAA,CAAM,OAAQ,CAAA,CAAA,EAAG,EAAE,IAAA,EAAM,KAAK,CAAA;AAC9B,UAAA,IAAA,CAAK,OAAO,GAAG,CAAA;AAAA;AACjB,OACD,CAAA;AAAA,KACF,CAAA;AAAA,IACH,oBAAA,EAAsB,KACnB,CAAA,qBAAA,CAAsB,oCAAsC,EAAA;AAAA,MAC3D,WAAa,EAAA;AAAA,KACd,CAAA,CACA,WAAY,CAAA,OAAM,KAAS,KAAA;AAC1B,MAAA,MAAM,KAAQ,GAAA,MAAM,IAAqB,CAAA,WAAW,EAAE,KAAM,CAAA;AAAA,QAC1D,KAAO,EAAA;AAAA,OACR,CAAA;AACD,MAAA,KAAA,CAAM,QAAQ,MAAO,CAAA,KAAA,CAAM,CAAC,CAAA,CAAE,KAAK,CAAC,CAAA;AAAA,KACrC,CAAA;AAAA,IACH,SAAA,EAAW,KACR,CAAA,qBAAA,CAAsB,yBAA2B,EAAA;AAAA,MAChD,WAAa,EAAA;AAAA,KACd,CAAA,CACA,WAAY,CAAA,OAAM,KAAS,KAAA;AAC1B,MAAA,MAAM,KAAQ,GAAA,MAAM,IAAqB,CAAA,WAAW,EAAE,KAAM,CAAA;AAAA,QAC1D,KAAO,EAAA;AAAA,OACR,CAAA;AACD,MAAA,KAAA,CAAM,QAAQ,MAAO,CAAA,KAAA,CAAM,CAAC,CAAA,CAAE,KAAK,CAAC,CAAA;AAAA,KACrC;AAAA,GACL;AACF;;;;"}
+{"version":3,"file":"metrics.cjs.js","sources":["../../src/database/metrics.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport { createGaugeMetric } from '../util/metrics';\nimport { DbRelationsRow, DbLocationsRow, DbSearchRow } from './tables';\nimport { metrics } from '@opentelemetry/api';\n\nexport function initDatabaseMetrics(knex: Knex) {\n  const seenProm = new Set<string>();\n  const seen = new Set<string>();\n  const meter = metrics.getMeter('default');\n  return {\n    entities_count_prom: createGaugeMetric({\n      name: 'catalog_entities_count',\n      help: 'Total amount of entities in the catalog. DEPRECATED: Please use opentelemetry metrics instead.',\n      labelNames: ['kind'],\n      async collect() {\n        const results = await knex<DbSearchRow>('search')\n          .where('key', '=', 'kind')\n          .whereNotNull('value')\n          .select({ kind: 'value', count: knex.raw('count(*)') })\n          .groupBy('value');\n\n        results.forEach(({ kind, count }) => {\n          seenProm.add(kind);\n          this.set({ kind }, Number(count));\n        });\n\n        // Set all the entities that were not seenProm to 0 and delete them from the seenProm set.\n        seenProm.forEach(kind => {\n          if (!results.some(r => r.kind === kind)) {\n            this.set({ kind }, 0);\n            seenProm.delete(kind);\n          }\n        });\n      },\n    }),\n    registered_locations_prom: createGaugeMetric({\n      name: 'catalog_registered_locations_count',\n      help: 'Total amount of registered locations in the catalog. DEPRECATED: Please use opentelemetry metrics instead.',\n      async collect() {\n        const total = await knex<DbLocationsRow>('locations').count({\n          count: '*',\n        });\n        this.set(Number(total[0].count));\n      },\n    }),\n    relations_prom: createGaugeMetric({\n      name: 'catalog_relations_count',\n      help: 'Total amount of relations between entities. DEPRECATED: Please use opentelemetry metrics instead.',\n      async collect() {\n        const total = await knex<DbRelationsRow>('relations').count({\n          count: '*',\n        });\n        this.set(Number(total[0].count));\n      },\n    }),\n    entities_count: meter\n      .createObservableGauge('catalog_entities_count', {\n        description: 'Total amount of entities in the catalog',\n      })\n      .addCallback(async gauge => {\n        const results = await knex<DbSearchRow>('search')\n          .where('key', '=', 'kind')\n          .whereNotNull('value')\n          .select({ kind: 'value', count: knex.raw('count(*)') })\n          .groupBy('value');\n\n        results.forEach(({ kind, count }) => {\n          seen.add(kind);\n          gauge.observe(Number(count), { kind });\n        });\n\n        // Set all the entities that were not seen to 0 and delete them from the seen set.\n        seen.forEach(kind => {\n          if (!results.some(r => r.kind === kind)) {\n            gauge.observe(0, { kind });\n            seen.delete(kind);\n          }\n        });\n      }),\n    registered_locations: meter\n      .createObservableGauge('catalog_registered_locations_count', {\n        description: 'Total amount of registered locations in the catalog',\n      })\n      .addCallback(async gauge => {\n        const total = await knex<DbLocationsRow>('locations').count({\n          count: '*',\n        });\n        gauge.observe(Number(total[0].count));\n      }),\n    relations: meter\n      .createObservableGauge('catalog_relations_count', {\n        description: 'Total amount of relations between entities',\n      })\n      .addCallback(async gauge => {\n        const total = await knex<DbRelationsRow>('relations').count({\n          count: '*',\n        });\n        gauge.observe(Number(total[0].count));\n      }),\n  };\n}\n"],"names":["metrics","createGaugeMetric"],"mappings":";;;;;AAqBO,SAAS,oBAAoB,IAAY,EAAA;AAC9C,EAAM,MAAA,QAAA,uBAAe,GAAY,EAAA;AACjC,EAAM,MAAA,IAAA,uBAAW,GAAY,EAAA;AAC7B,EAAM,MAAA,KAAA,GAAQA,WAAQ,CAAA,QAAA,CAAS,SAAS,CAAA;AACxC,EAAO,OAAA;AAAA,IACL,qBAAqBC,yBAAkB,CAAA;AAAA,MACrC,IAAM,EAAA,wBAAA;AAAA,MACN,IAAM,EAAA,gGAAA;AAAA,MACN,UAAA,EAAY,CAAC,MAAM,CAAA;AAAA,MACnB,MAAM,OAAU,GAAA;AACd,QAAM,MAAA,OAAA,GAAU,MAAM,IAAA,CAAkB,QAAQ,CAAA,CAC7C,MAAM,KAAO,EAAA,GAAA,EAAK,MAAM,CAAA,CACxB,YAAa,CAAA,OAAO,EACpB,MAAO,CAAA,EAAE,IAAM,EAAA,OAAA,EAAS,KAAO,EAAA,IAAA,CAAK,GAAI,CAAA,UAAU,CAAE,EAAC,CACrD,CAAA,OAAA,CAAQ,OAAO,CAAA;AAElB,QAAA,OAAA,CAAQ,OAAQ,CAAA,CAAC,EAAE,IAAA,EAAM,OAAY,KAAA;AACnC,UAAA,QAAA,CAAS,IAAI,IAAI,CAAA;AACjB,UAAA,IAAA,CAAK,IAAI,EAAE,IAAA,EAAQ,EAAA,MAAA,CAAO,KAAK,CAAC,CAAA;AAAA,SACjC,CAAA;AAGD,QAAA,QAAA,CAAS,QAAQ,CAAQ,IAAA,KAAA;AACvB,UAAA,IAAI,CAAC,OAAQ,CAAA,IAAA,CAAK,OAAK,CAAE,CAAA,IAAA,KAAS,IAAI,CAAG,EAAA;AACvC,YAAA,IAAA,CAAK,GAAI,CAAA,EAAE,IAAK,EAAA,EAAG,CAAC,CAAA;AACpB,YAAA,QAAA,CAAS,OAAO,IAAI,CAAA;AAAA;AACtB,SACD,CAAA;AAAA;AACH,KACD,CAAA;AAAA,IACD,2BAA2BA,yBAAkB,CAAA;AAAA,MAC3C,IAAM,EAAA,oCAAA;AAAA,MACN,IAAM,EAAA,4GAAA;AAAA,MACN,MAAM,OAAU,GAAA;AACd,QAAA,MAAM,KAAQ,GAAA,MAAM,IAAqB,CAAA,WAAW,EAAE,KAAM,CAAA;AAAA,UAC1D,KAAO,EAAA;AAAA,SACR,CAAA;AACD,QAAA,IAAA,CAAK,IAAI,MAAO,CAAA,KAAA,CAAM,CAAC,CAAA,CAAE,KAAK,CAAC,CAAA;AAAA;AACjC,KACD,CAAA;AAAA,IACD,gBAAgBA,yBAAkB,CAAA;AAAA,MAChC,IAAM,EAAA,yBAAA;AAAA,MACN,IAAM,EAAA,mGAAA;AAAA,MACN,MAAM,OAAU,GAAA;AACd,QAAA,MAAM,KAAQ,GAAA,MAAM,IAAqB,CAAA,WAAW,EAAE,KAAM,CAAA;AAAA,UAC1D,KAAO,EAAA;AAAA,SACR,CAAA;AACD,QAAA,IAAA,CAAK,IAAI,MAAO,CAAA,KAAA,CAAM,CAAC,CAAA,CAAE,KAAK,CAAC,CAAA;AAAA;AACjC,KACD,CAAA;AAAA,IACD,cAAA,EAAgB,KACb,CAAA,qBAAA,CAAsB,wBAA0B,EAAA;AAAA,MAC/C,WAAa,EAAA;AAAA,KACd,CAAA,CACA,WAAY,CAAA,OAAM,KAAS,KAAA;AAC1B,MAAM,MAAA,OAAA,GAAU,MAAM,IAAA,CAAkB,QAAQ,CAAA,CAC7C,MAAM,KAAO,EAAA,GAAA,EAAK,MAAM,CAAA,CACxB,YAAa,CAAA,OAAO,EACpB,MAAO,CAAA,EAAE,IAAM,EAAA,OAAA,EAAS,KAAO,EAAA,IAAA,CAAK,GAAI,CAAA,UAAU,CAAE,EAAC,CACrD,CAAA,OAAA,CAAQ,OAAO,CAAA;AAElB,MAAA,OAAA,CAAQ,OAAQ,CAAA,CAAC,EAAE,IAAA,EAAM,OAAY,KAAA;AACnC,QAAA,IAAA,CAAK,IAAI,IAAI,CAAA;AACb,QAAA,KAAA,CAAM,QAAQ,MAAO,CAAA,KAAK,CAAG,EAAA,EAAE,MAAM,CAAA;AAAA,OACtC,CAAA;AAGD,MAAA,IAAA,CAAK,QAAQ,CAAQ,IAAA,KAAA;AACnB,QAAA,IAAI,CAAC,OAAQ,CAAA,IAAA,CAAK,OAAK,CAAE,CAAA,IAAA,KAAS,IAAI,CAAG,EAAA;AACvC,UAAA,KAAA,CAAM,OAAQ,CAAA,CAAA,EAAG,EAAE,IAAA,EAAM,CAAA;AACzB,UAAA,IAAA,CAAK,OAAO,IAAI,CAAA;AAAA;AAClB,OACD,CAAA;AAAA,KACF,CAAA;AAAA,IACH,oBAAA,EAAsB,KACnB,CAAA,qBAAA,CAAsB,oCAAsC,EAAA;AAAA,MAC3D,WAAa,EAAA;AAAA,KACd,CAAA,CACA,WAAY,CAAA,OAAM,KAAS,KAAA;AAC1B,MAAA,MAAM,KAAQ,GAAA,MAAM,IAAqB,CAAA,WAAW,EAAE,KAAM,CAAA;AAAA,QAC1D,KAAO,EAAA;AAAA,OACR,CAAA;AACD,MAAA,KAAA,CAAM,QAAQ,MAAO,CAAA,KAAA,CAAM,CAAC,CAAA,CAAE,KAAK,CAAC,CAAA;AAAA,KACrC,CAAA;AAAA,IACH,SAAA,EAAW,KACR,CAAA,qBAAA,CAAsB,yBAA2B,EAAA;AAAA,MAChD,WAAa,EAAA;AAAA,KACd,CAAA,CACA,WAAY,CAAA,OAAM,KAAS,KAAA;AAC1B,MAAA,MAAM,KAAQ,GAAA,MAAM,IAAqB,CAAA,WAAW,EAAE,KAAM,CAAA;AAAA,QAC1D,KAAO,EAAA;AAAA,OACR,CAAA;AACD,MAAA,KAAA,CAAM,QAAQ,MAAO,CAAA,KAAA,CAAM,CAAC,CAAA,CAAE,KAAK,CAAC,CAAA;AAAA,KACrC;AAAA,GACL;AACF;;;;"}

package/dist/database/operations/refreshState/checkLocationKeyConflict.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"checkLocationKeyConflict.cjs.js","sources":["../../../../src/database/operations/refreshState/checkLocationKeyConflict.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport { DbRefreshStateRow } from '../../tables';\n\n/**\n * Checks whether a refresh state exists for the given entity that has a\n * location key that does not match the provided location key.\n *\n * @returns The conflicting key if there is one.\n */\nexport async function checkLocationKeyConflict(options: {\n  tx: Knex.Transaction;\n  entityRef: string;\n  locationKey?: string;\n}): Promise<string | undefined> {\n  const { tx, entityRef, locationKey } = options;\n\n  const row = await tx<DbRefreshStateRow>('refresh_state')\n    .select('location_key')\n    .where('entity_ref', entityRef)\n    .first();\n\n  const conflictingKey = row?.location_key;\n\n  // If there's no existing key we can't have a conflict\n  if (!conflictingKey) {\n    return undefined;\n  }\n\n  if (conflictingKey !== locationKey) {\n    return conflictingKey;\n  }\n  return undefined;\n}\n"],"names":[],"mappings":";;AAyBA,eAAsB,yBAAyB,OAIf,EAAA;AAC9B,EAAA,MAAM,EAAE,EAAA,EAAI,SAAW,EAAA,WAAA,EAAgB,GAAA,OAAA;AAEvC,EAAA,MAAM,GAAM,GAAA,MAAM,EAAsB,CAAA,eAAe,CACpD,CAAA,MAAA,CAAO,cAAc,CAAA,CACrB,KAAM,CAAA,YAAA,EAAc,SAAS,CAAA,CAC7B,KAAM,EAAA;AAET,EAAA,MAAM,iBAAiB,GAAK,EAAA,YAAA;AAG5B,EAAA,IAAI,CAAC,cAAgB,EAAA;AACnB,IAAO,OAAA,KAAA,CAAA;AAAA;AAGT,EAAA,IAAI,mBAAmB,WAAa,EAAA;AAClC,IAAO,OAAA,cAAA;AAAA;AAET,EAAO,OAAA,KAAA,CAAA;AACT;;;;"}
+{"version":3,"file":"checkLocationKeyConflict.cjs.js","sources":["../../../../src/database/operations/refreshState/checkLocationKeyConflict.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport { DbRefreshStateRow } from '../../tables';\n\n/**\n * Checks whether a refresh state exists for the given entity that has a\n * location key that does not match the provided location key.\n *\n * @returns The conflicting key if there is one.\n */\nexport async function checkLocationKeyConflict(options: {\n  tx: Knex | Knex.Transaction;\n  entityRef: string;\n  locationKey?: string;\n}): Promise<string | undefined> {\n  const { tx, entityRef, locationKey } = options;\n\n  const row = await tx<DbRefreshStateRow>('refresh_state')\n    .select('location_key')\n    .where('entity_ref', entityRef)\n    .first();\n\n  const conflictingKey = row?.location_key;\n\n  // If there's no existing key we can't have a conflict\n  if (!conflictingKey) {\n    return undefined;\n  }\n\n  if (conflictingKey !== locationKey) {\n    return conflictingKey;\n  }\n  return undefined;\n}\n"],"names":[],"mappings":";;AAyBA,eAAsB,yBAAyB,OAIf,EAAA;AAC9B,EAAA,MAAM,EAAE,EAAA,EAAI,SAAW,EAAA,WAAA,EAAgB,GAAA,OAAA;AAEvC,EAAA,MAAM,GAAM,GAAA,MAAM,EAAsB,CAAA,eAAe,CACpD,CAAA,MAAA,CAAO,cAAc,CAAA,CACrB,KAAM,CAAA,YAAA,EAAc,SAAS,CAAA,CAC7B,KAAM,EAAA;AAET,EAAA,MAAM,iBAAiB,GAAK,EAAA,YAAA;AAG5B,EAAA,IAAI,CAAC,cAAgB,EAAA;AACnB,IAAO,OAAA,KAAA,CAAA;AAAA;AAGT,EAAA,IAAI,mBAAmB,WAAa,EAAA;AAClC,IAAO,OAAA,cAAA;AAAA;AAET,EAAO,OAAA,KAAA,CAAA;AACT;;;;"}

package/dist/database/operations/refreshState/insertUnprocessedEntity.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"insertUnprocessedEntity.cjs.js","sources":["../../../../src/database/operations/refreshState/insertUnprocessedEntity.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { Knex } from 'knex';\nimport { DbRefreshStateRow } from '../../tables';\nimport { v4 as uuid } from 'uuid';\nimport {\n  LoggerService,\n  isDatabaseConflictError,\n} from '@backstage/backend-plugin-api';\n\n/**\n * Attempts to insert a new refresh state row for the given entity, returning\n * true if successful and false if there was a conflict.\n */\nexport async function insertUnprocessedEntity(options: {\n  tx: Knex.Transaction;\n  entity: Entity;\n  hash: string;\n  locationKey?: string;\n  logger: LoggerService;\n}): Promise<boolean> {\n  const { tx, entity, hash, logger, locationKey } = options;\n\n  const entityRef = stringifyEntityRef(entity);\n  const serializedEntity = JSON.stringify(entity);\n\n  try {\n    let query = tx<DbRefreshStateRow>('refresh_state').insert({\n      entity_id: uuid(),\n      entity_ref: entityRef,\n      unprocessed_entity: serializedEntity,\n      unprocessed_hash: hash,\n      errors: '',\n      location_key: locationKey,\n      next_update_at: tx.fn.now(),\n      last_discovery_at: tx.fn.now(),\n    });\n\n    // TODO(Rugvip): only tested towards MySQL, Postgres and SQLite.\n    // We have to do this because the only way to detect if there was a conflict with\n    // SQLite is to catch the error, while Postgres needs to ignore the conflict to not\n    // break the ongoing transaction.\n    if (tx.client.config.client.includes('pg')) {\n      query = query.onConflict('entity_ref').ignore() as any; // type here does not match runtime\n    }\n\n    // Postgres gives as an object with rowCount, SQLite gives us an array\n    const result: { rowCount?: number; length?: number } = await query;\n    return result.rowCount === 1 || result.length === 1;\n  } catch (error) {\n    // SQLite, or MySQL reached this rather than the rowCount check above\n    if (!isDatabaseConflictError(error)) {\n      throw error;\n    } else {\n      logger.debug(`Unable to insert a new refresh state row, ${error}`);\n      return false;\n    }\n  }\n}\n"],"names":["stringifyEntityRef","uuid","isDatabaseConflictError"],"mappings":";;;;;;AA6BA,eAAsB,wBAAwB,OAMzB,EAAA;AACnB,EAAA,MAAM,EAAE,EAAI,EAAA,MAAA,EAAQ,IAAM,EAAA,MAAA,EAAQ,aAAgB,GAAA,OAAA;AAElD,EAAM,MAAA,SAAA,GAAYA,gCAAmB,MAAM,CAAA;AAC3C,EAAM,MAAA,gBAAA,GAAmB,IAAK,CAAA,SAAA,CAAU,MAAM,CAAA;AAE9C,EAAI,IAAA;AACF,IAAA,IAAI,KAAQ,GAAA,EAAA,CAAsB,eAAe,CAAA,CAAE,MAAO,CAAA;AAAA,MACxD,WAAWC,OAAK,EAAA;AAAA,MAChB,UAAY,EAAA,SAAA;AAAA,MACZ,kBAAoB,EAAA,gBAAA;AAAA,MACpB,gBAAkB,EAAA,IAAA;AAAA,MAClB,MAAQ,EAAA,EAAA;AAAA,MACR,YAAc,EAAA,WAAA;AAAA,MACd,cAAA,EAAgB,EAAG,CAAA,EAAA,CAAG,GAAI,EAAA;AAAA,MAC1B,iBAAA,EAAmB,EAAG,CAAA,EAAA,CAAG,GAAI;AAAA,KAC9B,CAAA;AAMD,IAAA,IAAI,GAAG,MAAO,CAAA,MAAA,CAAO,MAAO,CAAA,QAAA,CAAS,IAAI,CAAG,EAAA;AAC1C,MAAA,KAAA,GAAQ,KAAM,CAAA,UAAA,CAAW,YAAY,CAAA,CAAE,MAAO,EAAA;AAAA;AAIhD,IAAA,MAAM,SAAiD,MAAM,KAAA;AAC7D,IAAA,OAAO,MAAO,CAAA,QAAA,KAAa,CAAK,IAAA,MAAA,CAAO,MAAW,KAAA,CAAA;AAAA,WAC3C,KAAO,EAAA;AAEd,IAAI,IAAA,CAACC,wCAAwB,CAAA,KAAK,CAAG,EAAA;AACnC,MAAM,MAAA,KAAA;AAAA,KACD,MAAA;AACL,MAAO,MAAA,CAAA,KAAA,CAAM,CAA6C,0CAAA,EAAA,KAAK,CAAE,CAAA,CAAA;AACjE,MAAO,OAAA,KAAA;AAAA;AACT;AAEJ;;;;"}
+{"version":3,"file":"insertUnprocessedEntity.cjs.js","sources":["../../../../src/database/operations/refreshState/insertUnprocessedEntity.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { Knex } from 'knex';\nimport { DbRefreshStateRow } from '../../tables';\nimport { v4 as uuid } from 'uuid';\nimport {\n  LoggerService,\n  isDatabaseConflictError,\n} from '@backstage/backend-plugin-api';\n\n/**\n * Attempts to insert a new refresh state row for the given entity, returning\n * true if successful and false if there was a conflict.\n */\nexport async function insertUnprocessedEntity(options: {\n  tx: Knex | Knex.Transaction;\n  entity: Entity;\n  hash: string;\n  locationKey?: string;\n  logger: LoggerService;\n}): Promise<boolean> {\n  const { tx, entity, hash, logger, locationKey } = options;\n\n  const entityRef = stringifyEntityRef(entity);\n  const serializedEntity = JSON.stringify(entity);\n\n  try {\n    let query = tx<DbRefreshStateRow>('refresh_state').insert({\n      entity_id: uuid(),\n      entity_ref: entityRef,\n      unprocessed_entity: serializedEntity,\n      unprocessed_hash: hash,\n      errors: '',\n      location_key: locationKey,\n      next_update_at: tx.fn.now(),\n      last_discovery_at: tx.fn.now(),\n    });\n\n    // TODO(Rugvip): only tested towards MySQL, Postgres and SQLite.\n    // We have to do this because the only way to detect if there was a conflict with\n    // SQLite is to catch the error, while Postgres needs to ignore the conflict to not\n    // break the ongoing transaction.\n    if (tx.client.config.client.includes('pg')) {\n      query = query.onConflict('entity_ref').ignore() as any; // type here does not match runtime\n    }\n\n    // Postgres gives as an object with rowCount, SQLite gives us an array\n    const result: { rowCount?: number; length?: number } = await query;\n    return result.rowCount === 1 || result.length === 1;\n  } catch (error) {\n    // SQLite, or MySQL reached this rather than the rowCount check above\n    if (!isDatabaseConflictError(error)) {\n      throw error;\n    } else {\n      logger.debug(`Unable to insert a new refresh state row, ${error}`);\n      return false;\n    }\n  }\n}\n"],"names":["stringifyEntityRef","uuid","isDatabaseConflictError"],"mappings":";;;;;;AA6BA,eAAsB,wBAAwB,OAMzB,EAAA;AACnB,EAAA,MAAM,EAAE,EAAI,EAAA,MAAA,EAAQ,IAAM,EAAA,MAAA,EAAQ,aAAgB,GAAA,OAAA;AAElD,EAAM,MAAA,SAAA,GAAYA,gCAAmB,MAAM,CAAA;AAC3C,EAAM,MAAA,gBAAA,GAAmB,IAAK,CAAA,SAAA,CAAU,MAAM,CAAA;AAE9C,EAAI,IAAA;AACF,IAAA,IAAI,KAAQ,GAAA,EAAA,CAAsB,eAAe,CAAA,CAAE,MAAO,CAAA;AAAA,MACxD,WAAWC,OAAK,EAAA;AAAA,MAChB,UAAY,EAAA,SAAA;AAAA,MACZ,kBAAoB,EAAA,gBAAA;AAAA,MACpB,gBAAkB,EAAA,IAAA;AAAA,MAClB,MAAQ,EAAA,EAAA;AAAA,MACR,YAAc,EAAA,WAAA;AAAA,MACd,cAAA,EAAgB,EAAG,CAAA,EAAA,CAAG,GAAI,EAAA;AAAA,MAC1B,iBAAA,EAAmB,EAAG,CAAA,EAAA,CAAG,GAAI;AAAA,KAC9B,CAAA;AAMD,IAAA,IAAI,GAAG,MAAO,CAAA,MAAA,CAAO,MAAO,CAAA,QAAA,CAAS,IAAI,CAAG,EAAA;AAC1C,MAAA,KAAA,GAAQ,KAAM,CAAA,UAAA,CAAW,YAAY,CAAA,CAAE,MAAO,EAAA;AAAA;AAIhD,IAAA,MAAM,SAAiD,MAAM,KAAA;AAC7D,IAAA,OAAO,MAAO,CAAA,QAAA,KAAa,CAAK,IAAA,MAAA,CAAO,MAAW,KAAA,CAAA;AAAA,WAC3C,KAAO,EAAA;AAEd,IAAI,IAAA,CAACC,wCAAwB,CAAA,KAAK,CAAG,EAAA;AACnC,MAAM,MAAA,KAAA;AAAA,KACD,MAAA;AACL,MAAO,MAAA,CAAA,KAAA,CAAM,CAA6C,0CAAA,EAAA,KAAK,CAAE,CAAA,CAAA;AACjE,MAAO,OAAA,KAAA;AAAA;AACT;AAEJ;;;;"}

package/dist/database/operations/refreshState/updateUnprocessedEntity.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"updateUnprocessedEntity.cjs.js","sources":["../../../../src/database/operations/refreshState/updateUnprocessedEntity.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { Knex } from 'knex';\nimport { DbRefreshStateRow } from '../../tables';\n\n/**\n * Attempts to update an existing refresh state row, returning true if it was\n * updated and false if there was no entity with a matching ref and location key.\n *\n * Updating the entity will also cause it to be scheduled for immediate processing.\n */\nexport async function updateUnprocessedEntity(options: {\n  tx: Knex.Transaction;\n  entity: Entity;\n  hash: string;\n  locationKey?: string;\n}): Promise<boolean> {\n  const { tx, entity, hash, locationKey } = options;\n\n  const entityRef = stringifyEntityRef(entity);\n  const serializedEntity = JSON.stringify(entity);\n\n  const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n    .update({\n      unprocessed_entity: serializedEntity,\n      unprocessed_hash: hash,\n      location_key: locationKey,\n      last_discovery_at: tx.fn.now(),\n      // We only get to this point if a processed entity actually had any changes, or\n      // if an entity provider requested this mutation, meaning that we can safely\n      // bump the deferred entities to the front of the queue for immediate processing.\n      next_update_at: tx.fn.now(),\n    })\n    .where('entity_ref', entityRef)\n    .andWhere(inner => {\n      if (!locationKey) {\n        return inner.whereNull('location_key');\n      }\n      return inner\n        .where('location_key', locationKey)\n        .orWhereNull('location_key');\n    });\n\n  return refreshResult === 1;\n}\n"],"names":["stringifyEntityRef"],"mappings":";;;;AA0BA,eAAsB,wBAAwB,OAKzB,EAAA;AACnB,EAAA,MAAM,EAAE,EAAA,EAAI,MAAQ,EAAA,IAAA,EAAM,aAAgB,GAAA,OAAA;AAE1C,EAAM,MAAA,SAAA,GAAYA,gCAAmB,MAAM,CAAA;AAC3C,EAAM,MAAA,gBAAA,GAAmB,IAAK,CAAA,SAAA,CAAU,MAAM,CAAA;AAE9C,EAAA,MAAM,aAAgB,GAAA,MAAM,EAAsB,CAAA,eAAe,EAC9D,MAAO,CAAA;AAAA,IACN,kBAAoB,EAAA,gBAAA;AAAA,IACpB,gBAAkB,EAAA,IAAA;AAAA,IAClB,YAAc,EAAA,WAAA;AAAA,IACd,iBAAA,EAAmB,EAAG,CAAA,EAAA,CAAG,GAAI,EAAA;AAAA;AAAA;AAAA;AAAA,IAI7B,cAAA,EAAgB,EAAG,CAAA,EAAA,CAAG,GAAI;AAAA,GAC3B,CACA,CAAA,KAAA,CAAM,cAAc,SAAS,CAAA,CAC7B,SAAS,CAAS,KAAA,KAAA;AACjB,IAAA,IAAI,CAAC,WAAa,EAAA;AAChB,MAAO,OAAA,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA;AAEvC,IAAA,OAAO,MACJ,KAAM,CAAA,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,GAC9B,CAAA;AAEH,EAAA,OAAO,aAAkB,KAAA,CAAA;AAC3B;;;;"}
+{"version":3,"file":"updateUnprocessedEntity.cjs.js","sources":["../../../../src/database/operations/refreshState/updateUnprocessedEntity.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { Knex } from 'knex';\nimport { DbRefreshStateRow } from '../../tables';\n\n/**\n * Attempts to update an existing refresh state row, returning true if it was\n * updated and false if there was no entity with a matching ref and location key.\n *\n * Updating the entity will also cause it to be scheduled for immediate processing.\n */\nexport async function updateUnprocessedEntity(options: {\n  tx: Knex | Knex.Transaction;\n  entity: Entity;\n  hash: string;\n  locationKey?: string;\n}): Promise<boolean> {\n  const { tx, entity, hash, locationKey } = options;\n\n  const entityRef = stringifyEntityRef(entity);\n  const serializedEntity = JSON.stringify(entity);\n\n  const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n    .update({\n      unprocessed_entity: serializedEntity,\n      unprocessed_hash: hash,\n      location_key: locationKey,\n      last_discovery_at: tx.fn.now(),\n      // We only get to this point if a processed entity actually had any changes, or\n      // if an entity provider requested this mutation, meaning that we can safely\n      // bump the deferred entities to the front of the queue for immediate processing.\n      next_update_at: tx.fn.now(),\n    })\n    .where('entity_ref', entityRef)\n    .andWhere(inner => {\n      if (!locationKey) {\n        return inner.whereNull('location_key');\n      }\n      return inner\n        .where('location_key', locationKey)\n        .orWhereNull('location_key');\n    });\n\n  return refreshResult === 1;\n}\n"],"names":["stringifyEntityRef"],"mappings":";;;;AA0BA,eAAsB,wBAAwB,OAKzB,EAAA;AACnB,EAAA,MAAM,EAAE,EAAA,EAAI,MAAQ,EAAA,IAAA,EAAM,aAAgB,GAAA,OAAA;AAE1C,EAAM,MAAA,SAAA,GAAYA,gCAAmB,MAAM,CAAA;AAC3C,EAAM,MAAA,gBAAA,GAAmB,IAAK,CAAA,SAAA,CAAU,MAAM,CAAA;AAE9C,EAAA,MAAM,aAAgB,GAAA,MAAM,EAAsB,CAAA,eAAe,EAC9D,MAAO,CAAA;AAAA,IACN,kBAAoB,EAAA,gBAAA;AAAA,IACpB,gBAAkB,EAAA,IAAA;AAAA,IAClB,YAAc,EAAA,WAAA;AAAA,IACd,iBAAA,EAAmB,EAAG,CAAA,EAAA,CAAG,GAAI,EAAA;AAAA;AAAA;AAAA;AAAA,IAI7B,cAAA,EAAgB,EAAG,CAAA,EAAA,CAAG,GAAI;AAAA,GAC3B,CACA,CAAA,KAAA,CAAM,cAAc,SAAS,CAAA,CAC7B,SAAS,CAAS,KAAA,KAAA;AACjB,IAAA,IAAI,CAAC,WAAa,EAAA;AAChB,MAAO,OAAA,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA;AAEvC,IAAA,OAAO,MACJ,KAAM,CAAA,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,GAC9B,CAAA;AAEH,EAAA,OAAO,aAAkB,KAAA,CAAA;AAC3B;;;;"}