@backstage/plugin-catalog-backend 1.28.0-next.2 → 1.28.0

package/CHANGELOG.md

@@ -1,5 +1,80 @@
 # @backstage/plugin-catalog-backend
 
+## 1.28.0
+
+### Minor Changes
+
+- 39fd704: Internal update to use the new generated server types from `backstage-cli package schema openapi generate --server`.
+- 76857da: Added `entity_ref` column to `final_entities` in order to move `refresh_state` away from the read path
+- 34d4360: Drop redundant indices from the database.
+
+  The following redundant indices are removed in this version:
+
+  - `final_entities_entity_id_idx` - overlaps with `final_entities_pkey`
+  - `refresh_state_entity_id_idx` - overlaps with `refresh_state_pkey`
+  - `refresh_state_entity_ref_idx` - overlaps with `refresh_state_entity_ref_uniq`
+  - `search_key_idx` and `search_value_idx` - these were replaced by the composite index `search_key_value_idx` in #22594
+
+  No negative end user impact is expected, but rather that performance should increase due to less index churn.
+
+### Patch Changes
+
+- d52d7f9: Support ISO and ms string forms of durations in config too
+- b89834b: Fixed an issue where entities would not be marked for restitching if only the target of a relationship changed.
+- 1bf02cc: Fixed bug when searching an entity by `spec.profile.displayName` in the catalog on the frontend. Text filter fields were not applied correctly to the database query resulting in empty results.
+- 4e58bc7: Upgrade to uuid v11 internally
+- 5efde17: Internal refactor to slightly speed up the processing loop
+- Updated dependencies
+  - @backstage/catalog-client@1.8.0
+  - @backstage/config@1.3.0
+  - @backstage/plugin-events-node@0.4.5
+  - @backstage/types@1.2.0
+  - @backstage/plugin-search-backend-module-catalog@0.2.5
+  - @backstage/plugin-catalog-node@1.14.0
+  - @backstage/backend-plugin-api@1.0.2
+  - @backstage/backend-openapi-utils@0.3.0
+  - @backstage/plugin-permission-common@0.8.2
+  - @backstage/catalog-model@1.7.1
+  - @backstage/errors@1.2.5
+  - @backstage/integration@1.15.2
+  - @backstage/plugin-catalog-common@1.1.1
+  - @backstage/plugin-permission-node@0.8.5
+
+## 1.28.0-next.3
+
+### Minor Changes
+
+- 76857da: Added `entity_ref` column to `final_entities` in order to move `refresh_state` away from the read path
+- 34d4360: Drop redundant indices from the database.
+
+  The following redundant indices are removed in this version:
+
+  - `final_entities_entity_id_idx` - overlaps with `final_entities_pkey`
+  - `refresh_state_entity_id_idx` - overlaps with `refresh_state_pkey`
+  - `refresh_state_entity_ref_idx` - overlaps with `refresh_state_entity_ref_uniq`
+  - `search_key_idx` and `search_value_idx` - these were replaced by the composite index `search_key_value_idx` in #22594
+
+  No negative end user impact is expected, but rather that performance should increase due to less index churn.
+
+### Patch Changes
+
+- b89834b: Fixed an issue where entities would not be marked for restitching if only the target of a relationship changed.
+- Updated dependencies
+  - @backstage/plugin-events-node@0.4.5-next.3
+  - @backstage/backend-openapi-utils@0.3.0-next.2
+  - @backstage/backend-plugin-api@1.0.2-next.2
+  - @backstage/catalog-client@1.8.0-next.1
+  - @backstage/catalog-model@1.7.0
+  - @backstage/config@1.2.0
+  - @backstage/errors@1.2.4
+  - @backstage/integration@1.15.1
+  - @backstage/types@1.1.1
+  - @backstage/plugin-catalog-common@1.1.0
+  - @backstage/plugin-catalog-node@1.14.0-next.2
+  - @backstage/plugin-permission-common@0.8.1
+  - @backstage/plugin-permission-node@0.8.5-next.2
+  - @backstage/plugin-search-backend-module-catalog@0.2.5-next.3
+
 ## 1.28.0-next.2
 
 ### Minor Changes

package/config.d.ts

@@ -157,9 +157,9 @@ export interface Config {
           /** Defer stitching to be performed asynchronously */
           mode: 'deferred';
           /** Polling interval for tasks in seconds */
-          pollingInterval?: HumanDuration;
+          pollingInterval?: HumanDuration | string;
           /** How long to wait for a stitch to complete before giving up in seconds */
-          stitchTimeout?: HumanDuration;
+          stitchTimeout?: HumanDuration | string;
         };
 
     /**

package/dist/database/DefaultProcessingDatabase.cjs.js

@@ -104,24 +104,32 @@ class DefaultProcessingDatabase {
     const { id, state } = options;
     await tx("refresh_state").update({ cache: JSON.stringify(state ?? {}) }).where("entity_id", id);
   }
-  async getProcessableEntities(txOpaque, request) {
-    const tx = txOpaque;
-    let itemsQuery = tx("refresh_state").select();
-    if (["mysql", "mysql2", "pg"].includes(tx.client.config.client)) {
+  async getProcessableEntities(maybeTx, request) {
+    const knex = maybeTx;
+    let itemsQuery = knex("refresh_state").select([
+      "entity_id",
+      "entity_ref",
+      "unprocessed_entity",
+      "result_hash",
+      "cache",
+      "errors",
+      "location_key",
+      "next_update_at"
+    ]);
+    if (["mysql", "mysql2", "pg"].includes(knex.client.config.client)) {
       itemsQuery = itemsQuery.forUpdate().skipLocked();
     }
-    const items = await itemsQuery.where("next_update_at", "<=", tx.fn.now()).limit(request.processBatchSize).orderBy("next_update_at", "asc");
+    const items = await itemsQuery.where("next_update_at", "<=", knex.fn.now()).limit(request.processBatchSize).orderBy("next_update_at", "asc");
     const interval = this.options.refreshInterval();
     const nextUpdateAt = (refreshInterval) => {
-      if (tx.client.config.client.includes("sqlite3")) {
-        return tx.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);
-      }
-      if (tx.client.config.client.includes("mysql")) {
-        return tx.raw(`now() + interval ${refreshInterval} second`);
+      if (knex.client.config.client.includes("sqlite3")) {
+        return knex.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);
+      } else if (knex.client.config.client.includes("mysql")) {
+        return knex.raw(`now() + interval ${refreshInterval} second`);
       }
-      return tx.raw(`now() + interval '${refreshInterval} seconds'`);
+      return knex.raw(`now() + interval '${refreshInterval} seconds'`);
     };
-    await tx("refresh_state").whereIn(
+    await knex("refresh_state").whereIn(
       "entity_ref",
       items.map((i) => i.entity_ref)
     ).update({
@@ -133,10 +141,8 @@ class DefaultProcessingDatabase {
           id: i.entity_id,
           entityRef: i.entity_ref,
           unprocessedEntity: JSON.parse(i.unprocessed_entity),
-          processedEntity: i.processed_entity ? JSON.parse(i.processed_entity) : void 0,
           resultHash: i.result_hash || "",
           nextUpdateAt: conversion.timestampToDateTime(i.next_update_at),
-          lastDiscoveryAt: conversion.timestampToDateTime(i.last_discovery_at),
           state: i.cache ? JSON.parse(i.cache) : void 0,
           errors: i.errors,
           locationKey: i.location_key

package/dist/database/DefaultProcessingDatabase.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultProcessingDatabase.cjs.js","sources":["../../src/database/DefaultProcessingDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError } from '@backstage/errors';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { ProcessingIntervalFunction } from '../processing';\nimport { rethrowError, timestampToDateTime } from './conversion';\nimport { initDatabaseMetrics } from './metrics';\nimport {\n  DbRefreshKeysRow,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n} from './tables';\nimport {\n  GetProcessableEntitiesResult,\n  ListParentsOptions,\n  ListParentsResult,\n  ProcessingDatabase,\n  RefreshStateItem,\n  Transaction,\n  UpdateEntityCacheOptions,\n  UpdateProcessedEntityOptions,\n} from './types';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { generateStableHash, generateTargetKey } from './util';\nimport {\n  EventBroker,\n  EventParams,\n  EventsService,\n} from '@backstage/plugin-events-node';\nimport { DateTime } from 'luxon';\nimport { CATALOG_CONFLICTS_TOPIC } from '../constants';\nimport { CatalogConflictEventPayload } from '../catalog/types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProcessingDatabase implements ProcessingDatabase {\n  constructor(\n    private readonly options: {\n      database: Knex;\n      logger: LoggerService;\n      refreshInterval: ProcessingIntervalFunction;\n      eventBroker?: EventBroker | EventsService;\n    },\n  ) {\n    initDatabaseMetrics(options.database);\n  }\n\n  async updateProcessedEntity(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<{ previous: { relations: DbRelationsRow[] } }> {\n    const tx = txOpaque as Knex.Transaction;\n    const {\n      id,\n      processedEntity,\n      resultHash,\n      errors,\n      relations,\n      deferredEntities,\n      refreshKeys,\n      locationKey,\n    } = options;\n    const configClient = tx.client.config.client;\n    const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        processed_entity: JSON.stringify(processedEntity),\n        result_hash: resultHash,\n        errors,\n        location_key: locationKey,\n      })\n      .where('entity_id', id)\n      .andWhere(inner => {\n        if (!locationKey) {\n          return inner.whereNull('location_key');\n        }\n        return inner\n          .where('location_key', locationKey)\n          .orWhereNull('location_key');\n      });\n    if (refreshResult === 0) {\n      throw new ConflictError(\n        `Conflicting write of processing result for ${id} with location key '${locationKey}'`,\n      );\n    }\n    const sourceEntityRef = stringifyEntityRef(processedEntity);\n\n    // Schedule all deferred entities for future processing.\n    await this.addUnprocessedEntities(tx, {\n      entities: deferredEntities,\n      sourceEntityRef,\n    });\n\n    // Delete old relations\n    // NOTE(freben): knex implemented support for returning() on update queries for sqlite, but at the current time of writing (Sep 2022) not for delete() queries.\n    let previousRelationRows: DbRelationsRow[];\n    if (configClient.includes('sqlite3') || configClient.includes('mysql')) {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .select('*')\n        .where({ originating_entity_id: id });\n      await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete();\n    } else {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete()\n        .returning('*');\n    }\n\n    // Batch insert new relations\n    const relationRows: DbRelationsRow[] = relations.map(\n      ({ source, target, type }) => ({\n        originating_entity_id: id,\n        source_entity_ref: stringifyEntityRef(source),\n        target_entity_ref: stringifyEntityRef(target),\n        type,\n      }),\n    );\n\n    await tx.batchInsert(\n      'relations',\n      this.deduplicateRelations(relationRows),\n      BATCH_SIZE,\n    );\n\n    // Delete old refresh keys\n    await tx<DbRefreshKeysRow>('refresh_keys')\n      .where({ entity_id: id })\n      .delete();\n\n    // Insert the refresh keys for the processed entity\n    await tx.batchInsert(\n      'refresh_keys',\n      refreshKeys.map(k => ({\n        entity_id: id,\n        key: generateTargetKey(k.key),\n      })),\n      BATCH_SIZE,\n    );\n\n    return {\n      previous: {\n        relations: previousRelationRows,\n      },\n    };\n  }\n\n  async updateProcessedEntityErrors(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, errors, resultHash } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        errors,\n        result_hash: resultHash,\n      })\n      .where('entity_id', id);\n  }\n\n  async updateEntityCache(\n    txOpaque: Transaction,\n    options: UpdateEntityCacheOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, state } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({ cache: JSON.stringify(state ?? {}) })\n      .where('entity_id', id);\n  }\n\n  async getProcessableEntities(\n    txOpaque: Transaction,\n    request: { processBatchSize: number },\n  ): Promise<GetProcessableEntitiesResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    let itemsQuery = tx<DbRefreshStateRow>('refresh_state').select();\n\n    // This avoids duplication of work because of race conditions and is\n    // also fast because locked rows are ignored rather than blocking.\n    // It's only available in MySQL and PostgreSQL\n    if (['mysql', 'mysql2', 'pg'].includes(tx.client.config.client)) {\n      itemsQuery = itemsQuery.forUpdate().skipLocked();\n    }\n\n    const items = await itemsQuery\n      .where('next_update_at', '<=', tx.fn.now())\n      .limit(request.processBatchSize)\n      .orderBy('next_update_at', 'asc');\n\n    const interval = this.options.refreshInterval();\n\n    const nextUpdateAt = (refreshInterval: number) => {\n      if (tx.client.config.client.includes('sqlite3')) {\n        return tx.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);\n      }\n\n      if (tx.client.config.client.includes('mysql')) {\n        return tx.raw(`now() + interval ${refreshInterval} second`);\n      }\n\n      return tx.raw(`now() + interval '${refreshInterval} seconds'`);\n    };\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .whereIn(\n        'entity_ref',\n        items.map(i => i.entity_ref),\n      )\n      .update({\n        next_update_at: nextUpdateAt(interval),\n      });\n\n    return {\n      items: items.map(\n        i =>\n          ({\n            id: i.entity_id,\n            entityRef: i.entity_ref,\n            unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity,\n            processedEntity: i.processed_entity\n              ? (JSON.parse(i.processed_entity) as Entity)\n              : undefined,\n            resultHash: i.result_hash || '',\n            nextUpdateAt: timestampToDateTime(i.next_update_at),\n            lastDiscoveryAt: timestampToDateTime(i.last_discovery_at),\n            state: i.cache ? JSON.parse(i.cache) : undefined,\n            errors: i.errors,\n            locationKey: i.location_key,\n          } as RefreshStateItem),\n      ),\n    };\n  }\n\n  async listParents(\n    txOpaque: Transaction,\n    options: ListParentsOptions,\n  ): Promise<ListParentsResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .where({ target_entity_ref: options.entityRef })\n      .select();\n\n    const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);\n\n    return { entityRefs };\n  }\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] {\n    return lodash.uniqBy(\n      rows,\n      r => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`,\n    );\n  }\n\n  /**\n   * Add a set of deferred entities for processing.\n   * The entities will be added at the front of the processing queue.\n   */\n  private async addUnprocessedEntities(\n    txOpaque: Transaction,\n    options: {\n      sourceEntityRef: string;\n      entities: DeferredEntity[];\n    },\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n\n    // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards\n    const stateReferences = new Array<string>();\n\n    // Upsert all of the unprocessed entities into the refresh_state table, by\n    // their entity ref.\n    for (const { entity, locationKey } of options.entities) {\n      const entityRef = stringifyEntityRef(entity);\n      const hash = generateStableHash(entity);\n\n      const updated = await updateUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n      });\n      if (updated) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      const inserted = await insertUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n        logger: this.options.logger,\n      });\n      if (inserted) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      // If the row can't be inserted, we have a conflict, but it could be either\n      // because of a conflicting locationKey or a race with another instance, so check\n      // whether the conflicting entity has the same entityRef but a different locationKey\n      const conflictingKey = await checkLocationKeyConflict({\n        tx,\n        entityRef,\n        locationKey,\n      });\n      if (conflictingKey) {\n        this.options.logger.warn(\n          `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n        );\n        if (this.options.eventBroker && locationKey) {\n          const eventParams: EventParams<CatalogConflictEventPayload> = {\n            topic: CATALOG_CONFLICTS_TOPIC,\n            eventPayload: {\n              unprocessedEntity: entity,\n              entityRef,\n              newLocationKey: locationKey,\n              existingLocationKey: conflictingKey,\n              lastConflictAt: DateTime.now().toISO()!,\n            },\n          };\n          await this.options.eventBroker?.publish(eventParams);\n        }\n      }\n    }\n\n    // Replace all references for the originating entity or source and then create new ones\n    await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n      .andWhere({ source_entity_ref: options.sourceEntityRef })\n      .delete();\n    await tx.batchInsert(\n      'refresh_state_references',\n      stateReferences.map(entityRef => ({\n        source_entity_ref: options.sourceEntityRef,\n        target_entity_ref: entityRef,\n      })),\n      BATCH_SIZE,\n    );\n  }\n}\n"],"names":["initDatabaseMetrics","errors","ConflictError","stringifyEntityRef","generateTargetKey","timestampToDateTime","rethrowError","lodash","generateStableHash","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","CATALOG_CONFLICTS_TOPIC","DateTime"],"mappings":";;;;;;;;;;;;;;;;;;AA0DA,MAAM,UAAa,GAAA,EAAA;AAEZ,MAAM,yBAAwD,CAAA;AAAA,EACnE,YACmB,OAMjB,EAAA;AANiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAOjB,IAAAA,2BAAA,CAAoB,QAAQ,QAAQ,CAAA;AAAA;AACtC,EAEA,MAAM,qBACJ,CAAA,QAAA,EACA,OACwD,EAAA;AACxD,IAAA,MAAM,EAAK,GAAA,QAAA;AACX,IAAM,MAAA;AAAA,MACJ,EAAA;AAAA,MACA,eAAA;AAAA,MACA,UAAA;AAAA,cACAC,QAAA;AAAA,MACA,SAAA;AAAA,MACA,gBAAA;AAAA,MACA,WAAA;AAAA,MACA;AAAA,KACE,GAAA,OAAA;AACJ,IAAM,MAAA,YAAA,GAAe,EAAG,CAAA,MAAA,CAAO,MAAO,CAAA,MAAA;AACtC,IAAA,MAAM,aAAgB,GAAA,MAAM,EAAsB,CAAA,eAAe,EAC9D,MAAO,CAAA;AAAA,MACN,gBAAA,EAAkB,IAAK,CAAA,SAAA,CAAU,eAAe,CAAA;AAAA,MAChD,WAAa,EAAA,UAAA;AAAA,cACbA,QAAA;AAAA,MACA,YAAc,EAAA;AAAA,KACf,CACA,CAAA,KAAA,CAAM,aAAa,EAAE,CAAA,CACrB,SAAS,CAAS,KAAA,KAAA;AACjB,MAAA,IAAI,CAAC,WAAa,EAAA;AAChB,QAAO,OAAA,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA;AAEvC,MAAA,OAAO,MACJ,KAAM,CAAA,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,KAC9B,CAAA;AACH,IAAA,IAAI,kBAAkB,CAAG,EAAA;AACvB,MAAA,MAAM,IAAIC,oBAAA;AAAA,QACR,CAAA,2CAAA,EAA8C,EAAE,CAAA,oBAAA,EAAuB,WAAW,CAAA,CAAA;AAAA,OACpF;AAAA;AAEF,IAAM,MAAA,eAAA,GAAkBC,gCAAmB,eAAe,CAAA;AAG1D,IAAM,MAAA,IAAA,CAAK,uBAAuB,EAAI,EAAA;AAAA,MACpC,QAAU,EAAA,gBAAA;AAAA,MACV;AAAA,KACD,CAAA;AAID,IAAI,IAAA,oBAAA;AACJ,IAAA,IAAI,aAAa,QAAS,CAAA,SAAS,KAAK,YAAa,CAAA,QAAA,CAAS,OAAO,CAAG,EAAA;AACtE,MAAuB,oBAAA,GAAA,MAAM,EAAmB,CAAA,WAAW,CACxD,CAAA,MAAA,CAAO,GAAG,CAAA,CACV,KAAM,CAAA,EAAE,qBAAuB,EAAA,EAAA,EAAI,CAAA;AACtC,MAAM,MAAA,EAAA,CAAmB,WAAW,CACjC,CAAA,KAAA,CAAM,EAAE,qBAAuB,EAAA,EAAA,EAAI,CAAA,CACnC,MAAO,EAAA;AAAA,KACL,MAAA;AACL,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,KAAM,CAAA,EAAE,qBAAuB,EAAA,EAAA,EAAI,CAAA,CACnC,MAAO,EAAA,CACP,UAAU,GAAG,CAAA;AAAA;AAIlB,IAAA,MAAM,eAAiC,SAAU,CAAA,GAAA;AAAA,MAC/C,CAAC,EAAE,MAAQ,EAAA,MAAA,EAAQ,MAAY,MAAA;AAAA,QAC7B,qBAAuB,EAAA,EAAA;AAAA,QACvB,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C;AAAA,OACF;AAAA,KACF;AAEA,IAAA,MAAM,EAAG,CAAA,WAAA;AAAA,MACP,WAAA;AAAA,MACA,IAAA,CAAK,qBAAqB,YAAY,CAAA;AAAA,MACtC;AAAA,KACF;AAGA,IAAM,MAAA,EAAA,CAAqB,cAAc,CACtC,CAAA,KAAA,CAAM,EAAE,SAAW,EAAA,EAAA,EAAI,CAAA,CACvB,MAAO,EAAA;AAGV,IAAA,MAAM,EAAG,CAAA,WAAA;AAAA,MACP,cAAA;AAAA,MACA,WAAA,CAAY,IAAI,CAAM,CAAA,MAAA;AAAA,QACpB,SAAW,EAAA,EAAA;AAAA,QACX,GAAA,EAAKC,sBAAkB,CAAA,CAAA,CAAE,GAAG;AAAA,OAC5B,CAAA,CAAA;AAAA,MACF;AAAA,KACF;AAEA,IAAO,OAAA;AAAA,MACL,QAAU,EAAA;AAAA,QACR,SAAW,EAAA;AAAA;AACb,KACF;AAAA;AACF,EAEA,MAAM,2BACJ,CAAA,QAAA,EACA,OACe,EAAA;AACf,IAAA,MAAM,EAAK,GAAA,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,MAAQ,EAAA,UAAA,EAAe,GAAA,OAAA;AAEnC,IAAM,MAAA,EAAA,CAAsB,eAAe,CAAA,CACxC,MAAO,CAAA;AAAA,MACN,MAAA;AAAA,MACA,WAAa,EAAA;AAAA,KACd,CAAA,CACA,KAAM,CAAA,WAAA,EAAa,EAAE,CAAA;AAAA;AAC1B,EAEA,MAAM,iBACJ,CAAA,QAAA,EACA,OACe,EAAA;AACf,IAAA,MAAM,EAAK,GAAA,QAAA;AACX,IAAM,MAAA,EAAE,EAAI,EAAA,KAAA,EAAU,GAAA,OAAA;AAEtB,IAAA,MAAM,GAAsB,eAAe,CAAA,CACxC,MAAO,CAAA,EAAE,OAAO,IAAK,CAAA,SAAA,CAAU,KAAS,IAAA,EAAE,CAAE,EAAC,CAC7C,CAAA,KAAA,CAAM,aAAa,EAAE,CAAA;AAAA;AAC1B,EAEA,MAAM,sBACJ,CAAA,QAAA,EACA,OACuC,EAAA;AACvC,IAAA,MAAM,EAAK,GAAA,QAAA;AAEX,IAAA,IAAI,UAAa,GAAA,EAAA,CAAsB,eAAe,CAAA,CAAE,MAAO,EAAA;AAK/D,IAAI,IAAA,CAAC,OAAS,EAAA,QAAA,EAAU,IAAI,CAAA,CAAE,SAAS,EAAG,CAAA,MAAA,CAAO,MAAO,CAAA,MAAM,CAAG,EAAA;AAC/D,MAAa,UAAA,GAAA,UAAA,CAAW,SAAU,EAAA,CAAE,UAAW,EAAA;AAAA;AAGjD,IAAA,MAAM,QAAQ,MAAM,UAAA,CACjB,KAAM,CAAA,gBAAA,EAAkB,MAAM,EAAG,CAAA,EAAA,CAAG,GAAI,EAAC,EACzC,KAAM,CAAA,OAAA,CAAQ,gBAAgB,CAC9B,CAAA,OAAA,CAAQ,kBAAkB,KAAK,CAAA;AAElC,IAAM,MAAA,QAAA,GAAW,IAAK,CAAA,OAAA,CAAQ,eAAgB,EAAA;AAE9C,IAAM,MAAA,YAAA,GAAe,CAAC,eAA4B,KAAA;AAChD,MAAA,IAAI,GAAG,MAAO,CAAA,MAAA,CAAO,MAAO,CAAA,QAAA,CAAS,SAAS,CAAG,EAAA;AAC/C,QAAA,OAAO,GAAG,GAAI,CAAA,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAG,EAAA,eAAe,UAAU,CAAC,CAAA;AAAA;AAGpE,MAAA,IAAI,GAAG,MAAO,CAAA,MAAA,CAAO,MAAO,CAAA,QAAA,CAAS,OAAO,CAAG,EAAA;AAC7C,QAAA,OAAO,EAAG,CAAA,GAAA,CAAI,CAAoB,iBAAA,EAAA,eAAe,CAAS,OAAA,CAAA,CAAA;AAAA;AAG5D,MAAA,OAAO,EAAG,CAAA,GAAA,CAAI,CAAqB,kBAAA,EAAA,eAAe,CAAW,SAAA,CAAA,CAAA;AAAA,KAC/D;AAEA,IAAM,MAAA,EAAA,CAAsB,eAAe,CACxC,CAAA,OAAA;AAAA,MACC,YAAA;AAAA,MACA,KAAM,CAAA,GAAA,CAAI,CAAK,CAAA,KAAA,CAAA,CAAE,UAAU;AAAA,MAE5B,MAAO,CAAA;AAAA,MACN,cAAA,EAAgB,aAAa,QAAQ;AAAA,KACtC,CAAA;AAEH,IAAO,OAAA;AAAA,MACL,OAAO,KAAM,CAAA,GAAA;AAAA,QACX,CACG,CAAA,MAAA;AAAA,UACC,IAAI,CAAE,CAAA,SAAA;AAAA,UACN,WAAW,CAAE,CAAA,UAAA;AAAA,UACb,iBAAmB,EAAA,IAAA,CAAK,KAAM,CAAA,CAAA,CAAE,kBAAkB,CAAA;AAAA,UAClD,iBAAiB,CAAE,CAAA,gBAAA,GACd,KAAK,KAAM,CAAA,CAAA,CAAE,gBAAgB,CAC9B,GAAA,KAAA,CAAA;AAAA,UACJ,UAAA,EAAY,EAAE,WAAe,IAAA,EAAA;AAAA,UAC7B,YAAA,EAAcC,8BAAoB,CAAA,CAAA,CAAE,cAAc,CAAA;AAAA,UAClD,eAAA,EAAiBA,8BAAoB,CAAA,CAAA,CAAE,iBAAiB,CAAA;AAAA,UACxD,OAAO,CAAE,CAAA,KAAA,GAAQ,KAAK,KAAM,CAAA,CAAA,CAAE,KAAK,CAAI,GAAA,KAAA,CAAA;AAAA,UACvC,QAAQ,CAAE,CAAA,MAAA;AAAA,UACV,aAAa,CAAE,CAAA;AAAA,SACjB;AAAA;AACJ,KACF;AAAA;AACF,EAEA,MAAM,WACJ,CAAA,QAAA,EACA,OAC4B,EAAA;AAC5B,IAAA,MAAM,EAAK,GAAA,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,KACF,CACG,MAAM,EAAE,iBAAA,EAAmB,QAAQ,SAAU,EAAC,EAC9C,MAAO,EAAA;AAEV,IAAM,MAAA,UAAA,GAAa,KAAK,GAAI,CAAA,CAAA,CAAA,KAAK,EAAE,iBAAkB,CAAA,CAAE,OAAO,OAAO,CAAA;AAErE,IAAA,OAAO,EAAE,UAAW,EAAA;AAAA;AACtB,EAEA,MAAM,YAAe,EAAiD,EAAA;AACpE,IAAI,IAAA;AACF,MAAA,IAAI,MAAwB,GAAA,KAAA,CAAA;AAE5B,MAAM,MAAA,IAAA,CAAK,QAAQ,QAAS,CAAA,WAAA;AAAA,QAC1B,OAAM,EAAM,KAAA;AAGV,UAAS,MAAA,GAAA,MAAM,GAAG,EAAE,CAAA;AAAA,SACtB;AAAA,QACA;AAAA;AAAA,UAEE,qBAAuB,EAAA;AAAA;AACzB,OACF;AAEA,MAAO,OAAA,MAAA;AAAA,aACA,CAAG,EAAA;AACV,MAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,CAAO,KAAM,CAAA,CAAA,0BAAA,EAA6B,CAAC,CAAE,CAAA,CAAA;AAC1D,MAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA;AACtB;AACF,EAEQ,qBAAqB,IAA0C,EAAA;AACrE,IAAA,OAAOC,uBAAO,CAAA,MAAA;AAAA,MACZ,IAAA;AAAA,MACA,CAAA,CAAA,KAAK,GAAG,CAAE,CAAA,iBAAiB,IAAI,CAAE,CAAA,iBAAiB,CAAI,CAAA,EAAA,CAAA,CAAE,IAAI,CAAA;AAAA,KAC9D;AAAA;AACF;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,sBACZ,CAAA,QAAA,EACA,OAIe,EAAA;AACf,IAAA,MAAM,EAAK,GAAA,QAAA;AAGX,IAAM,MAAA,eAAA,GAAkB,IAAI,KAAc,EAAA;AAI1C,IAAA,KAAA,MAAW,EAAE,MAAA,EAAQ,WAAY,EAAA,IAAK,QAAQ,QAAU,EAAA;AACtD,MAAM,MAAA,SAAA,GAAYJ,gCAAmB,MAAM,CAAA;AAC3C,MAAM,MAAA,IAAA,GAAOK,wBAAmB,MAAM,CAAA;AAEtC,MAAM,MAAA,OAAA,GAAU,MAAMC,+CAAwB,CAAA;AAAA,QAC5C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,OAAS,EAAA;AACX,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA;AAGF,MAAM,MAAA,QAAA,GAAW,MAAMC,+CAAwB,CAAA;AAAA,QAC7C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA,WAAA;AAAA,QACA,MAAA,EAAQ,KAAK,OAAQ,CAAA;AAAA,OACtB,CAAA;AACD,MAAA,IAAI,QAAU,EAAA;AACZ,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA;AAMF,MAAM,MAAA,cAAA,GAAiB,MAAMC,iDAAyB,CAAA;AAAA,QACpD,EAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,cAAgB,EAAA;AAClB,QAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,UAClB,CAAkC,+BAAA,EAAA,SAAS,CAA0B,uBAAA,EAAA,cAAc,iBAAiB,WAAW,CAAA;AAAA,SACjH;AACA,QAAI,IAAA,IAAA,CAAK,OAAQ,CAAA,WAAA,IAAe,WAAa,EAAA;AAC3C,UAAA,MAAM,WAAwD,GAAA;AAAA,YAC5D,KAAO,EAAAC,iCAAA;AAAA,YACP,YAAc,EAAA;AAAA,cACZ,iBAAmB,EAAA,MAAA;AAAA,cACnB,SAAA;AAAA,cACA,cAAgB,EAAA,WAAA;AAAA,cAChB,mBAAqB,EAAA,cAAA;AAAA,cACrB,cAAgB,EAAAC,cAAA,CAAS,GAAI,EAAA,CAAE,KAAM;AAAA;AACvC,WACF;AACA,UAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,WAAa,EAAA,OAAA,CAAQ,WAAW,CAAA;AAAA;AACrD;AACF;AAIF,IAAM,MAAA,EAAA,CAAgC,0BAA0B,CAAA,CAC7D,QAAS,CAAA,EAAE,mBAAmB,OAAQ,CAAA,eAAA,EAAiB,CAAA,CACvD,MAAO,EAAA;AACV,IAAA,MAAM,EAAG,CAAA,WAAA;AAAA,MACP,0BAAA;AAAA,MACA,eAAA,CAAgB,IAAI,CAAc,SAAA,MAAA;AAAA,QAChC,mBAAmB,OAAQ,CAAA,eAAA;AAAA,QAC3B,iBAAmB,EAAA;AAAA,OACnB,CAAA,CAAA;AAAA,MACF;AAAA,KACF;AAAA;AAEJ;;;;"}
+{"version":3,"file":"DefaultProcessingDatabase.cjs.js","sources":["../../src/database/DefaultProcessingDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError } from '@backstage/errors';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { ProcessingIntervalFunction } from '../processing';\nimport { rethrowError, timestampToDateTime } from './conversion';\nimport { initDatabaseMetrics } from './metrics';\nimport {\n  DbRefreshKeysRow,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n} from './tables';\nimport {\n  GetProcessableEntitiesResult,\n  ListParentsOptions,\n  ListParentsResult,\n  ProcessingDatabase,\n  RefreshStateItem,\n  Transaction,\n  UpdateEntityCacheOptions,\n  UpdateProcessedEntityOptions,\n} from './types';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { generateStableHash, generateTargetKey } from './util';\nimport {\n  EventBroker,\n  EventParams,\n  EventsService,\n} from '@backstage/plugin-events-node';\nimport { DateTime } from 'luxon';\nimport { CATALOG_CONFLICTS_TOPIC } from '../constants';\nimport { CatalogConflictEventPayload } from '../catalog/types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProcessingDatabase implements ProcessingDatabase {\n  constructor(\n    private readonly options: {\n      database: Knex;\n      logger: LoggerService;\n      refreshInterval: ProcessingIntervalFunction;\n      eventBroker?: EventBroker | EventsService;\n    },\n  ) {\n    initDatabaseMetrics(options.database);\n  }\n\n  async updateProcessedEntity(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<{ previous: { relations: DbRelationsRow[] } }> {\n    const tx = txOpaque as Knex.Transaction;\n    const {\n      id,\n      processedEntity,\n      resultHash,\n      errors,\n      relations,\n      deferredEntities,\n      refreshKeys,\n      locationKey,\n    } = options;\n    const configClient = tx.client.config.client;\n    const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        processed_entity: JSON.stringify(processedEntity),\n        result_hash: resultHash,\n        errors,\n        location_key: locationKey,\n      })\n      .where('entity_id', id)\n      .andWhere(inner => {\n        if (!locationKey) {\n          return inner.whereNull('location_key');\n        }\n        return inner\n          .where('location_key', locationKey)\n          .orWhereNull('location_key');\n      });\n    if (refreshResult === 0) {\n      throw new ConflictError(\n        `Conflicting write of processing result for ${id} with location key '${locationKey}'`,\n      );\n    }\n    const sourceEntityRef = stringifyEntityRef(processedEntity);\n\n    // Schedule all deferred entities for future processing.\n    await this.addUnprocessedEntities(tx, {\n      entities: deferredEntities,\n      sourceEntityRef,\n    });\n\n    // Delete old relations\n    // NOTE(freben): knex implemented support for returning() on update queries for sqlite, but at the current time of writing (Sep 2022) not for delete() queries.\n    let previousRelationRows: DbRelationsRow[];\n    if (configClient.includes('sqlite3') || configClient.includes('mysql')) {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .select('*')\n        .where({ originating_entity_id: id });\n      await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete();\n    } else {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete()\n        .returning('*');\n    }\n\n    // Batch insert new relations\n    const relationRows: DbRelationsRow[] = relations.map(\n      ({ source, target, type }) => ({\n        originating_entity_id: id,\n        source_entity_ref: stringifyEntityRef(source),\n        target_entity_ref: stringifyEntityRef(target),\n        type,\n      }),\n    );\n\n    await tx.batchInsert(\n      'relations',\n      this.deduplicateRelations(relationRows),\n      BATCH_SIZE,\n    );\n\n    // Delete old refresh keys\n    await tx<DbRefreshKeysRow>('refresh_keys')\n      .where({ entity_id: id })\n      .delete();\n\n    // Insert the refresh keys for the processed entity\n    await tx.batchInsert(\n      'refresh_keys',\n      refreshKeys.map(k => ({\n        entity_id: id,\n        key: generateTargetKey(k.key),\n      })),\n      BATCH_SIZE,\n    );\n\n    return {\n      previous: {\n        relations: previousRelationRows,\n      },\n    };\n  }\n\n  async updateProcessedEntityErrors(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, errors, resultHash } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        errors,\n        result_hash: resultHash,\n      })\n      .where('entity_id', id);\n  }\n\n  async updateEntityCache(\n    txOpaque: Transaction,\n    options: UpdateEntityCacheOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, state } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({ cache: JSON.stringify(state ?? {}) })\n      .where('entity_id', id);\n  }\n\n  async getProcessableEntities(\n    maybeTx: Transaction | Knex,\n    request: { processBatchSize: number },\n  ): Promise<GetProcessableEntitiesResult> {\n    const knex = maybeTx as Knex.Transaction | Knex;\n\n    let itemsQuery = knex<DbRefreshStateRow>('refresh_state').select([\n      'entity_id',\n      'entity_ref',\n      'unprocessed_entity',\n      'result_hash',\n      'cache',\n      'errors',\n      'location_key',\n      'next_update_at',\n    ]);\n\n    // This avoids duplication of work because of race conditions and is\n    // also fast because locked rows are ignored rather than blocking.\n    // It's only available in MySQL and PostgreSQL\n    if (['mysql', 'mysql2', 'pg'].includes(knex.client.config.client)) {\n      itemsQuery = itemsQuery.forUpdate().skipLocked();\n    }\n\n    const items = await itemsQuery\n      .where('next_update_at', '<=', knex.fn.now())\n      .limit(request.processBatchSize)\n      .orderBy('next_update_at', 'asc');\n\n    const interval = this.options.refreshInterval();\n\n    const nextUpdateAt = (refreshInterval: number) => {\n      if (knex.client.config.client.includes('sqlite3')) {\n        return knex.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);\n      } else if (knex.client.config.client.includes('mysql')) {\n        return knex.raw(`now() + interval ${refreshInterval} second`);\n      }\n      return knex.raw(`now() + interval '${refreshInterval} seconds'`);\n    };\n\n    await knex<DbRefreshStateRow>('refresh_state')\n      .whereIn(\n        'entity_ref',\n        items.map(i => i.entity_ref),\n      )\n      .update({\n        next_update_at: nextUpdateAt(interval),\n      });\n\n    return {\n      items: items.map(\n        i =>\n          ({\n            id: i.entity_id,\n            entityRef: i.entity_ref,\n            unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity,\n            resultHash: i.result_hash || '',\n            nextUpdateAt: timestampToDateTime(i.next_update_at),\n            state: i.cache ? JSON.parse(i.cache) : undefined,\n            errors: i.errors,\n            locationKey: i.location_key,\n          } satisfies RefreshStateItem),\n      ),\n    };\n  }\n\n  async listParents(\n    txOpaque: Transaction,\n    options: ListParentsOptions,\n  ): Promise<ListParentsResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .where({ target_entity_ref: options.entityRef })\n      .select();\n\n    const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);\n\n    return { entityRefs };\n  }\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] {\n    return lodash.uniqBy(\n      rows,\n      r => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`,\n    );\n  }\n\n  /**\n   * Add a set of deferred entities for processing.\n   * The entities will be added at the front of the processing queue.\n   */\n  private async addUnprocessedEntities(\n    txOpaque: Transaction,\n    options: {\n      sourceEntityRef: string;\n      entities: DeferredEntity[];\n    },\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n\n    // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards\n    const stateReferences = new Array<string>();\n\n    // Upsert all of the unprocessed entities into the refresh_state table, by\n    // their entity ref.\n    for (const { entity, locationKey } of options.entities) {\n      const entityRef = stringifyEntityRef(entity);\n      const hash = generateStableHash(entity);\n\n      const updated = await updateUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n      });\n      if (updated) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      const inserted = await insertUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n        logger: this.options.logger,\n      });\n      if (inserted) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      // If the row can't be inserted, we have a conflict, but it could be either\n      // because of a conflicting locationKey or a race with another instance, so check\n      // whether the conflicting entity has the same entityRef but a different locationKey\n      const conflictingKey = await checkLocationKeyConflict({\n        tx,\n        entityRef,\n        locationKey,\n      });\n      if (conflictingKey) {\n        this.options.logger.warn(\n          `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n        );\n        if (this.options.eventBroker && locationKey) {\n          const eventParams: EventParams<CatalogConflictEventPayload> = {\n            topic: CATALOG_CONFLICTS_TOPIC,\n            eventPayload: {\n              unprocessedEntity: entity,\n              entityRef,\n              newLocationKey: locationKey,\n              existingLocationKey: conflictingKey,\n              lastConflictAt: DateTime.now().toISO()!,\n            },\n          };\n          await this.options.eventBroker?.publish(eventParams);\n        }\n      }\n    }\n\n    // Replace all references for the originating entity or source and then create new ones\n    await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n      .andWhere({ source_entity_ref: options.sourceEntityRef })\n      .delete();\n    await tx.batchInsert(\n      'refresh_state_references',\n      stateReferences.map(entityRef => ({\n        source_entity_ref: options.sourceEntityRef,\n        target_entity_ref: entityRef,\n      })),\n      BATCH_SIZE,\n    );\n  }\n}\n"],"names":["initDatabaseMetrics","errors","ConflictError","stringifyEntityRef","generateTargetKey","timestampToDateTime","rethrowError","lodash","generateStableHash","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","CATALOG_CONFLICTS_TOPIC","DateTime"],"mappings":";;;;;;;;;;;;;;;;;;AA0DA,MAAM,UAAa,GAAA,EAAA;AAEZ,MAAM,yBAAwD,CAAA;AAAA,EACnE,YACmB,OAMjB,EAAA;AANiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAOjB,IAAAA,2BAAA,CAAoB,QAAQ,QAAQ,CAAA;AAAA;AACtC,EAEA,MAAM,qBACJ,CAAA,QAAA,EACA,OACwD,EAAA;AACxD,IAAA,MAAM,EAAK,GAAA,QAAA;AACX,IAAM,MAAA;AAAA,MACJ,EAAA;AAAA,MACA,eAAA;AAAA,MACA,UAAA;AAAA,cACAC,QAAA;AAAA,MACA,SAAA;AAAA,MACA,gBAAA;AAAA,MACA,WAAA;AAAA,MACA;AAAA,KACE,GAAA,OAAA;AACJ,IAAM,MAAA,YAAA,GAAe,EAAG,CAAA,MAAA,CAAO,MAAO,CAAA,MAAA;AACtC,IAAA,MAAM,aAAgB,GAAA,MAAM,EAAsB,CAAA,eAAe,EAC9D,MAAO,CAAA;AAAA,MACN,gBAAA,EAAkB,IAAK,CAAA,SAAA,CAAU,eAAe,CAAA;AAAA,MAChD,WAAa,EAAA,UAAA;AAAA,cACbA,QAAA;AAAA,MACA,YAAc,EAAA;AAAA,KACf,CACA,CAAA,KAAA,CAAM,aAAa,EAAE,CAAA,CACrB,SAAS,CAAS,KAAA,KAAA;AACjB,MAAA,IAAI,CAAC,WAAa,EAAA;AAChB,QAAO,OAAA,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA;AAEvC,MAAA,OAAO,MACJ,KAAM,CAAA,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,KAC9B,CAAA;AACH,IAAA,IAAI,kBAAkB,CAAG,EAAA;AACvB,MAAA,MAAM,IAAIC,oBAAA;AAAA,QACR,CAAA,2CAAA,EAA8C,EAAE,CAAA,oBAAA,EAAuB,WAAW,CAAA,CAAA;AAAA,OACpF;AAAA;AAEF,IAAM,MAAA,eAAA,GAAkBC,gCAAmB,eAAe,CAAA;AAG1D,IAAM,MAAA,IAAA,CAAK,uBAAuB,EAAI,EAAA;AAAA,MACpC,QAAU,EAAA,gBAAA;AAAA,MACV;AAAA,KACD,CAAA;AAID,IAAI,IAAA,oBAAA;AACJ,IAAA,IAAI,aAAa,QAAS,CAAA,SAAS,KAAK,YAAa,CAAA,QAAA,CAAS,OAAO,CAAG,EAAA;AACtE,MAAuB,oBAAA,GAAA,MAAM,EAAmB,CAAA,WAAW,CACxD,CAAA,MAAA,CAAO,GAAG,CAAA,CACV,KAAM,CAAA,EAAE,qBAAuB,EAAA,EAAA,EAAI,CAAA;AACtC,MAAM,MAAA,EAAA,CAAmB,WAAW,CACjC,CAAA,KAAA,CAAM,EAAE,qBAAuB,EAAA,EAAA,EAAI,CAAA,CACnC,MAAO,EAAA;AAAA,KACL,MAAA;AACL,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,KAAM,CAAA,EAAE,qBAAuB,EAAA,EAAA,EAAI,CAAA,CACnC,MAAO,EAAA,CACP,UAAU,GAAG,CAAA;AAAA;AAIlB,IAAA,MAAM,eAAiC,SAAU,CAAA,GAAA;AAAA,MAC/C,CAAC,EAAE,MAAQ,EAAA,MAAA,EAAQ,MAAY,MAAA;AAAA,QAC7B,qBAAuB,EAAA,EAAA;AAAA,QACvB,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C;AAAA,OACF;AAAA,KACF;AAEA,IAAA,MAAM,EAAG,CAAA,WAAA;AAAA,MACP,WAAA;AAAA,MACA,IAAA,CAAK,qBAAqB,YAAY,CAAA;AAAA,MACtC;AAAA,KACF;AAGA,IAAM,MAAA,EAAA,CAAqB,cAAc,CACtC,CAAA,KAAA,CAAM,EAAE,SAAW,EAAA,EAAA,EAAI,CAAA,CACvB,MAAO,EAAA;AAGV,IAAA,MAAM,EAAG,CAAA,WAAA;AAAA,MACP,cAAA;AAAA,MACA,WAAA,CAAY,IAAI,CAAM,CAAA,MAAA;AAAA,QACpB,SAAW,EAAA,EAAA;AAAA,QACX,GAAA,EAAKC,sBAAkB,CAAA,CAAA,CAAE,GAAG;AAAA,OAC5B,CAAA,CAAA;AAAA,MACF;AAAA,KACF;AAEA,IAAO,OAAA;AAAA,MACL,QAAU,EAAA;AAAA,QACR,SAAW,EAAA;AAAA;AACb,KACF;AAAA;AACF,EAEA,MAAM,2BACJ,CAAA,QAAA,EACA,OACe,EAAA;AACf,IAAA,MAAM,EAAK,GAAA,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,MAAQ,EAAA,UAAA,EAAe,GAAA,OAAA;AAEnC,IAAM,MAAA,EAAA,CAAsB,eAAe,CAAA,CACxC,MAAO,CAAA;AAAA,MACN,MAAA;AAAA,MACA,WAAa,EAAA;AAAA,KACd,CAAA,CACA,KAAM,CAAA,WAAA,EAAa,EAAE,CAAA;AAAA;AAC1B,EAEA,MAAM,iBACJ,CAAA,QAAA,EACA,OACe,EAAA;AACf,IAAA,MAAM,EAAK,GAAA,QAAA;AACX,IAAM,MAAA,EAAE,EAAI,EAAA,KAAA,EAAU,GAAA,OAAA;AAEtB,IAAA,MAAM,GAAsB,eAAe,CAAA,CACxC,MAAO,CAAA,EAAE,OAAO,IAAK,CAAA,SAAA,CAAU,KAAS,IAAA,EAAE,CAAE,EAAC,CAC7C,CAAA,KAAA,CAAM,aAAa,EAAE,CAAA;AAAA;AAC1B,EAEA,MAAM,sBACJ,CAAA,OAAA,EACA,OACuC,EAAA;AACvC,IAAA,MAAM,IAAO,GAAA,OAAA;AAEb,IAAA,IAAI,UAAa,GAAA,IAAA,CAAwB,eAAe,CAAA,CAAE,MAAO,CAAA;AAAA,MAC/D,WAAA;AAAA,MACA,YAAA;AAAA,MACA,oBAAA;AAAA,MACA,aAAA;AAAA,MACA,OAAA;AAAA,MACA,QAAA;AAAA,MACA,cAAA;AAAA,MACA;AAAA,KACD,CAAA;AAKD,IAAI,IAAA,CAAC,OAAS,EAAA,QAAA,EAAU,IAAI,CAAA,CAAE,SAAS,IAAK,CAAA,MAAA,CAAO,MAAO,CAAA,MAAM,CAAG,EAAA;AACjE,MAAa,UAAA,GAAA,UAAA,CAAW,SAAU,EAAA,CAAE,UAAW,EAAA;AAAA;AAGjD,IAAA,MAAM,QAAQ,MAAM,UAAA,CACjB,KAAM,CAAA,gBAAA,EAAkB,MAAM,IAAK,CAAA,EAAA,CAAG,GAAI,EAAC,EAC3C,KAAM,CAAA,OAAA,CAAQ,gBAAgB,CAC9B,CAAA,OAAA,CAAQ,kBAAkB,KAAK,CAAA;AAElC,IAAM,MAAA,QAAA,GAAW,IAAK,CAAA,OAAA,CAAQ,eAAgB,EAAA;AAE9C,IAAM,MAAA,YAAA,GAAe,CAAC,eAA4B,KAAA;AAChD,MAAA,IAAI,KAAK,MAAO,CAAA,MAAA,CAAO,MAAO,CAAA,QAAA,CAAS,SAAS,CAAG,EAAA;AACjD,QAAA,OAAO,KAAK,GAAI,CAAA,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAG,EAAA,eAAe,UAAU,CAAC,CAAA;AAAA,iBAC3D,IAAK,CAAA,MAAA,CAAO,OAAO,MAAO,CAAA,QAAA,CAAS,OAAO,CAAG,EAAA;AACtD,QAAA,OAAO,IAAK,CAAA,GAAA,CAAI,CAAoB,iBAAA,EAAA,eAAe,CAAS,OAAA,CAAA,CAAA;AAAA;AAE9D,MAAA,OAAO,IAAK,CAAA,GAAA,CAAI,CAAqB,kBAAA,EAAA,eAAe,CAAW,SAAA,CAAA,CAAA;AAAA,KACjE;AAEA,IAAM,MAAA,IAAA,CAAwB,eAAe,CAC1C,CAAA,OAAA;AAAA,MACC,YAAA;AAAA,MACA,KAAM,CAAA,GAAA,CAAI,CAAK,CAAA,KAAA,CAAA,CAAE,UAAU;AAAA,MAE5B,MAAO,CAAA;AAAA,MACN,cAAA,EAAgB,aAAa,QAAQ;AAAA,KACtC,CAAA;AAEH,IAAO,OAAA;AAAA,MACL,OAAO,KAAM,CAAA,GAAA;AAAA,QACX,CACG,CAAA,MAAA;AAAA,UACC,IAAI,CAAE,CAAA,SAAA;AAAA,UACN,WAAW,CAAE,CAAA,UAAA;AAAA,UACb,iBAAmB,EAAA,IAAA,CAAK,KAAM,CAAA,CAAA,CAAE,kBAAkB,CAAA;AAAA,UAClD,UAAA,EAAY,EAAE,WAAe,IAAA,EAAA;AAAA,UAC7B,YAAA,EAAcC,8BAAoB,CAAA,CAAA,CAAE,cAAc,CAAA;AAAA,UAClD,OAAO,CAAE,CAAA,KAAA,GAAQ,KAAK,KAAM,CAAA,CAAA,CAAE,KAAK,CAAI,GAAA,KAAA,CAAA;AAAA,UACvC,QAAQ,CAAE,CAAA,MAAA;AAAA,UACV,aAAa,CAAE,CAAA;AAAA,SACjB;AAAA;AACJ,KACF;AAAA;AACF,EAEA,MAAM,WACJ,CAAA,QAAA,EACA,OAC4B,EAAA;AAC5B,IAAA,MAAM,EAAK,GAAA,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,KACF,CACG,MAAM,EAAE,iBAAA,EAAmB,QAAQ,SAAU,EAAC,EAC9C,MAAO,EAAA;AAEV,IAAM,MAAA,UAAA,GAAa,KAAK,GAAI,CAAA,CAAA,CAAA,KAAK,EAAE,iBAAkB,CAAA,CAAE,OAAO,OAAO,CAAA;AAErE,IAAA,OAAO,EAAE,UAAW,EAAA;AAAA;AACtB,EAEA,MAAM,YAAe,EAAiD,EAAA;AACpE,IAAI,IAAA;AACF,MAAA,IAAI,MAAwB,GAAA,KAAA,CAAA;AAE5B,MAAM,MAAA,IAAA,CAAK,QAAQ,QAAS,CAAA,WAAA;AAAA,QAC1B,OAAM,EAAM,KAAA;AAGV,UAAS,MAAA,GAAA,MAAM,GAAG,EAAE,CAAA;AAAA,SACtB;AAAA,QACA;AAAA;AAAA,UAEE,qBAAuB,EAAA;AAAA;AACzB,OACF;AAEA,MAAO,OAAA,MAAA;AAAA,aACA,CAAG,EAAA;AACV,MAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,CAAO,KAAM,CAAA,CAAA,0BAAA,EAA6B,CAAC,CAAE,CAAA,CAAA;AAC1D,MAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA;AACtB;AACF,EAEQ,qBAAqB,IAA0C,EAAA;AACrE,IAAA,OAAOC,uBAAO,CAAA,MAAA;AAAA,MACZ,IAAA;AAAA,MACA,CAAA,CAAA,KAAK,GAAG,CAAE,CAAA,iBAAiB,IAAI,CAAE,CAAA,iBAAiB,CAAI,CAAA,EAAA,CAAA,CAAE,IAAI,CAAA;AAAA,KAC9D;AAAA;AACF;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,sBACZ,CAAA,QAAA,EACA,OAIe,EAAA;AACf,IAAA,MAAM,EAAK,GAAA,QAAA;AAGX,IAAM,MAAA,eAAA,GAAkB,IAAI,KAAc,EAAA;AAI1C,IAAA,KAAA,MAAW,EAAE,MAAA,EAAQ,WAAY,EAAA,IAAK,QAAQ,QAAU,EAAA;AACtD,MAAM,MAAA,SAAA,GAAYJ,gCAAmB,MAAM,CAAA;AAC3C,MAAM,MAAA,IAAA,GAAOK,wBAAmB,MAAM,CAAA;AAEtC,MAAM,MAAA,OAAA,GAAU,MAAMC,+CAAwB,CAAA;AAAA,QAC5C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,OAAS,EAAA;AACX,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA;AAGF,MAAM,MAAA,QAAA,GAAW,MAAMC,+CAAwB,CAAA;AAAA,QAC7C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA,WAAA;AAAA,QACA,MAAA,EAAQ,KAAK,OAAQ,CAAA;AAAA,OACtB,CAAA;AACD,MAAA,IAAI,QAAU,EAAA;AACZ,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA;AAMF,MAAM,MAAA,cAAA,GAAiB,MAAMC,iDAAyB,CAAA;AAAA,QACpD,EAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,cAAgB,EAAA;AAClB,QAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,UAClB,CAAkC,+BAAA,EAAA,SAAS,CAA0B,uBAAA,EAAA,cAAc,iBAAiB,WAAW,CAAA;AAAA,SACjH;AACA,QAAI,IAAA,IAAA,CAAK,OAAQ,CAAA,WAAA,IAAe,WAAa,EAAA;AAC3C,UAAA,MAAM,WAAwD,GAAA;AAAA,YAC5D,KAAO,EAAAC,iCAAA;AAAA,YACP,YAAc,EAAA;AAAA,cACZ,iBAAmB,EAAA,MAAA;AAAA,cACnB,SAAA;AAAA,cACA,cAAgB,EAAA,WAAA;AAAA,cAChB,mBAAqB,EAAA,cAAA;AAAA,cACrB,cAAgB,EAAAC,cAAA,CAAS,GAAI,EAAA,CAAE,KAAM;AAAA;AACvC,WACF;AACA,UAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,WAAa,EAAA,OAAA,CAAQ,WAAW,CAAA;AAAA;AACrD;AACF;AAIF,IAAM,MAAA,EAAA,CAAgC,0BAA0B,CAAA,CAC7D,QAAS,CAAA,EAAE,mBAAmB,OAAQ,CAAA,eAAA,EAAiB,CAAA,CACvD,MAAO,EAAA;AACV,IAAA,MAAM,EAAG,CAAA,WAAA;AAAA,MACP,0BAAA;AAAA,MACA,eAAA,CAAgB,IAAI,CAAc,SAAA,MAAA;AAAA,QAChC,mBAAmB,OAAQ,CAAA,eAAA;AAAA,QAC3B,iBAAmB,EAAA;AAAA,OACnB,CAAA,CAAA;AAAA,MACF;AAAA,KACF;AAAA;AAEJ;;;;"}

package/dist/database/operations/stitcher/performStitching.cjs.js

@@ -21,6 +21,7 @@ async function performStitching(options) {
   await knex("final_entities").insert({
     entity_id: entityResult[0].entity_id,
     hash: "",
+    entity_ref: entityRef,
     stitch_ticket: stitchTicket
   }).onConflict("entity_id").merge(["stitch_ticket"]);
   const [processedResult, relationsResult] = await Promise.all([
@@ -112,7 +113,8 @@ async function performStitching(options) {
   const amountOfRowsChanged = await knex("final_entities").update({
     final_entity: JSON.stringify(entity),
     hash,
-    last_updated_at: knex.fn.now()
+    last_updated_at: knex.fn.now(),
+    entity_ref: entityRef
   }).where("entity_id", entityId).where("stitch_ticket", stitchTicket).onConflict("entity_id").merge(["final_entity", "hash", "last_updated_at"]);
   const markDeferred = async () => {
     if (options.strategy.mode !== "deferred") {

package/dist/database/operations/stitcher/performStitching.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"performStitching.cjs.js","sources":["../../../../src/database/operations/stitcher/performStitching.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from '@backstage/catalog-client';\nimport {\n  ANNOTATION_EDIT_URL,\n  ANNOTATION_VIEW_URL,\n  EntityRelation,\n} from '@backstage/catalog-model';\nimport { AlphaEntity, EntityStatusItem } from '@backstage/catalog-model/alpha';\nimport { SerializedError } from '@backstage/errors';\nimport { Knex } from 'knex';\nimport { v4 as uuid } from 'uuid';\nimport { StitchingStrategy } from '../../../stitching/types';\nimport {\n  DbFinalEntitiesRow,\n  DbRefreshStateRow,\n  DbSearchRow,\n} from '../../tables';\nimport { buildEntitySearch } from './buildEntitySearch';\nimport { markDeferredStitchCompleted } from './markDeferredStitchCompleted';\nimport { BATCH_SIZE, generateStableHash } from './util';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\n// See https://github.com/facebook/react/blob/f0cf832e1d0c8544c36aa8b310960885a11a847c/packages/react-dom-bindings/src/shared/sanitizeURL.js\nconst scriptProtocolPattern =\n  // eslint-disable-next-line no-control-regex\n  /^[\\u0000-\\u001F ]*j[\\r\\n\\t]*a[\\r\\n\\t]*v[\\r\\n\\t]*a[\\r\\n\\t]*s[\\r\\n\\t]*c[\\r\\n\\t]*r[\\r\\n\\t]*i[\\r\\n\\t]*p[\\r\\n\\t]*t[\\r\\n\\t]*\\:/i;\n\n/**\n * Performs the act of stitching - to take all of the various outputs from the\n * ingestion process, and stitching them together into the final entity JSON\n * shape.\n */\nexport async function performStitching(options: {\n  knex: Knex | Knex.Transaction;\n  logger: LoggerService;\n  strategy: StitchingStrategy;\n  entityRef: string;\n  stitchTicket?: string;\n}): Promise<'changed' | 'unchanged' | 'abandoned'> {\n  const { knex, logger, entityRef } = options;\n  const stitchTicket = options.stitchTicket ?? uuid();\n\n  const entityResult = await knex<DbRefreshStateRow>('refresh_state')\n    .where({ entity_ref: entityRef })\n    .limit(1)\n    .select('entity_id');\n  if (!entityResult.length) {\n    // Entity does no exist in refresh state table, no stitching required.\n    return 'abandoned';\n  }\n\n  // Insert stitching ticket that will be compared before inserting the final entity.\n  await knex<DbFinalEntitiesRow>('final_entities')\n    .insert({\n      entity_id: entityResult[0].entity_id,\n      hash: '',\n      stitch_ticket: stitchTicket,\n    })\n    .onConflict('entity_id')\n    .merge(['stitch_ticket']);\n\n  // Selecting from refresh_state and final_entities should yield exactly\n  // one row (except in abnormal cases where the stitch was invoked for\n  // something that didn't exist at all, in which case it's zero rows).\n  // The join with the temporary incoming_references still gives one row.\n  const [processedResult, relationsResult] = await Promise.all([\n    knex\n      .with('incoming_references', function incomingReferences(builder) {\n        return builder\n          .from('refresh_state_references')\n          .where({ target_entity_ref: entityRef })\n          .count({ count: '*' });\n      })\n      .select({\n        entityId: 'refresh_state.entity_id',\n        processedEntity: 'refresh_state.processed_entity',\n        errors: 'refresh_state.errors',\n        incomingReferenceCount: 'incoming_references.count',\n        previousHash: 'final_entities.hash',\n      })\n      .from('refresh_state')\n      .where({ 'refresh_state.entity_ref': entityRef })\n      .crossJoin(knex.raw('incoming_references'))\n      .leftOuterJoin('final_entities', {\n        'final_entities.entity_id': 'refresh_state.entity_id',\n      }),\n    knex\n      .distinct({\n        relationType: 'type',\n        relationTarget: 'target_entity_ref',\n      })\n      .from('relations')\n      .where({ source_entity_ref: entityRef })\n      .orderBy('relationType', 'asc')\n      .orderBy('relationTarget', 'asc'),\n  ]);\n\n  // If there were no rows returned, it would mean that there was no\n  // matching row even in the refresh_state. This can happen for example\n  // if we emit a relation to something that hasn't been ingested yet.\n  // It's safe to ignore this stitch attempt in that case.\n  if (!processedResult.length) {\n    logger.debug(\n      `Unable to stitch ${entityRef}, item does not exist in refresh state table`,\n    );\n    return 'abandoned';\n  }\n\n  const {\n    entityId,\n    processedEntity,\n    errors,\n    incomingReferenceCount,\n    previousHash,\n  } = processedResult[0];\n\n  // If there was no processed entity in place, the target hasn't been\n  // through the processing steps yet. It's safe to ignore this stitch\n  // attempt in that case, since another stitch will be triggered when\n  // that processing has finished.\n  if (!processedEntity) {\n    logger.debug(\n      `Unable to stitch ${entityRef}, the entity has not yet been processed`,\n    );\n    return 'abandoned';\n  }\n\n  // Grab the processed entity and stitch all of the relevant data into\n  // it\n  const entity = JSON.parse(processedEntity) as AlphaEntity;\n  const isOrphan = Number(incomingReferenceCount) === 0;\n  let statusItems: EntityStatusItem[] = [];\n\n  if (isOrphan) {\n    logger.debug(`${entityRef} is an orphan`);\n    entity.metadata.annotations = {\n      ...entity.metadata.annotations,\n      ['backstage.io/orphan']: 'true',\n    };\n  }\n  if (errors) {\n    const parsedErrors = JSON.parse(errors) as SerializedError[];\n    if (Array.isArray(parsedErrors) && parsedErrors.length) {\n      statusItems = parsedErrors.map(e => ({\n        type: ENTITY_STATUS_CATALOG_PROCESSING_TYPE,\n        level: 'error',\n        message: `${e.name}: ${e.message}`,\n        error: e,\n      }));\n    }\n  }\n  // We opt to do this check here as we otherwise can't guarantee that it will be run after all processors\n  for (const annotation of [ANNOTATION_VIEW_URL, ANNOTATION_EDIT_URL]) {\n    const value = entity.metadata.annotations?.[annotation];\n    if (typeof value === 'string' && scriptProtocolPattern.test(value)) {\n      entity.metadata.annotations![annotation] =\n        'https://backstage.io/annotation-rejected-for-security-reasons';\n    }\n  }\n\n  // TODO: entityRef is lower case and should be uppercase in the final\n  // result\n  entity.relations = relationsResult\n    .filter(row => row.relationType /* exclude null row, if relevant */)\n    .map<EntityRelation>(row => ({\n      type: row.relationType!,\n      targetRef: row.relationTarget!,\n    }));\n  if (statusItems.length) {\n    entity.status = {\n      ...entity.status,\n      items: [...(entity.status?.items ?? []), ...statusItems],\n    };\n  }\n\n  // If the output entity was actually not changed, just abort\n  const hash = generateStableHash(entity);\n  if (hash === previousHash) {\n    logger.debug(`Skipped stitching of ${entityRef}, no changes`);\n    return 'unchanged';\n  }\n\n  entity.metadata.uid = entityId;\n  if (!entity.metadata.etag) {\n    // If the original data source did not have its own etag handling,\n    // use the hash as a good-quality etag\n    entity.metadata.etag = hash;\n  }\n\n  // This may throw if the entity is invalid, so we call it before\n  // the final_entities write, even though we may end up not needing\n  // to write the search index.\n  const searchEntries = buildEntitySearch(entityId, entity);\n\n  const amountOfRowsChanged = await knex<DbFinalEntitiesRow>('final_entities')\n    .update({\n      final_entity: JSON.stringify(entity),\n      hash,\n      last_updated_at: knex.fn.now(),\n    })\n    .where('entity_id', entityId)\n    .where('stitch_ticket', stitchTicket)\n    .onConflict('entity_id')\n    .merge(['final_entity', 'hash', 'last_updated_at']);\n\n  const markDeferred = async () => {\n    if (options.strategy.mode !== 'deferred') {\n      return;\n    }\n    await markDeferredStitchCompleted({\n      knex: knex,\n      entityRef,\n      stitchTicket,\n    });\n  };\n\n  if (amountOfRowsChanged === 0) {\n    logger.debug(`Entity ${entityRef} is already stitched, skipping write.`);\n    await markDeferred();\n    return 'abandoned';\n  }\n\n  await knex.transaction(async trx => {\n    await trx<DbSearchRow>('search').where({ entity_id: entityId }).delete();\n    await trx.batchInsert('search', searchEntries, BATCH_SIZE);\n  });\n\n  await markDeferred();\n  return 'changed';\n}\n"],"names":["uuid","ENTITY_STATUS_CATALOG_PROCESSING_TYPE","ANNOTATION_VIEW_URL","ANNOTATION_EDIT_URL","generateStableHash","buildEntitySearch","markDeferredStitchCompleted","BATCH_SIZE"],"mappings":";;;;;;;;;AAsCA,MAAM,qBAAA;AAAA;AAAA,EAEJ;AAAA,CAAA;AAOF,eAAsB,iBAAiB,OAMY,EAAA;AACjD,EAAA,MAAM,EAAE,IAAA,EAAM,MAAQ,EAAA,SAAA,EAAc,GAAA,OAAA;AACpC,EAAM,MAAA,YAAA,GAAe,OAAQ,CAAA,YAAA,IAAgBA,OAAK,EAAA;AAElD,EAAA,MAAM,YAAe,GAAA,MAAM,IAAwB,CAAA,eAAe,EAC/D,KAAM,CAAA,EAAE,UAAY,EAAA,SAAA,EAAW,CAC/B,CAAA,KAAA,CAAM,CAAC,CAAA,CACP,OAAO,WAAW,CAAA;AACrB,EAAI,IAAA,CAAC,aAAa,MAAQ,EAAA;AAExB,IAAO,OAAA,WAAA;AAAA;AAIT,EAAM,MAAA,IAAA,CAAyB,gBAAgB,CAAA,CAC5C,MAAO,CAAA;AAAA,IACN,SAAA,EAAW,YAAa,CAAA,CAAC,CAAE,CAAA,SAAA;AAAA,IAC3B,IAAM,EAAA,EAAA;AAAA,IACN,aAAe,EAAA;AAAA,GAChB,EACA,UAAW,CAAA,WAAW,EACtB,KAAM,CAAA,CAAC,eAAe,CAAC,CAAA;AAM1B,EAAA,MAAM,CAAC,eAAiB,EAAA,eAAe,CAAI,GAAA,MAAM,QAAQ,GAAI,CAAA;AAAA,IAC3D,IACG,CAAA,IAAA,CAAK,qBAAuB,EAAA,SAAS,mBAAmB,OAAS,EAAA;AAChE,MAAA,OAAO,OACJ,CAAA,IAAA,CAAK,0BAA0B,CAAA,CAC/B,MAAM,EAAE,iBAAA,EAAmB,SAAU,EAAC,CACtC,CAAA,KAAA,CAAM,EAAE,KAAA,EAAO,KAAK,CAAA;AAAA,KACxB,EACA,MAAO,CAAA;AAAA,MACN,QAAU,EAAA,yBAAA;AAAA,MACV,eAAiB,EAAA,gCAAA;AAAA,MACjB,MAAQ,EAAA,sBAAA;AAAA,MACR,sBAAwB,EAAA,2BAAA;AAAA,MACxB,YAAc,EAAA;AAAA,KACf,CACA,CAAA,IAAA,CAAK,eAAe,CACpB,CAAA,KAAA,CAAM,EAAE,0BAA4B,EAAA,SAAA,EAAW,CAAA,CAC/C,UAAU,IAAK,CAAA,GAAA,CAAI,qBAAqB,CAAC,CAAA,CACzC,cAAc,gBAAkB,EAAA;AAAA,MAC/B,0BAA4B,EAAA;AAAA,KAC7B,CAAA;AAAA,IACH,KACG,QAAS,CAAA;AAAA,MACR,YAAc,EAAA,MAAA;AAAA,MACd,cAAgB,EAAA;AAAA,KACjB,CACA,CAAA,IAAA,CAAK,WAAW,CAAA,CAChB,MAAM,EAAE,iBAAA,EAAmB,SAAU,EAAC,EACtC,OAAQ,CAAA,cAAA,EAAgB,KAAK,CAC7B,CAAA,OAAA,CAAQ,kBAAkB,KAAK;AAAA,GACnC,CAAA;AAMD,EAAI,IAAA,CAAC,gBAAgB,MAAQ,EAAA;AAC3B,IAAO,MAAA,CAAA,KAAA;AAAA,MACL,oBAAoB,SAAS,CAAA,4CAAA;AAAA,KAC/B;AACA,IAAO,OAAA,WAAA;AAAA;AAGT,EAAM,MAAA;AAAA,IACJ,QAAA;AAAA,IACA,eAAA;AAAA,IACA,MAAA;AAAA,IACA,sBAAA;AAAA,IACA;AAAA,GACF,GAAI,gBAAgB,CAAC,CAAA;AAMrB,EAAA,IAAI,CAAC,eAAiB,EAAA;AACpB,IAAO,MAAA,CAAA,KAAA;AAAA,MACL,oBAAoB,SAAS,CAAA,uCAAA;AAAA,KAC/B;AACA,IAAO,OAAA,WAAA;AAAA;AAKT,EAAM,MAAA,MAAA,GAAS,IAAK,CAAA,KAAA,CAAM,eAAe,CAAA;AACzC,EAAM,MAAA,QAAA,GAAW,MAAO,CAAA,sBAAsB,CAAM,KAAA,CAAA;AACpD,EAAA,IAAI,cAAkC,EAAC;AAEvC,EAAA,IAAI,QAAU,EAAA;AACZ,IAAO,MAAA,CAAA,KAAA,CAAM,CAAG,EAAA,SAAS,CAAe,aAAA,CAAA,CAAA;AACxC,IAAA,MAAA,CAAO,SAAS,WAAc,GAAA;AAAA,MAC5B,GAAG,OAAO,QAAS,CAAA,WAAA;AAAA,MACnB,CAAC,qBAAqB,GAAG;AAAA,KAC3B;AAAA;AAEF,EAAA,IAAI,MAAQ,EAAA;AACV,IAAM,MAAA,YAAA,GAAe,IAAK,CAAA,KAAA,CAAM,MAAM,CAAA;AACtC,IAAA,IAAI,KAAM,CAAA,OAAA,CAAQ,YAAY,CAAA,IAAK,aAAa,MAAQ,EAAA;AACtD,MAAc,WAAA,GAAA,YAAA,CAAa,IAAI,CAAM,CAAA,MAAA;AAAA,QACnC,IAAM,EAAAC,mDAAA;AAAA,QACN,KAAO,EAAA,OAAA;AAAA,QACP,SAAS,CAAG,EAAA,CAAA,CAAE,IAAI,CAAA,EAAA,EAAK,EAAE,OAAO,CAAA,CAAA;AAAA,QAChC,KAAO,EAAA;AAAA,OACP,CAAA,CAAA;AAAA;AACJ;AAGF,EAAA,KAAA,MAAW,UAAc,IAAA,CAACC,gCAAqB,EAAAC,gCAAmB,CAAG,EAAA;AACnE,IAAA,MAAM,KAAQ,GAAA,MAAA,CAAO,QAAS,CAAA,WAAA,GAAc,UAAU,CAAA;AACtD,IAAA,IAAI,OAAO,KAAU,KAAA,QAAA,IAAY,qBAAsB,CAAA,IAAA,CAAK,KAAK,CAAG,EAAA;AAClE,MAAO,MAAA,CAAA,QAAA,CAAS,WAAa,CAAA,UAAU,CACrC,GAAA,+DAAA;AAAA;AACJ;AAKF,EAAA,MAAA,CAAO,YAAY,eAChB,CAAA,MAAA;AAAA,IAAO,SAAO,GAAI,CAAA;AAAA;AAAA,GAAgD,CAClE,IAAoB,CAAQ,GAAA,MAAA;AAAA,IAC3B,MAAM,GAAI,CAAA,YAAA;AAAA,IACV,WAAW,GAAI,CAAA;AAAA,GACf,CAAA,CAAA;AACJ,EAAA,IAAI,YAAY,MAAQ,EAAA;AACtB,IAAA,MAAA,CAAO,MAAS,GAAA;AAAA,MACd,GAAG,MAAO,CAAA,MAAA;AAAA,MACV,KAAA,EAAO,CAAC,GAAI,MAAA,CAAO,QAAQ,KAAS,IAAA,EAAK,EAAA,GAAG,WAAW;AAAA,KACzD;AAAA;AAIF,EAAM,MAAA,IAAA,GAAOC,wBAAmB,MAAM,CAAA;AACtC,EAAA,IAAI,SAAS,YAAc,EAAA;AACzB,IAAO,MAAA,CAAA,KAAA,CAAM,CAAwB,qBAAA,EAAA,SAAS,CAAc,YAAA,CAAA,CAAA;AAC5D,IAAO,OAAA,WAAA;AAAA;AAGT,EAAA,MAAA,CAAO,SAAS,GAAM,GAAA,QAAA;AACtB,EAAI,IAAA,CAAC,MAAO,CAAA,QAAA,CAAS,IAAM,EAAA;AAGzB,IAAA,MAAA,CAAO,SAAS,IAAO,GAAA,IAAA;AAAA;AAMzB,EAAM,MAAA,aAAA,GAAgBC,mCAAkB,CAAA,QAAA,EAAU,MAAM,CAAA;AAExD,EAAA,MAAM,mBAAsB,GAAA,MAAM,IAAyB,CAAA,gBAAgB,EACxE,MAAO,CAAA;AAAA,IACN,YAAA,EAAc,IAAK,CAAA,SAAA,CAAU,MAAM,CAAA;AAAA,IACnC,IAAA;AAAA,IACA,eAAA,EAAiB,IAAK,CAAA,EAAA,CAAG,GAAI;AAAA,GAC9B,CACA,CAAA,KAAA,CAAM,aAAa,QAAQ,CAAA,CAC3B,MAAM,eAAiB,EAAA,YAAY,CACnC,CAAA,UAAA,CAAW,WAAW,CACtB,CAAA,KAAA,CAAM,CAAC,cAAgB,EAAA,MAAA,EAAQ,iBAAiB,CAAC,CAAA;AAEpD,EAAA,MAAM,eAAe,YAAY;AAC/B,IAAI,IAAA,OAAA,CAAQ,QAAS,CAAA,IAAA,KAAS,UAAY,EAAA;AACxC,MAAA;AAAA;AAEF,IAAA,MAAMC,uDAA4B,CAAA;AAAA,MAChC,IAAA;AAAA,MACA,SAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA,GACH;AAEA,EAAA,IAAI,wBAAwB,CAAG,EAAA;AAC7B,IAAO,MAAA,CAAA,KAAA,CAAM,CAAU,OAAA,EAAA,SAAS,CAAuC,qCAAA,CAAA,CAAA;AACvE,IAAA,MAAM,YAAa,EAAA;AACnB,IAAO,OAAA,WAAA;AAAA;AAGT,EAAM,MAAA,IAAA,CAAK,WAAY,CAAA,OAAM,GAAO,KAAA;AAClC,IAAM,MAAA,GAAA,CAAiB,QAAQ,CAAE,CAAA,KAAA,CAAM,EAAE,SAAW,EAAA,QAAA,EAAU,CAAA,CAAE,MAAO,EAAA;AACvE,IAAA,MAAM,GAAI,CAAA,WAAA,CAAY,QAAU,EAAA,aAAA,EAAeC,eAAU,CAAA;AAAA,GAC1D,CAAA;AAED,EAAA,MAAM,YAAa,EAAA;AACnB,EAAO,OAAA,SAAA;AACT;;;;"}
+{"version":3,"file":"performStitching.cjs.js","sources":["../../../../src/database/operations/stitcher/performStitching.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from '@backstage/catalog-client';\nimport {\n  ANNOTATION_EDIT_URL,\n  ANNOTATION_VIEW_URL,\n  EntityRelation,\n} from '@backstage/catalog-model';\nimport { AlphaEntity, EntityStatusItem } from '@backstage/catalog-model/alpha';\nimport { SerializedError } from '@backstage/errors';\nimport { Knex } from 'knex';\nimport { v4 as uuid } from 'uuid';\nimport { StitchingStrategy } from '../../../stitching/types';\nimport {\n  DbFinalEntitiesRow,\n  DbRefreshStateRow,\n  DbSearchRow,\n} from '../../tables';\nimport { buildEntitySearch } from './buildEntitySearch';\nimport { markDeferredStitchCompleted } from './markDeferredStitchCompleted';\nimport { BATCH_SIZE, generateStableHash } from './util';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\n// See https://github.com/facebook/react/blob/f0cf832e1d0c8544c36aa8b310960885a11a847c/packages/react-dom-bindings/src/shared/sanitizeURL.js\nconst scriptProtocolPattern =\n  // eslint-disable-next-line no-control-regex\n  /^[\\u0000-\\u001F ]*j[\\r\\n\\t]*a[\\r\\n\\t]*v[\\r\\n\\t]*a[\\r\\n\\t]*s[\\r\\n\\t]*c[\\r\\n\\t]*r[\\r\\n\\t]*i[\\r\\n\\t]*p[\\r\\n\\t]*t[\\r\\n\\t]*\\:/i;\n\n/**\n * Performs the act of stitching - to take all of the various outputs from the\n * ingestion process, and stitching them together into the final entity JSON\n * shape.\n */\nexport async function performStitching(options: {\n  knex: Knex | Knex.Transaction;\n  logger: LoggerService;\n  strategy: StitchingStrategy;\n  entityRef: string;\n  stitchTicket?: string;\n}): Promise<'changed' | 'unchanged' | 'abandoned'> {\n  const { knex, logger, entityRef } = options;\n  const stitchTicket = options.stitchTicket ?? uuid();\n\n  const entityResult = await knex<DbRefreshStateRow>('refresh_state')\n    .where({ entity_ref: entityRef })\n    .limit(1)\n    .select('entity_id');\n  if (!entityResult.length) {\n    // Entity does no exist in refresh state table, no stitching required.\n    return 'abandoned';\n  }\n\n  // Insert stitching ticket that will be compared before inserting the final entity.\n  await knex<DbFinalEntitiesRow>('final_entities')\n    .insert({\n      entity_id: entityResult[0].entity_id,\n      hash: '',\n      entity_ref: entityRef,\n      stitch_ticket: stitchTicket,\n    })\n    .onConflict('entity_id')\n    .merge(['stitch_ticket']);\n\n  // Selecting from refresh_state and final_entities should yield exactly\n  // one row (except in abnormal cases where the stitch was invoked for\n  // something that didn't exist at all, in which case it's zero rows).\n  // The join with the temporary incoming_references still gives one row.\n  const [processedResult, relationsResult] = await Promise.all([\n    knex\n      .with('incoming_references', function incomingReferences(builder) {\n        return builder\n          .from('refresh_state_references')\n          .where({ target_entity_ref: entityRef })\n          .count({ count: '*' });\n      })\n      .select({\n        entityId: 'refresh_state.entity_id',\n        processedEntity: 'refresh_state.processed_entity',\n        errors: 'refresh_state.errors',\n        incomingReferenceCount: 'incoming_references.count',\n        previousHash: 'final_entities.hash',\n      })\n      .from('refresh_state')\n      .where({ 'refresh_state.entity_ref': entityRef })\n      .crossJoin(knex.raw('incoming_references'))\n      .leftOuterJoin('final_entities', {\n        'final_entities.entity_id': 'refresh_state.entity_id',\n      }),\n    knex\n      .distinct({\n        relationType: 'type',\n        relationTarget: 'target_entity_ref',\n      })\n      .from('relations')\n      .where({ source_entity_ref: entityRef })\n      .orderBy('relationType', 'asc')\n      .orderBy('relationTarget', 'asc'),\n  ]);\n\n  // If there were no rows returned, it would mean that there was no\n  // matching row even in the refresh_state. This can happen for example\n  // if we emit a relation to something that hasn't been ingested yet.\n  // It's safe to ignore this stitch attempt in that case.\n  if (!processedResult.length) {\n    logger.debug(\n      `Unable to stitch ${entityRef}, item does not exist in refresh state table`,\n    );\n    return 'abandoned';\n  }\n\n  const {\n    entityId,\n    processedEntity,\n    errors,\n    incomingReferenceCount,\n    previousHash,\n  } = processedResult[0];\n\n  // If there was no processed entity in place, the target hasn't been\n  // through the processing steps yet. It's safe to ignore this stitch\n  // attempt in that case, since another stitch will be triggered when\n  // that processing has finished.\n  if (!processedEntity) {\n    logger.debug(\n      `Unable to stitch ${entityRef}, the entity has not yet been processed`,\n    );\n    return 'abandoned';\n  }\n\n  // Grab the processed entity and stitch all of the relevant data into\n  // it\n  const entity = JSON.parse(processedEntity) as AlphaEntity;\n  const isOrphan = Number(incomingReferenceCount) === 0;\n  let statusItems: EntityStatusItem[] = [];\n\n  if (isOrphan) {\n    logger.debug(`${entityRef} is an orphan`);\n    entity.metadata.annotations = {\n      ...entity.metadata.annotations,\n      ['backstage.io/orphan']: 'true',\n    };\n  }\n  if (errors) {\n    const parsedErrors = JSON.parse(errors) as SerializedError[];\n    if (Array.isArray(parsedErrors) && parsedErrors.length) {\n      statusItems = parsedErrors.map(e => ({\n        type: ENTITY_STATUS_CATALOG_PROCESSING_TYPE,\n        level: 'error',\n        message: `${e.name}: ${e.message}`,\n        error: e,\n      }));\n    }\n  }\n  // We opt to do this check here as we otherwise can't guarantee that it will be run after all processors\n  for (const annotation of [ANNOTATION_VIEW_URL, ANNOTATION_EDIT_URL]) {\n    const value = entity.metadata.annotations?.[annotation];\n    if (typeof value === 'string' && scriptProtocolPattern.test(value)) {\n      entity.metadata.annotations![annotation] =\n        'https://backstage.io/annotation-rejected-for-security-reasons';\n    }\n  }\n\n  // TODO: entityRef is lower case and should be uppercase in the final\n  // result\n  entity.relations = relationsResult\n    .filter(row => row.relationType /* exclude null row, if relevant */)\n    .map<EntityRelation>(row => ({\n      type: row.relationType!,\n      targetRef: row.relationTarget!,\n    }));\n  if (statusItems.length) {\n    entity.status = {\n      ...entity.status,\n      items: [...(entity.status?.items ?? []), ...statusItems],\n    };\n  }\n\n  // If the output entity was actually not changed, just abort\n  const hash = generateStableHash(entity);\n  if (hash === previousHash) {\n    logger.debug(`Skipped stitching of ${entityRef}, no changes`);\n    return 'unchanged';\n  }\n\n  entity.metadata.uid = entityId;\n  if (!entity.metadata.etag) {\n    // If the original data source did not have its own etag handling,\n    // use the hash as a good-quality etag\n    entity.metadata.etag = hash;\n  }\n\n  // This may throw if the entity is invalid, so we call it before\n  // the final_entities write, even though we may end up not needing\n  // to write the search index.\n  const searchEntries = buildEntitySearch(entityId, entity);\n\n  const amountOfRowsChanged = await knex<DbFinalEntitiesRow>('final_entities')\n    .update({\n      final_entity: JSON.stringify(entity),\n      hash,\n      last_updated_at: knex.fn.now(),\n      entity_ref: entityRef,\n    })\n    .where('entity_id', entityId)\n    .where('stitch_ticket', stitchTicket)\n    .onConflict('entity_id')\n    .merge(['final_entity', 'hash', 'last_updated_at']);\n\n  const markDeferred = async () => {\n    if (options.strategy.mode !== 'deferred') {\n      return;\n    }\n    await markDeferredStitchCompleted({\n      knex: knex,\n      entityRef,\n      stitchTicket,\n    });\n  };\n\n  if (amountOfRowsChanged === 0) {\n    logger.debug(`Entity ${entityRef} is already stitched, skipping write.`);\n    await markDeferred();\n    return 'abandoned';\n  }\n\n  await knex.transaction(async trx => {\n    await trx<DbSearchRow>('search').where({ entity_id: entityId }).delete();\n    await trx.batchInsert('search', searchEntries, BATCH_SIZE);\n  });\n\n  await markDeferred();\n  return 'changed';\n}\n"],"names":["uuid","ENTITY_STATUS_CATALOG_PROCESSING_TYPE","ANNOTATION_VIEW_URL","ANNOTATION_EDIT_URL","generateStableHash","buildEntitySearch","markDeferredStitchCompleted","BATCH_SIZE"],"mappings":";;;;;;;;;AAsCA,MAAM,qBAAA;AAAA;AAAA,EAEJ;AAAA,CAAA;AAOF,eAAsB,iBAAiB,OAMY,EAAA;AACjD,EAAA,MAAM,EAAE,IAAA,EAAM,MAAQ,EAAA,SAAA,EAAc,GAAA,OAAA;AACpC,EAAM,MAAA,YAAA,GAAe,OAAQ,CAAA,YAAA,IAAgBA,OAAK,EAAA;AAElD,EAAA,MAAM,YAAe,GAAA,MAAM,IAAwB,CAAA,eAAe,EAC/D,KAAM,CAAA,EAAE,UAAY,EAAA,SAAA,EAAW,CAC/B,CAAA,KAAA,CAAM,CAAC,CAAA,CACP,OAAO,WAAW,CAAA;AACrB,EAAI,IAAA,CAAC,aAAa,MAAQ,EAAA;AAExB,IAAO,OAAA,WAAA;AAAA;AAIT,EAAM,MAAA,IAAA,CAAyB,gBAAgB,CAAA,CAC5C,MAAO,CAAA;AAAA,IACN,SAAA,EAAW,YAAa,CAAA,CAAC,CAAE,CAAA,SAAA;AAAA,IAC3B,IAAM,EAAA,EAAA;AAAA,IACN,UAAY,EAAA,SAAA;AAAA,IACZ,aAAe,EAAA;AAAA,GAChB,EACA,UAAW,CAAA,WAAW,EACtB,KAAM,CAAA,CAAC,eAAe,CAAC,CAAA;AAM1B,EAAA,MAAM,CAAC,eAAiB,EAAA,eAAe,CAAI,GAAA,MAAM,QAAQ,GAAI,CAAA;AAAA,IAC3D,IACG,CAAA,IAAA,CAAK,qBAAuB,EAAA,SAAS,mBAAmB,OAAS,EAAA;AAChE,MAAA,OAAO,OACJ,CAAA,IAAA,CAAK,0BAA0B,CAAA,CAC/B,MAAM,EAAE,iBAAA,EAAmB,SAAU,EAAC,CACtC,CAAA,KAAA,CAAM,EAAE,KAAA,EAAO,KAAK,CAAA;AAAA,KACxB,EACA,MAAO,CAAA;AAAA,MACN,QAAU,EAAA,yBAAA;AAAA,MACV,eAAiB,EAAA,gCAAA;AAAA,MACjB,MAAQ,EAAA,sBAAA;AAAA,MACR,sBAAwB,EAAA,2BAAA;AAAA,MACxB,YAAc,EAAA;AAAA,KACf,CACA,CAAA,IAAA,CAAK,eAAe,CACpB,CAAA,KAAA,CAAM,EAAE,0BAA4B,EAAA,SAAA,EAAW,CAAA,CAC/C,UAAU,IAAK,CAAA,GAAA,CAAI,qBAAqB,CAAC,CAAA,CACzC,cAAc,gBAAkB,EAAA;AAAA,MAC/B,0BAA4B,EAAA;AAAA,KAC7B,CAAA;AAAA,IACH,KACG,QAAS,CAAA;AAAA,MACR,YAAc,EAAA,MAAA;AAAA,MACd,cAAgB,EAAA;AAAA,KACjB,CACA,CAAA,IAAA,CAAK,WAAW,CAAA,CAChB,MAAM,EAAE,iBAAA,EAAmB,SAAU,EAAC,EACtC,OAAQ,CAAA,cAAA,EAAgB,KAAK,CAC7B,CAAA,OAAA,CAAQ,kBAAkB,KAAK;AAAA,GACnC,CAAA;AAMD,EAAI,IAAA,CAAC,gBAAgB,MAAQ,EAAA;AAC3B,IAAO,MAAA,CAAA,KAAA;AAAA,MACL,oBAAoB,SAAS,CAAA,4CAAA;AAAA,KAC/B;AACA,IAAO,OAAA,WAAA;AAAA;AAGT,EAAM,MAAA;AAAA,IACJ,QAAA;AAAA,IACA,eAAA;AAAA,IACA,MAAA;AAAA,IACA,sBAAA;AAAA,IACA;AAAA,GACF,GAAI,gBAAgB,CAAC,CAAA;AAMrB,EAAA,IAAI,CAAC,eAAiB,EAAA;AACpB,IAAO,MAAA,CAAA,KAAA;AAAA,MACL,oBAAoB,SAAS,CAAA,uCAAA;AAAA,KAC/B;AACA,IAAO,OAAA,WAAA;AAAA;AAKT,EAAM,MAAA,MAAA,GAAS,IAAK,CAAA,KAAA,CAAM,eAAe,CAAA;AACzC,EAAM,MAAA,QAAA,GAAW,MAAO,CAAA,sBAAsB,CAAM,KAAA,CAAA;AACpD,EAAA,IAAI,cAAkC,EAAC;AAEvC,EAAA,IAAI,QAAU,EAAA;AACZ,IAAO,MAAA,CAAA,KAAA,CAAM,CAAG,EAAA,SAAS,CAAe,aAAA,CAAA,CAAA;AACxC,IAAA,MAAA,CAAO,SAAS,WAAc,GAAA;AAAA,MAC5B,GAAG,OAAO,QAAS,CAAA,WAAA;AAAA,MACnB,CAAC,qBAAqB,GAAG;AAAA,KAC3B;AAAA;AAEF,EAAA,IAAI,MAAQ,EAAA;AACV,IAAM,MAAA,YAAA,GAAe,IAAK,CAAA,KAAA,CAAM,MAAM,CAAA;AACtC,IAAA,IAAI,KAAM,CAAA,OAAA,CAAQ,YAAY,CAAA,IAAK,aAAa,MAAQ,EAAA;AACtD,MAAc,WAAA,GAAA,YAAA,CAAa,IAAI,CAAM,CAAA,MAAA;AAAA,QACnC,IAAM,EAAAC,mDAAA;AAAA,QACN,KAAO,EAAA,OAAA;AAAA,QACP,SAAS,CAAG,EAAA,CAAA,CAAE,IAAI,CAAA,EAAA,EAAK,EAAE,OAAO,CAAA,CAAA;AAAA,QAChC,KAAO,EAAA;AAAA,OACP,CAAA,CAAA;AAAA;AACJ;AAGF,EAAA,KAAA,MAAW,UAAc,IAAA,CAACC,gCAAqB,EAAAC,gCAAmB,CAAG,EAAA;AACnE,IAAA,MAAM,KAAQ,GAAA,MAAA,CAAO,QAAS,CAAA,WAAA,GAAc,UAAU,CAAA;AACtD,IAAA,IAAI,OAAO,KAAU,KAAA,QAAA,IAAY,qBAAsB,CAAA,IAAA,CAAK,KAAK,CAAG,EAAA;AAClE,MAAO,MAAA,CAAA,QAAA,CAAS,WAAa,CAAA,UAAU,CACrC,GAAA,+DAAA;AAAA;AACJ;AAKF,EAAA,MAAA,CAAO,YAAY,eAChB,CAAA,MAAA;AAAA,IAAO,SAAO,GAAI,CAAA;AAAA;AAAA,GAAgD,CAClE,IAAoB,CAAQ,GAAA,MAAA;AAAA,IAC3B,MAAM,GAAI,CAAA,YAAA;AAAA,IACV,WAAW,GAAI,CAAA;AAAA,GACf,CAAA,CAAA;AACJ,EAAA,IAAI,YAAY,MAAQ,EAAA;AACtB,IAAA,MAAA,CAAO,MAAS,GAAA;AAAA,MACd,GAAG,MAAO,CAAA,MAAA;AAAA,MACV,KAAA,EAAO,CAAC,GAAI,MAAA,CAAO,QAAQ,KAAS,IAAA,EAAK,EAAA,GAAG,WAAW;AAAA,KACzD;AAAA;AAIF,EAAM,MAAA,IAAA,GAAOC,wBAAmB,MAAM,CAAA;AACtC,EAAA,IAAI,SAAS,YAAc,EAAA;AACzB,IAAO,MAAA,CAAA,KAAA,CAAM,CAAwB,qBAAA,EAAA,SAAS,CAAc,YAAA,CAAA,CAAA;AAC5D,IAAO,OAAA,WAAA;AAAA;AAGT,EAAA,MAAA,CAAO,SAAS,GAAM,GAAA,QAAA;AACtB,EAAI,IAAA,CAAC,MAAO,CAAA,QAAA,CAAS,IAAM,EAAA;AAGzB,IAAA,MAAA,CAAO,SAAS,IAAO,GAAA,IAAA;AAAA;AAMzB,EAAM,MAAA,aAAA,GAAgBC,mCAAkB,CAAA,QAAA,EAAU,MAAM,CAAA;AAExD,EAAA,MAAM,mBAAsB,GAAA,MAAM,IAAyB,CAAA,gBAAgB,EACxE,MAAO,CAAA;AAAA,IACN,YAAA,EAAc,IAAK,CAAA,SAAA,CAAU,MAAM,CAAA;AAAA,IACnC,IAAA;AAAA,IACA,eAAA,EAAiB,IAAK,CAAA,EAAA,CAAG,GAAI,EAAA;AAAA,IAC7B,UAAY,EAAA;AAAA,GACb,CACA,CAAA,KAAA,CAAM,aAAa,QAAQ,CAAA,CAC3B,MAAM,eAAiB,EAAA,YAAY,CACnC,CAAA,UAAA,CAAW,WAAW,CACtB,CAAA,KAAA,CAAM,CAAC,cAAgB,EAAA,MAAA,EAAQ,iBAAiB,CAAC,CAAA;AAEpD,EAAA,MAAM,eAAe,YAAY;AAC/B,IAAI,IAAA,OAAA,CAAQ,QAAS,CAAA,IAAA,KAAS,UAAY,EAAA;AACxC,MAAA;AAAA;AAEF,IAAA,MAAMC,uDAA4B,CAAA;AAAA,MAChC,IAAA;AAAA,MACA,SAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA,GACH;AAEA,EAAA,IAAI,wBAAwB,CAAG,EAAA;AAC7B,IAAO,MAAA,CAAA,KAAA,CAAM,CAAU,OAAA,EAAA,SAAS,CAAuC,qCAAA,CAAA,CAAA;AACvE,IAAA,MAAM,YAAa,EAAA;AACnB,IAAO,OAAA,WAAA;AAAA;AAGT,EAAM,MAAA,IAAA,CAAK,WAAY,CAAA,OAAM,GAAO,KAAA;AAClC,IAAM,MAAA,GAAA,CAAiB,QAAQ,CAAE,CAAA,KAAA,CAAM,EAAE,SAAW,EAAA,QAAA,EAAU,CAAA,CAAE,MAAO,EAAA;AACvE,IAAA,MAAM,GAAI,CAAA,WAAA,CAAY,QAAU,EAAA,aAAA,EAAeC,eAAU,CAAA;AAAA,GAC1D,CAAA;AAED,EAAA,MAAM,YAAa,EAAA;AACnB,EAAO,OAAA,SAAA;AACT;;;;"}

package/dist/index.d.ts

@@ -9,7 +9,7 @@ import { CatalogProcessor as CatalogProcessor$1, CatalogProcessorEmit as Catalog
 import { Config } from '@backstage/config';
 import { PluginEndpointDiscovery, TokenManager } from '@backstage/backend-common';
 import { GetEntitiesRequest, CatalogApi } from '@backstage/catalog-client';
-import { Permission, PermissionRuleParams, PermissionAuthorizer } from '@backstage/plugin-permission-common';
+import { Permission, PermissionAuthorizer, PermissionRuleParams } from '@backstage/plugin-permission-common';
 import { Router } from 'express';
 import { PermissionRule } from '@backstage/plugin-permission-node';
 import { EventBroker, EventsService } from '@backstage/plugin-events-node';

package/dist/processing/DefaultCatalogProcessingEngine.cjs.js

@@ -72,13 +72,9 @@ class DefaultCatalogProcessingEngine {
       pollingIntervalMs: this.pollingIntervalMs,
       loadTasks: async (count) => {
         try {
-          const { items } = await this.processingDatabase.transaction(
-            async (tx) => {
-              return this.processingDatabase.getProcessableEntities(tx, {
-                processBatchSize: count
-              });
-            }
-          );
+          const { items } = await this.processingDatabase.getProcessableEntities(this.knex, {
+            processBatchSize: count
+          });
           return items;
         } catch (error) {
           this.logger.warn("Failed to load processing items", error);
@@ -195,7 +191,7 @@ class DefaultCatalogProcessingEngine {
               });
               oldRelationSources = new Map(
                 previous.relations.map((r) => [
-                  `${r.source_entity_ref}:${r.type}`,
+                  `${r.source_entity_ref}:${r.type}->${r.target_entity_ref}`,
                   r.source_entity_ref
                 ])
               );
@@ -203,7 +199,11 @@ class DefaultCatalogProcessingEngine {
             const newRelationSources = new Map(
               result.relations.map((relation) => {
                 const sourceEntityRef = catalogModel.stringifyEntityRef(relation.source);
-                return [`${sourceEntityRef}:${relation.type}`, sourceEntityRef];
+                const targetEntityRef = catalogModel.stringifyEntityRef(relation.target);
+                return [
+                  `${sourceEntityRef}:${relation.type}->${targetEntityRef}`,
+                  sourceEntityRef
+                ];
               })
             );
             const setOfThingsToStitch = /* @__PURE__ */ new Set([

package/dist/processing/DefaultCatalogProcessingEngine.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultCatalogProcessingEngine.cjs.js","sources":["../../src/processing/DefaultCatalogProcessingEngine.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  ANNOTATION_LOCATION,\n  Entity,\n  stringifyEntityRef,\n} from '@backstage/catalog-model';\nimport { assertError, serializeError, stringifyError } from '@backstage/errors';\nimport { Hash } from 'crypto';\nimport stableStringify from 'fast-json-stable-stringify';\nimport { Knex } from 'knex';\nimport { metrics, trace } from '@opentelemetry/api';\nimport { ProcessingDatabase, RefreshStateItem } from '../database/types';\nimport { createCounterMetric, createSummaryMetric } from '../util/metrics';\nimport { CatalogProcessingOrchestrator, EntityProcessingResult } from './types';\nimport { Stitcher, stitchingStrategyFromConfig } from '../stitching/types';\nimport { startTaskPipeline } from './TaskPipeline';\nimport { Config } from '@backstage/config';\nimport {\n  addEntityAttributes,\n  TRACER_ID,\n  withActiveSpan,\n} from '../util/opentelemetry';\nimport { deleteOrphanedEntities } from '../database/operations/util/deleteOrphanedEntities';\nimport { EventBroker, EventsService } from '@backstage/plugin-events-node';\nimport { CATALOG_ERRORS_TOPIC } from '../constants';\nimport { LoggerService, SchedulerService } from '@backstage/backend-plugin-api';\n\nconst CACHE_TTL = 5;\n\nconst tracer = trace.getTracer(TRACER_ID);\n\nexport type ProgressTracker = ReturnType<typeof progressTracker>;\n\n// NOTE(freben): Perhaps surprisingly, this class does not implement the\n// CatalogProcessingEngine type. That type is externally visible and its name is\n// the way it is for historic reasons. This class has no particular reason to\n// implement that precise interface; nowadays there are several different\n// engines \"hiding\" behind the CatalogProcessingEngine interface, of which this\n// is just one.\nexport class DefaultCatalogProcessingEngine {\n  private readonly config: Config;\n  private readonly scheduler?: SchedulerService;\n  private readonly logger: LoggerService;\n  private readonly knex: Knex;\n  private readonly processingDatabase: ProcessingDatabase;\n  private readonly orchestrator: CatalogProcessingOrchestrator;\n  private readonly stitcher: Stitcher;\n  private readonly createHash: () => Hash;\n  private readonly pollingIntervalMs: number;\n  private readonly orphanCleanupIntervalMs: number;\n  private readonly onProcessingError?: (event: {\n    unprocessedEntity: Entity;\n    errors: Error[];\n  }) => Promise<void> | void;\n  private readonly tracker: ProgressTracker;\n  private readonly eventBroker?: EventBroker | EventsService;\n\n  private stopFunc?: () => void;\n\n  constructor(options: {\n    config: Config;\n    scheduler?: SchedulerService;\n    logger: LoggerService;\n    knex: Knex;\n    processingDatabase: ProcessingDatabase;\n    orchestrator: CatalogProcessingOrchestrator;\n    stitcher: Stitcher;\n    createHash: () => Hash;\n    pollingIntervalMs?: number;\n    orphanCleanupIntervalMs?: number;\n    onProcessingError?: (event: {\n      unprocessedEntity: Entity;\n      errors: Error[];\n    }) => Promise<void> | void;\n    tracker?: ProgressTracker;\n    eventBroker?: EventBroker | EventsService;\n  }) {\n    this.config = options.config;\n    this.scheduler = options.scheduler;\n    this.logger = options.logger;\n    this.knex = options.knex;\n    this.processingDatabase = options.processingDatabase;\n    this.orchestrator = options.orchestrator;\n    this.stitcher = options.stitcher;\n    this.createHash = options.createHash;\n    this.pollingIntervalMs = options.pollingIntervalMs ?? 1_000;\n    this.orphanCleanupIntervalMs = options.orphanCleanupIntervalMs ?? 30_000;\n    this.onProcessingError = options.onProcessingError;\n    this.tracker = options.tracker ?? progressTracker();\n    this.eventBroker = options.eventBroker;\n\n    this.stopFunc = undefined;\n  }\n\n  async start() {\n    if (this.stopFunc) {\n      throw new Error('Processing engine is already started');\n    }\n\n    const stopPipeline = this.startPipeline();\n    const stopCleanup = this.startOrphanCleanup();\n\n    this.stopFunc = () => {\n      stopPipeline();\n      stopCleanup();\n    };\n  }\n\n  async stop() {\n    if (this.stopFunc) {\n      this.stopFunc();\n      this.stopFunc = undefined;\n    }\n  }\n\n  private startPipeline(): () => void {\n    return startTaskPipeline<RefreshStateItem>({\n      lowWatermark: 5,\n      highWatermark: 10,\n      pollingIntervalMs: this.pollingIntervalMs,\n      loadTasks: async count => {\n        try {\n          const { items } = await this.processingDatabase.transaction(\n            async tx => {\n              return this.processingDatabase.getProcessableEntities(tx, {\n                processBatchSize: count,\n              });\n            },\n          );\n          return items;\n        } catch (error) {\n          this.logger.warn('Failed to load processing items', error);\n          return [];\n        }\n      },\n      processTask: async item => {\n        await withActiveSpan(tracer, 'ProcessingRun', async span => {\n          const track = this.tracker.processStart(item, this.logger);\n          addEntityAttributes(span, item.unprocessedEntity);\n\n          try {\n            const {\n              id,\n              state,\n              unprocessedEntity,\n              entityRef,\n              locationKey,\n              resultHash: previousResultHash,\n            } = item;\n            const result = await this.orchestrator.process({\n              entity: unprocessedEntity,\n              state,\n            });\n\n            track.markProcessorsCompleted(result);\n\n            if (result.ok) {\n              const { ttl: _, ...stateWithoutTtl } = state ?? {};\n              if (\n                stableStringify(stateWithoutTtl) !==\n                stableStringify(result.state)\n              ) {\n                await this.processingDatabase.transaction(async tx => {\n                  await this.processingDatabase.updateEntityCache(tx, {\n                    id,\n                    state: {\n                      ttl: CACHE_TTL,\n                      ...result.state,\n                    },\n                  });\n                });\n              }\n            } else {\n              const maybeTtl = state?.ttl;\n              const ttl = Number.isInteger(maybeTtl) ? (maybeTtl as number) : 0;\n              await this.processingDatabase.transaction(async tx => {\n                await this.processingDatabase.updateEntityCache(tx, {\n                  id,\n                  state: ttl > 0 ? { ...state, ttl: ttl - 1 } : {},\n                });\n              });\n            }\n\n            const location =\n              unprocessedEntity?.metadata?.annotations?.[ANNOTATION_LOCATION];\n            if (result.errors.length) {\n              this.eventBroker?.publish({\n                topic: CATALOG_ERRORS_TOPIC,\n                eventPayload: {\n                  entity: entityRef,\n                  location,\n                  errors: result.errors,\n                },\n              });\n            }\n            const errorsString = JSON.stringify(\n              result.errors.map(e => serializeError(e)),\n            );\n\n            let hashBuilder = this.createHash().update(errorsString);\n\n            if (result.ok) {\n              const { entityRefs: parents } =\n                await this.processingDatabase.transaction(tx =>\n                  this.processingDatabase.listParents(tx, {\n                    entityRef,\n                  }),\n                );\n\n              hashBuilder = hashBuilder\n                .update(stableStringify({ ...result.completedEntity }))\n                .update(stableStringify([...result.deferredEntities]))\n                .update(stableStringify([...result.relations]))\n                .update(stableStringify([...result.refreshKeys]))\n                .update(stableStringify([...parents]));\n            }\n\n            const resultHash = hashBuilder.digest('hex');\n            if (resultHash === previousResultHash) {\n              // If nothing changed in our produced outputs, we cannot have any\n              // significant effect on our surroundings; therefore, we just abort\n              // without any updates / stitching.\n              track.markSuccessfulWithNoChanges();\n              return;\n            }\n\n            // If the result was marked as not OK, it signals that some part of the\n            // processing pipeline threw an exception. This can happen both as part of\n            // non-catastrophic things such as due to validation errors, as well as if\n            // something fatal happens inside the processing for other reasons. In any\n            // case, this means we can't trust that anything in the output is okay. So\n            // just store the errors and trigger a stich so that they become visible to\n            // the outside.\n            if (!result.ok) {\n              // notify the error listener if the entity can not be processed.\n              Promise.resolve(undefined)\n                .then(() =>\n                  this.onProcessingError?.({\n                    unprocessedEntity,\n                    errors: result.errors,\n                  }),\n                )\n                .catch(error => {\n                  this.logger.debug(\n                    `Processing error listener threw an exception, ${stringifyError(\n                      error,\n                    )}`,\n                  );\n                });\n\n              await this.processingDatabase.transaction(async tx => {\n                await this.processingDatabase.updateProcessedEntityErrors(tx, {\n                  id,\n                  errors: errorsString,\n                  resultHash,\n                });\n              });\n\n              await this.stitcher.stitch({\n                entityRefs: [stringifyEntityRef(unprocessedEntity)],\n              });\n\n              track.markSuccessfulWithErrors();\n              return;\n            }\n\n            result.completedEntity.metadata.uid = id;\n            let oldRelationSources: Map<string, string>;\n            await this.processingDatabase.transaction(async tx => {\n              const { previous } =\n                await this.processingDatabase.updateProcessedEntity(tx, {\n                  id,\n                  processedEntity: result.completedEntity,\n                  resultHash,\n                  errors: errorsString,\n                  relations: result.relations,\n                  deferredEntities: result.deferredEntities,\n                  locationKey,\n                  refreshKeys: result.refreshKeys,\n                });\n              oldRelationSources = new Map(\n                previous.relations.map(r => [\n                  `${r.source_entity_ref}:${r.type}`,\n                  r.source_entity_ref,\n                ]),\n              );\n            });\n\n            const newRelationSources = new Map<string, string>(\n              result.relations.map(relation => {\n                const sourceEntityRef = stringifyEntityRef(relation.source);\n                return [`${sourceEntityRef}:${relation.type}`, sourceEntityRef];\n              }),\n            );\n\n            const setOfThingsToStitch = new Set<string>([\n              stringifyEntityRef(result.completedEntity),\n            ]);\n            newRelationSources.forEach((sourceEntityRef, uniqueKey) => {\n              if (!oldRelationSources.has(uniqueKey)) {\n                setOfThingsToStitch.add(sourceEntityRef);\n              }\n            });\n            oldRelationSources!.forEach((sourceEntityRef, uniqueKey) => {\n              if (!newRelationSources.has(uniqueKey)) {\n                setOfThingsToStitch.add(sourceEntityRef);\n              }\n            });\n\n            await this.stitcher.stitch({\n              entityRefs: setOfThingsToStitch,\n            });\n\n            track.markSuccessfulWithChanges();\n          } catch (error) {\n            assertError(error);\n            track.markFailed(error);\n          }\n        });\n      },\n    });\n  }\n\n  private startOrphanCleanup(): () => void {\n    const orphanStrategy =\n      this.config.getOptionalString('catalog.orphanStrategy') ?? 'keep';\n    if (orphanStrategy !== 'delete') {\n      return () => {};\n    }\n\n    const stitchingStrategy = stitchingStrategyFromConfig(this.config);\n\n    const runOnce = async () => {\n      try {\n        const n = await deleteOrphanedEntities({\n          knex: this.knex,\n          strategy: stitchingStrategy,\n        });\n        if (n > 0) {\n          this.logger.info(`Deleted ${n} orphaned entities`);\n        }\n      } catch (error) {\n        this.logger.warn(`Failed to delete orphaned entities`, error);\n      }\n    };\n\n    if (this.scheduler) {\n      const abortController = new AbortController();\n\n      this.scheduler.scheduleTask({\n        id: 'catalog_orphan_cleanup',\n        frequency: { milliseconds: this.orphanCleanupIntervalMs },\n        timeout: { milliseconds: this.orphanCleanupIntervalMs * 0.8 },\n        fn: runOnce,\n        signal: abortController.signal,\n      });\n\n      return () => {\n        abortController.abort();\n      };\n    }\n\n    const intervalKey = setInterval(runOnce, this.orphanCleanupIntervalMs);\n    return () => {\n      clearInterval(intervalKey);\n    };\n  }\n}\n\n// Helps wrap the timing and logging behaviors\nfunction progressTracker() {\n  // prom-client metrics are deprecated in favour of OpenTelemetry metrics.\n  const promProcessedEntities = createCounterMetric({\n    name: 'catalog_processed_entities_count',\n    help: 'Amount of entities processed, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingDuration = createSummaryMetric({\n    name: 'catalog_processing_duration_seconds',\n    help: 'Time spent executing the full processing flow, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessorsDuration = createSummaryMetric({\n    name: 'catalog_processors_duration_seconds',\n    help: 'Time spent executing catalog processors, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingQueueDelay = createSummaryMetric({\n    name: 'catalog_processing_queue_delay_seconds',\n    help: 'The amount of delay between being scheduled for processing, and the start of actually being processed, DEPRECATED, use OpenTelemetry metrics instead',\n  });\n\n  const meter = metrics.getMeter('default');\n  const processedEntities = meter.createCounter(\n    'catalog.processed.entities.count',\n    { description: 'Amount of entities processed' },\n  );\n\n  const processingDuration = meter.createHistogram(\n    'catalog.processing.duration',\n    {\n      description: 'Time spent executing the full processing flow',\n      unit: 'seconds',\n    },\n  );\n\n  const processorsDuration = meter.createHistogram(\n    'catalog.processors.duration',\n    {\n      description: 'Time spent executing catalog processors',\n      unit: 'seconds',\n    },\n  );\n\n  const processingQueueDelay = meter.createHistogram(\n    'catalog.processing.queue.delay',\n    {\n      description:\n        'The amount of delay between being scheduled for processing, and the start of actually being processed',\n      unit: 'seconds',\n    },\n  );\n\n  function processStart(item: RefreshStateItem, logger: LoggerService) {\n    const startTime = process.hrtime();\n    const endOverallTimer = promProcessingDuration.startTimer();\n    const endProcessorsTimer = promProcessorsDuration.startTimer();\n\n    logger.debug(`Processing ${item.entityRef}`);\n\n    if (item.nextUpdateAt) {\n      const seconds = -item.nextUpdateAt.diffNow().as('seconds');\n      promProcessingQueueDelay.observe(seconds);\n      processingQueueDelay.record(seconds);\n    }\n\n    function endTime() {\n      const delta = process.hrtime(startTime);\n      return delta[0] + delta[1] / 1e9;\n    }\n\n    function markProcessorsCompleted(result: EntityProcessingResult) {\n      endProcessorsTimer({ result: result.ok ? 'ok' : 'failed' });\n      processorsDuration.record(endTime(), {\n        result: result.ok ? 'ok' : 'failed',\n      });\n    }\n\n    function markSuccessfulWithNoChanges() {\n      endOverallTimer({ result: 'unchanged' });\n      promProcessedEntities.inc({ result: 'unchanged' }, 1);\n\n      processingDuration.record(endTime(), { result: 'unchanged' });\n      processedEntities.add(1, { result: 'unchanged' });\n    }\n\n    function markSuccessfulWithErrors() {\n      endOverallTimer({ result: 'errors' });\n      promProcessedEntities.inc({ result: 'errors' }, 1);\n\n      processingDuration.record(endTime(), { result: 'errors' });\n      processedEntities.add(1, { result: 'errors' });\n    }\n\n    function markSuccessfulWithChanges() {\n      endOverallTimer({ result: 'changed' });\n      promProcessedEntities.inc({ result: 'changed' }, 1);\n\n      processingDuration.record(endTime(), { result: 'changed' });\n      processedEntities.add(1, { result: 'changed' });\n    }\n\n    function markFailed(error: Error) {\n      promProcessedEntities.inc({ result: 'failed' }, 1);\n      processedEntities.add(1, { result: 'failed' });\n      logger.warn(`Processing of ${item.entityRef} failed`, error);\n    }\n\n    return {\n      markProcessorsCompleted,\n      markSuccessfulWithNoChanges,\n      markSuccessfulWithErrors,\n      markSuccessfulWithChanges,\n      markFailed,\n    };\n  }\n\n  return { processStart };\n}\n"],"names":["trace","TRACER_ID","startTaskPipeline","withActiveSpan","addEntityAttributes","stableStringify","ANNOTATION_LOCATION","CATALOG_ERRORS_TOPIC","serializeError","stringifyError","stringifyEntityRef","assertError","stitchingStrategyFromConfig","deleteOrphanedEntities","createCounterMetric","createSummaryMetric","metrics"],"mappings":";;;;;;;;;;;;;;;;;AA0CA,MAAM,SAAY,GAAA,CAAA;AAElB,MAAM,MAAA,GAASA,SAAM,CAAA,SAAA,CAAUC,uBAAS,CAAA;AAUjC,MAAM,8BAA+B,CAAA;AAAA,EACzB,MAAA;AAAA,EACA,SAAA;AAAA,EACA,MAAA;AAAA,EACA,IAAA;AAAA,EACA,kBAAA;AAAA,EACA,YAAA;AAAA,EACA,QAAA;AAAA,EACA,UAAA;AAAA,EACA,iBAAA;AAAA,EACA,uBAAA;AAAA,EACA,iBAAA;AAAA,EAIA,OAAA;AAAA,EACA,WAAA;AAAA,EAET,QAAA;AAAA,EAER,YAAY,OAiBT,EAAA;AACD,IAAA,IAAA,CAAK,SAAS,OAAQ,CAAA,MAAA;AACtB,IAAA,IAAA,CAAK,YAAY,OAAQ,CAAA,SAAA;AACzB,IAAA,IAAA,CAAK,SAAS,OAAQ,CAAA,MAAA;AACtB,IAAA,IAAA,CAAK,OAAO,OAAQ,CAAA,IAAA;AACpB,IAAA,IAAA,CAAK,qBAAqB,OAAQ,CAAA,kBAAA;AAClC,IAAA,IAAA,CAAK,eAAe,OAAQ,CAAA,YAAA;AAC5B,IAAA,IAAA,CAAK,WAAW,OAAQ,CAAA,QAAA;AACxB,IAAA,IAAA,CAAK,aAAa,OAAQ,CAAA,UAAA;AAC1B,IAAK,IAAA,CAAA,iBAAA,GAAoB,QAAQ,iBAAqB,IAAA,GAAA;AACtD,IAAK,IAAA,CAAA,uBAAA,GAA0B,QAAQ,uBAA2B,IAAA,GAAA;AAClE,IAAA,IAAA,CAAK,oBAAoB,OAAQ,CAAA,iBAAA;AACjC,IAAK,IAAA,CAAA,OAAA,GAAU,OAAQ,CAAA,OAAA,IAAW,eAAgB,EAAA;AAClD,IAAA,IAAA,CAAK,cAAc,OAAQ,CAAA,WAAA;AAE3B,IAAA,IAAA,CAAK,QAAW,GAAA,KAAA,CAAA;AAAA;AAClB,EAEA,MAAM,KAAQ,GAAA;AACZ,IAAA,IAAI,KAAK,QAAU,EAAA;AACjB,MAAM,MAAA,IAAI,MAAM,sCAAsC,CAAA;AAAA;AAGxD,IAAM,MAAA,YAAA,GAAe,KAAK,aAAc,EAAA;AACxC,IAAM,MAAA,WAAA,GAAc,KAAK,kBAAmB,EAAA;AAE5C,IAAA,IAAA,CAAK,WAAW,MAAM;AACpB,MAAa,YAAA,EAAA;AACb,MAAY,WAAA,EAAA;AAAA,KACd;AAAA;AACF,EAEA,MAAM,IAAO,GAAA;AACX,IAAA,IAAI,KAAK,QAAU,EAAA;AACjB,MAAA,IAAA,CAAK,QAAS,EAAA;AACd,MAAA,IAAA,CAAK,QAAW,GAAA,KAAA,CAAA;AAAA;AAClB;AACF,EAEQ,aAA4B,GAAA;AAClC,IAAA,OAAOC,8BAAoC,CAAA;AAAA,MACzC,YAAc,EAAA,CAAA;AAAA,MACd,aAAe,EAAA,EAAA;AAAA,MACf,mBAAmB,IAAK,CAAA,iBAAA;AAAA,MACxB,SAAA,EAAW,OAAM,KAAS,KAAA;AACxB,QAAI,IAAA;AACF,UAAA,MAAM,EAAE,KAAA,EAAU,GAAA,MAAM,KAAK,kBAAmB,CAAA,WAAA;AAAA,YAC9C,OAAM,EAAM,KAAA;AACV,cAAO,OAAA,IAAA,CAAK,kBAAmB,CAAA,sBAAA,CAAuB,EAAI,EAAA;AAAA,gBACxD,gBAAkB,EAAA;AAAA,eACnB,CAAA;AAAA;AACH,WACF;AACA,UAAO,OAAA,KAAA;AAAA,iBACA,KAAO,EAAA;AACd,UAAK,IAAA,CAAA,MAAA,CAAO,IAAK,CAAA,iCAAA,EAAmC,KAAK,CAAA;AACzD,UAAA,OAAO,EAAC;AAAA;AACV,OACF;AAAA,MACA,WAAA,EAAa,OAAM,IAAQ,KAAA;AACzB,QAAA,MAAMC,4BAAe,CAAA,MAAA,EAAQ,eAAiB,EAAA,OAAM,IAAQ,KAAA;AAC1D,UAAA,MAAM,QAAQ,IAAK,CAAA,OAAA,CAAQ,YAAa,CAAA,IAAA,EAAM,KAAK,MAAM,CAAA;AACzD,UAAoBC,iCAAA,CAAA,IAAA,EAAM,KAAK,iBAAiB,CAAA;AAEhD,UAAI,IAAA;AACF,YAAM,MAAA;AAAA,cACJ,EAAA;AAAA,cACA,KAAA;AAAA,cACA,iBAAA;AAAA,cACA,SAAA;AAAA,cACA,WAAA;AAAA,cACA,UAAY,EAAA;AAAA,aACV,GAAA,IAAA;AACJ,YAAA,MAAM,MAAS,GAAA,MAAM,IAAK,CAAA,YAAA,CAAa,OAAQ,CAAA;AAAA,cAC7C,MAAQ,EAAA,iBAAA;AAAA,cACR;AAAA,aACD,CAAA;AAED,YAAA,KAAA,CAAM,wBAAwB,MAAM,CAAA;AAEpC,YAAA,IAAI,OAAO,EAAI,EAAA;AACb,cAAA,MAAM,EAAE,GAAK,EAAA,CAAA,EAAG,GAAG,eAAgB,EAAA,GAAI,SAAS,EAAC;AACjD,cAAA,IACEC,iCAAgB,eAAe,CAAA,KAC/BA,gCAAgB,CAAA,MAAA,CAAO,KAAK,CAC5B,EAAA;AACA,gBAAA,MAAM,IAAK,CAAA,kBAAA,CAAmB,WAAY,CAAA,OAAM,EAAM,KAAA;AACpD,kBAAM,MAAA,IAAA,CAAK,kBAAmB,CAAA,iBAAA,CAAkB,EAAI,EAAA;AAAA,oBAClD,EAAA;AAAA,oBACA,KAAO,EAAA;AAAA,sBACL,GAAK,EAAA,SAAA;AAAA,sBACL,GAAG,MAAO,CAAA;AAAA;AACZ,mBACD,CAAA;AAAA,iBACF,CAAA;AAAA;AACH,aACK,MAAA;AACL,cAAA,MAAM,WAAW,KAAO,EAAA,GAAA;AACxB,cAAA,MAAM,GAAM,GAAA,MAAA,CAAO,SAAU,CAAA,QAAQ,IAAK,QAAsB,GAAA,CAAA;AAChE,cAAA,MAAM,IAAK,CAAA,kBAAA,CAAmB,WAAY,CAAA,OAAM,EAAM,KAAA;AACpD,gBAAM,MAAA,IAAA,CAAK,kBAAmB,CAAA,iBAAA,CAAkB,EAAI,EAAA;AAAA,kBAClD,EAAA;AAAA,kBACA,KAAA,EAAO,GAAM,GAAA,CAAA,GAAI,EAAE,GAAG,OAAO,GAAK,EAAA,GAAA,GAAM,CAAE,EAAA,GAAI;AAAC,iBAChD,CAAA;AAAA,eACF,CAAA;AAAA;AAGH,YAAA,MAAM,QACJ,GAAA,iBAAA,EAAmB,QAAU,EAAA,WAAA,GAAcC,gCAAmB,CAAA;AAChE,YAAI,IAAA,MAAA,CAAO,OAAO,MAAQ,EAAA;AACxB,cAAA,IAAA,CAAK,aAAa,OAAQ,CAAA;AAAA,gBACxB,KAAO,EAAAC,8BAAA;AAAA,gBACP,YAAc,EAAA;AAAA,kBACZ,MAAQ,EAAA,SAAA;AAAA,kBACR,QAAA;AAAA,kBACA,QAAQ,MAAO,CAAA;AAAA;AACjB,eACD,CAAA;AAAA;AAEH,YAAA,MAAM,eAAe,IAAK,CAAA,SAAA;AAAA,cACxB,OAAO,MAAO,CAAA,GAAA,CAAI,CAAK,CAAA,KAAAC,qBAAA,CAAe,CAAC,CAAC;AAAA,aAC1C;AAEA,YAAA,IAAI,WAAc,GAAA,IAAA,CAAK,UAAW,EAAA,CAAE,OAAO,YAAY,CAAA;AAEvD,YAAA,IAAI,OAAO,EAAI,EAAA;AACb,cAAA,MAAM,EAAE,UAAY,EAAA,OAAA,EAClB,GAAA,MAAM,KAAK,kBAAmB,CAAA,WAAA;AAAA,gBAAY,CACxC,EAAA,KAAA,IAAA,CAAK,kBAAmB,CAAA,WAAA,CAAY,EAAI,EAAA;AAAA,kBACtC;AAAA,iBACD;AAAA,eACH;AAEF,cAAA,WAAA,GAAc,YACX,MAAO,CAAAH,gCAAA,CAAgB,EAAE,GAAG,MAAA,CAAO,iBAAiB,CAAC,EACrD,MAAO,CAAAA,gCAAA,CAAgB,CAAC,GAAG,MAAA,CAAO,gBAAgB,CAAC,CAAC,EACpD,MAAO,CAAAA,gCAAA,CAAgB,CAAC,GAAG,MAAA,CAAO,SAAS,CAAC,CAAC,EAC7C,MAAO,CAAAA,gCAAA,CAAgB,CAAC,GAAG,MAAA,CAAO,WAAW,CAAC,CAAC,EAC/C,MAAO,CAAAA,gCAAA,CAAgB,CAAC,GAAG,OAAO,CAAC,CAAC,CAAA;AAAA;AAGzC,YAAM,MAAA,UAAA,GAAa,WAAY,CAAA,MAAA,CAAO,KAAK,CAAA;AAC3C,YAAA,IAAI,eAAe,kBAAoB,EAAA;AAIrC,cAAA,KAAA,CAAM,2BAA4B,EAAA;AAClC,cAAA;AAAA;AAUF,YAAI,IAAA,CAAC,OAAO,EAAI,EAAA;AAEd,cAAQ,OAAA,CAAA,OAAA,CAAQ,MAAS,CACtB,CAAA,IAAA;AAAA,gBAAK,MACJ,KAAK,iBAAoB,GAAA;AAAA,kBACvB,iBAAA;AAAA,kBACA,QAAQ,MAAO,CAAA;AAAA,iBAChB;AAAA,eACH,CACC,MAAM,CAAS,KAAA,KAAA;AACd,gBAAA,IAAA,CAAK,MAAO,CAAA,KAAA;AAAA,kBACV,CAAiD,8CAAA,EAAAI,qBAAA;AAAA,oBAC/C;AAAA,mBACD,CAAA;AAAA,iBACH;AAAA,eACD,CAAA;AAEH,cAAA,MAAM,IAAK,CAAA,kBAAA,CAAmB,WAAY,CAAA,OAAM,EAAM,KAAA;AACpD,gBAAM,MAAA,IAAA,CAAK,kBAAmB,CAAA,2BAAA,CAA4B,EAAI,EAAA;AAAA,kBAC5D,EAAA;AAAA,kBACA,MAAQ,EAAA,YAAA;AAAA,kBACR;AAAA,iBACD,CAAA;AAAA,eACF,CAAA;AAED,cAAM,MAAA,IAAA,CAAK,SAAS,MAAO,CAAA;AAAA,gBACzB,UAAY,EAAA,CAACC,+BAAmB,CAAA,iBAAiB,CAAC;AAAA,eACnD,CAAA;AAED,cAAA,KAAA,CAAM,wBAAyB,EAAA;AAC/B,cAAA;AAAA;AAGF,YAAO,MAAA,CAAA,eAAA,CAAgB,SAAS,GAAM,GAAA,EAAA;AACtC,YAAI,IAAA,kBAAA;AACJ,YAAA,MAAM,IAAK,CAAA,kBAAA,CAAmB,WAAY,CAAA,OAAM,EAAM,KAAA;AACpD,cAAA,MAAM,EAAE,QAAS,EAAA,GACf,MAAM,IAAK,CAAA,kBAAA,CAAmB,sBAAsB,EAAI,EAAA;AAAA,gBACtD,EAAA;AAAA,gBACA,iBAAiB,MAAO,CAAA,eAAA;AAAA,gBACxB,UAAA;AAAA,gBACA,MAAQ,EAAA,YAAA;AAAA,gBACR,WAAW,MAAO,CAAA,SAAA;AAAA,gBAClB,kBAAkB,MAAO,CAAA,gBAAA;AAAA,gBACzB,WAAA;AAAA,gBACA,aAAa,MAAO,CAAA;AAAA,eACrB,CAAA;AACH,cAAA,kBAAA,GAAqB,IAAI,GAAA;AAAA,gBACvB,QAAA,CAAS,SAAU,CAAA,GAAA,CAAI,CAAK,CAAA,KAAA;AAAA,kBAC1B,CAAG,EAAA,CAAA,CAAE,iBAAiB,CAAA,CAAA,EAAI,EAAE,IAAI,CAAA,CAAA;AAAA,kBAChC,CAAE,CAAA;AAAA,iBACH;AAAA,eACH;AAAA,aACD,CAAA;AAED,YAAA,MAAM,qBAAqB,IAAI,GAAA;AAAA,cAC7B,MAAA,CAAO,SAAU,CAAA,GAAA,CAAI,CAAY,QAAA,KAAA;AAC/B,gBAAM,MAAA,eAAA,GAAkBA,+BAAmB,CAAA,QAAA,CAAS,MAAM,CAAA;AAC1D,gBAAA,OAAO,CAAC,CAAG,EAAA,eAAe,IAAI,QAAS,CAAA,IAAI,IAAI,eAAe,CAAA;AAAA,eAC/D;AAAA,aACH;AAEA,YAAM,MAAA,mBAAA,uBAA0B,GAAY,CAAA;AAAA,cAC1CA,+BAAA,CAAmB,OAAO,eAAe;AAAA,aAC1C,CAAA;AACD,YAAmB,kBAAA,CAAA,OAAA,CAAQ,CAAC,eAAA,EAAiB,SAAc,KAAA;AACzD,cAAA,IAAI,CAAC,kBAAA,CAAmB,GAAI,CAAA,SAAS,CAAG,EAAA;AACtC,gBAAA,mBAAA,CAAoB,IAAI,eAAe,CAAA;AAAA;AACzC,aACD,CAAA;AACD,YAAoB,kBAAA,CAAA,OAAA,CAAQ,CAAC,eAAA,EAAiB,SAAc,KAAA;AAC1D,cAAA,IAAI,CAAC,kBAAA,CAAmB,GAAI,CAAA,SAAS,CAAG,EAAA;AACtC,gBAAA,mBAAA,CAAoB,IAAI,eAAe,CAAA;AAAA;AACzC,aACD,CAAA;AAED,YAAM,MAAA,IAAA,CAAK,SAAS,MAAO,CAAA;AAAA,cACzB,UAAY,EAAA;AAAA,aACb,CAAA;AAED,YAAA,KAAA,CAAM,yBAA0B,EAAA;AAAA,mBACzB,KAAO,EAAA;AACd,YAAAC,kBAAA,CAAY,KAAK,CAAA;AACjB,YAAA,KAAA,CAAM,WAAW,KAAK,CAAA;AAAA;AACxB,SACD,CAAA;AAAA;AACH,KACD,CAAA;AAAA;AACH,EAEQ,kBAAiC,GAAA;AACvC,IAAA,MAAM,cACJ,GAAA,IAAA,CAAK,MAAO,CAAA,iBAAA,CAAkB,wBAAwB,CAAK,IAAA,MAAA;AAC7D,IAAA,IAAI,mBAAmB,QAAU,EAAA;AAC/B,MAAA,OAAO,MAAM;AAAA,OAAC;AAAA;AAGhB,IAAM,MAAA,iBAAA,GAAoBC,iCAA4B,CAAA,IAAA,CAAK,MAAM,CAAA;AAEjE,IAAA,MAAM,UAAU,YAAY;AAC1B,MAAI,IAAA;AACF,QAAM,MAAA,CAAA,GAAI,MAAMC,6CAAuB,CAAA;AAAA,UACrC,MAAM,IAAK,CAAA,IAAA;AAAA,UACX,QAAU,EAAA;AAAA,SACX,CAAA;AACD,QAAA,IAAI,IAAI,CAAG,EAAA;AACT,UAAA,IAAA,CAAK,MAAO,CAAA,IAAA,CAAK,CAAW,QAAA,EAAA,CAAC,CAAoB,kBAAA,CAAA,CAAA;AAAA;AACnD,eACO,KAAO,EAAA;AACd,QAAK,IAAA,CAAA,MAAA,CAAO,IAAK,CAAA,CAAA,kCAAA,CAAA,EAAsC,KAAK,CAAA;AAAA;AAC9D,KACF;AAEA,IAAA,IAAI,KAAK,SAAW,EAAA;AAClB,MAAM,MAAA,eAAA,GAAkB,IAAI,eAAgB,EAAA;AAE5C,MAAA,IAAA,CAAK,UAAU,YAAa,CAAA;AAAA,QAC1B,EAAI,EAAA,wBAAA;AAAA,QACJ,SAAW,EAAA,EAAE,YAAc,EAAA,IAAA,CAAK,uBAAwB,EAAA;AAAA,QACxD,OAAS,EAAA,EAAE,YAAc,EAAA,IAAA,CAAK,0BAA0B,GAAI,EAAA;AAAA,QAC5D,EAAI,EAAA,OAAA;AAAA,QACJ,QAAQ,eAAgB,CAAA;AAAA,OACzB,CAAA;AAED,MAAA,OAAO,MAAM;AACX,QAAA,eAAA,CAAgB,KAAM,EAAA;AAAA,OACxB;AAAA;AAGF,IAAA,MAAM,WAAc,GAAA,WAAA,CAAY,OAAS,EAAA,IAAA,CAAK,uBAAuB,CAAA;AACrE,IAAA,OAAO,MAAM;AACX,MAAA,aAAA,CAAc,WAAW,CAAA;AAAA,KAC3B;AAAA;AAEJ;AAGA,SAAS,eAAkB,GAAA;AAEzB,EAAA,MAAM,wBAAwBC,2BAAoB,CAAA;AAAA,IAChD,IAAM,EAAA,kCAAA;AAAA,IACN,IAAM,EAAA,6EAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBC,2BAAoB,CAAA;AAAA,IACjD,IAAM,EAAA,qCAAA;AAAA,IACN,IAAM,EAAA,8FAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBA,2BAAoB,CAAA;AAAA,IACjD,IAAM,EAAA,qCAAA;AAAA,IACN,IAAM,EAAA,wFAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,2BAA2BA,2BAAoB,CAAA;AAAA,IACnD,IAAM,EAAA,wCAAA;AAAA,IACN,IAAM,EAAA;AAAA,GACP,CAAA;AAED,EAAM,MAAA,KAAA,GAAQC,WAAQ,CAAA,QAAA,CAAS,SAAS,CAAA;AACxC,EAAA,MAAM,oBAAoB,KAAM,CAAA,aAAA;AAAA,IAC9B,kCAAA;AAAA,IACA,EAAE,aAAa,8BAA+B;AAAA,GAChD;AAEA,EAAA,MAAM,qBAAqB,KAAM,CAAA,eAAA;AAAA,IAC/B,6BAAA;AAAA,IACA;AAAA,MACE,WAAa,EAAA,+CAAA;AAAA,MACb,IAAM,EAAA;AAAA;AACR,GACF;AAEA,EAAA,MAAM,qBAAqB,KAAM,CAAA,eAAA;AAAA,IAC/B,6BAAA;AAAA,IACA;AAAA,MACE,WAAa,EAAA,yCAAA;AAAA,MACb,IAAM,EAAA;AAAA;AACR,GACF;AAEA,EAAA,MAAM,uBAAuB,KAAM,CAAA,eAAA;AAAA,IACjC,gCAAA;AAAA,IACA;AAAA,MACE,WACE,EAAA,uGAAA;AAAA,MACF,IAAM,EAAA;AAAA;AACR,GACF;AAEA,EAAS,SAAA,YAAA,CAAa,MAAwB,MAAuB,EAAA;AACnE,IAAM,MAAA,SAAA,GAAY,QAAQ,MAAO,EAAA;AACjC,IAAM,MAAA,eAAA,GAAkB,uBAAuB,UAAW,EAAA;AAC1D,IAAM,MAAA,kBAAA,GAAqB,uBAAuB,UAAW,EAAA;AAE7D,IAAA,MAAA,CAAO,KAAM,CAAA,CAAA,WAAA,EAAc,IAAK,CAAA,SAAS,CAAE,CAAA,CAAA;AAE3C,IAAA,IAAI,KAAK,YAAc,EAAA;AACrB,MAAA,MAAM,UAAU,CAAC,IAAA,CAAK,aAAa,OAAQ,EAAA,CAAE,GAAG,SAAS,CAAA;AACzD,MAAA,wBAAA,CAAyB,QAAQ,OAAO,CAAA;AACxC,MAAA,oBAAA,CAAqB,OAAO,OAAO,CAAA;AAAA;AAGrC,IAAA,SAAS,OAAU,GAAA;AACjB,MAAM,MAAA,KAAA,GAAQ,OAAQ,CAAA,MAAA,CAAO,SAAS,CAAA;AACtC,MAAA,OAAO,KAAM,CAAA,CAAC,CAAI,GAAA,KAAA,CAAM,CAAC,CAAI,GAAA,GAAA;AAAA;AAG/B,IAAA,SAAS,wBAAwB,MAAgC,EAAA;AAC/D,MAAA,kBAAA,CAAmB,EAAE,MAAQ,EAAA,MAAA,CAAO,EAAK,GAAA,IAAA,GAAO,UAAU,CAAA;AAC1D,MAAmB,kBAAA,CAAA,MAAA,CAAO,SAAW,EAAA;AAAA,QACnC,MAAA,EAAQ,MAAO,CAAA,EAAA,GAAK,IAAO,GAAA;AAAA,OAC5B,CAAA;AAAA;AAGH,IAAA,SAAS,2BAA8B,GAAA;AACrC,MAAgB,eAAA,CAAA,EAAE,MAAQ,EAAA,WAAA,EAAa,CAAA;AACvC,MAAA,qBAAA,CAAsB,GAAI,CAAA,EAAE,MAAQ,EAAA,WAAA,IAAe,CAAC,CAAA;AAEpD,MAAA,kBAAA,CAAmB,OAAO,OAAQ,EAAA,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAC5D,MAAA,iBAAA,CAAkB,GAAI,CAAA,CAAA,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAAA;AAGlD,IAAA,SAAS,wBAA2B,GAAA;AAClC,MAAgB,eAAA,CAAA,EAAE,MAAQ,EAAA,QAAA,EAAU,CAAA;AACpC,MAAA,qBAAA,CAAsB,GAAI,CAAA,EAAE,MAAQ,EAAA,QAAA,IAAY,CAAC,CAAA;AAEjD,MAAA,kBAAA,CAAmB,OAAO,OAAQ,EAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AACzD,MAAA,iBAAA,CAAkB,GAAI,CAAA,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAAA;AAG/C,IAAA,SAAS,yBAA4B,GAAA;AACnC,MAAgB,eAAA,CAAA,EAAE,MAAQ,EAAA,SAAA,EAAW,CAAA;AACrC,MAAA,qBAAA,CAAsB,GAAI,CAAA,EAAE,MAAQ,EAAA,SAAA,IAAa,CAAC,CAAA;AAElD,MAAA,kBAAA,CAAmB,OAAO,OAAQ,EAAA,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAC1D,MAAA,iBAAA,CAAkB,GAAI,CAAA,CAAA,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAAA;AAGhD,IAAA,SAAS,WAAW,KAAc,EAAA;AAChC,MAAA,qBAAA,CAAsB,GAAI,CAAA,EAAE,MAAQ,EAAA,QAAA,IAAY,CAAC,CAAA;AACjD,MAAA,iBAAA,CAAkB,GAAI,CAAA,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAC7C,MAAA,MAAA,CAAO,IAAK,CAAA,CAAA,cAAA,EAAiB,IAAK,CAAA,SAAS,WAAW,KAAK,CAAA;AAAA;AAG7D,IAAO,OAAA;AAAA,MACL,uBAAA;AAAA,MACA,2BAAA;AAAA,MACA,wBAAA;AAAA,MACA,yBAAA;AAAA,MACA;AAAA,KACF;AAAA;AAGF,EAAA,OAAO,EAAE,YAAa,EAAA;AACxB;;;;"}
+{"version":3,"file":"DefaultCatalogProcessingEngine.cjs.js","sources":["../../src/processing/DefaultCatalogProcessingEngine.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  ANNOTATION_LOCATION,\n  Entity,\n  stringifyEntityRef,\n} from '@backstage/catalog-model';\nimport { assertError, serializeError, stringifyError } from '@backstage/errors';\nimport { Hash } from 'crypto';\nimport stableStringify from 'fast-json-stable-stringify';\nimport { Knex } from 'knex';\nimport { metrics, trace } from '@opentelemetry/api';\nimport { ProcessingDatabase, RefreshStateItem } from '../database/types';\nimport { createCounterMetric, createSummaryMetric } from '../util/metrics';\nimport { CatalogProcessingOrchestrator, EntityProcessingResult } from './types';\nimport { Stitcher, stitchingStrategyFromConfig } from '../stitching/types';\nimport { startTaskPipeline } from './TaskPipeline';\nimport { Config } from '@backstage/config';\nimport {\n  addEntityAttributes,\n  TRACER_ID,\n  withActiveSpan,\n} from '../util/opentelemetry';\nimport { deleteOrphanedEntities } from '../database/operations/util/deleteOrphanedEntities';\nimport { EventBroker, EventsService } from '@backstage/plugin-events-node';\nimport { CATALOG_ERRORS_TOPIC } from '../constants';\nimport { LoggerService, SchedulerService } from '@backstage/backend-plugin-api';\n\nconst CACHE_TTL = 5;\n\nconst tracer = trace.getTracer(TRACER_ID);\n\nexport type ProgressTracker = ReturnType<typeof progressTracker>;\n\n// NOTE(freben): Perhaps surprisingly, this class does not implement the\n// CatalogProcessingEngine type. That type is externally visible and its name is\n// the way it is for historic reasons. This class has no particular reason to\n// implement that precise interface; nowadays there are several different\n// engines \"hiding\" behind the CatalogProcessingEngine interface, of which this\n// is just one.\nexport class DefaultCatalogProcessingEngine {\n  private readonly config: Config;\n  private readonly scheduler?: SchedulerService;\n  private readonly logger: LoggerService;\n  private readonly knex: Knex;\n  private readonly processingDatabase: ProcessingDatabase;\n  private readonly orchestrator: CatalogProcessingOrchestrator;\n  private readonly stitcher: Stitcher;\n  private readonly createHash: () => Hash;\n  private readonly pollingIntervalMs: number;\n  private readonly orphanCleanupIntervalMs: number;\n  private readonly onProcessingError?: (event: {\n    unprocessedEntity: Entity;\n    errors: Error[];\n  }) => Promise<void> | void;\n  private readonly tracker: ProgressTracker;\n  private readonly eventBroker?: EventBroker | EventsService;\n\n  private stopFunc?: () => void;\n\n  constructor(options: {\n    config: Config;\n    scheduler?: SchedulerService;\n    logger: LoggerService;\n    knex: Knex;\n    processingDatabase: ProcessingDatabase;\n    orchestrator: CatalogProcessingOrchestrator;\n    stitcher: Stitcher;\n    createHash: () => Hash;\n    pollingIntervalMs?: number;\n    orphanCleanupIntervalMs?: number;\n    onProcessingError?: (event: {\n      unprocessedEntity: Entity;\n      errors: Error[];\n    }) => Promise<void> | void;\n    tracker?: ProgressTracker;\n    eventBroker?: EventBroker | EventsService;\n  }) {\n    this.config = options.config;\n    this.scheduler = options.scheduler;\n    this.logger = options.logger;\n    this.knex = options.knex;\n    this.processingDatabase = options.processingDatabase;\n    this.orchestrator = options.orchestrator;\n    this.stitcher = options.stitcher;\n    this.createHash = options.createHash;\n    this.pollingIntervalMs = options.pollingIntervalMs ?? 1_000;\n    this.orphanCleanupIntervalMs = options.orphanCleanupIntervalMs ?? 30_000;\n    this.onProcessingError = options.onProcessingError;\n    this.tracker = options.tracker ?? progressTracker();\n    this.eventBroker = options.eventBroker;\n\n    this.stopFunc = undefined;\n  }\n\n  async start() {\n    if (this.stopFunc) {\n      throw new Error('Processing engine is already started');\n    }\n\n    const stopPipeline = this.startPipeline();\n    const stopCleanup = this.startOrphanCleanup();\n\n    this.stopFunc = () => {\n      stopPipeline();\n      stopCleanup();\n    };\n  }\n\n  async stop() {\n    if (this.stopFunc) {\n      this.stopFunc();\n      this.stopFunc = undefined;\n    }\n  }\n\n  private startPipeline(): () => void {\n    return startTaskPipeline<RefreshStateItem>({\n      lowWatermark: 5,\n      highWatermark: 10,\n      pollingIntervalMs: this.pollingIntervalMs,\n      loadTasks: async count => {\n        try {\n          const { items } =\n            await this.processingDatabase.getProcessableEntities(this.knex, {\n              processBatchSize: count,\n            });\n          return items;\n        } catch (error) {\n          this.logger.warn('Failed to load processing items', error);\n          return [];\n        }\n      },\n      processTask: async item => {\n        await withActiveSpan(tracer, 'ProcessingRun', async span => {\n          const track = this.tracker.processStart(item, this.logger);\n          addEntityAttributes(span, item.unprocessedEntity);\n\n          try {\n            const {\n              id,\n              state,\n              unprocessedEntity,\n              entityRef,\n              locationKey,\n              resultHash: previousResultHash,\n            } = item;\n            const result = await this.orchestrator.process({\n              entity: unprocessedEntity,\n              state,\n            });\n\n            track.markProcessorsCompleted(result);\n\n            if (result.ok) {\n              const { ttl: _, ...stateWithoutTtl } = state ?? {};\n              if (\n                stableStringify(stateWithoutTtl) !==\n                stableStringify(result.state)\n              ) {\n                await this.processingDatabase.transaction(async tx => {\n                  await this.processingDatabase.updateEntityCache(tx, {\n                    id,\n                    state: {\n                      ttl: CACHE_TTL,\n                      ...result.state,\n                    },\n                  });\n                });\n              }\n            } else {\n              const maybeTtl = state?.ttl;\n              const ttl = Number.isInteger(maybeTtl) ? (maybeTtl as number) : 0;\n              await this.processingDatabase.transaction(async tx => {\n                await this.processingDatabase.updateEntityCache(tx, {\n                  id,\n                  state: ttl > 0 ? { ...state, ttl: ttl - 1 } : {},\n                });\n              });\n            }\n\n            const location =\n              unprocessedEntity?.metadata?.annotations?.[ANNOTATION_LOCATION];\n            if (result.errors.length) {\n              this.eventBroker?.publish({\n                topic: CATALOG_ERRORS_TOPIC,\n                eventPayload: {\n                  entity: entityRef,\n                  location,\n                  errors: result.errors,\n                },\n              });\n            }\n            const errorsString = JSON.stringify(\n              result.errors.map(e => serializeError(e)),\n            );\n\n            let hashBuilder = this.createHash().update(errorsString);\n\n            if (result.ok) {\n              const { entityRefs: parents } =\n                await this.processingDatabase.transaction(tx =>\n                  this.processingDatabase.listParents(tx, {\n                    entityRef,\n                  }),\n                );\n\n              hashBuilder = hashBuilder\n                .update(stableStringify({ ...result.completedEntity }))\n                .update(stableStringify([...result.deferredEntities]))\n                .update(stableStringify([...result.relations]))\n                .update(stableStringify([...result.refreshKeys]))\n                .update(stableStringify([...parents]));\n            }\n\n            const resultHash = hashBuilder.digest('hex');\n            if (resultHash === previousResultHash) {\n              // If nothing changed in our produced outputs, we cannot have any\n              // significant effect on our surroundings; therefore, we just abort\n              // without any updates / stitching.\n              track.markSuccessfulWithNoChanges();\n              return;\n            }\n\n            // If the result was marked as not OK, it signals that some part of the\n            // processing pipeline threw an exception. This can happen both as part of\n            // non-catastrophic things such as due to validation errors, as well as if\n            // something fatal happens inside the processing for other reasons. In any\n            // case, this means we can't trust that anything in the output is okay. So\n            // just store the errors and trigger a stich so that they become visible to\n            // the outside.\n            if (!result.ok) {\n              // notify the error listener if the entity can not be processed.\n              Promise.resolve(undefined)\n                .then(() =>\n                  this.onProcessingError?.({\n                    unprocessedEntity,\n                    errors: result.errors,\n                  }),\n                )\n                .catch(error => {\n                  this.logger.debug(\n                    `Processing error listener threw an exception, ${stringifyError(\n                      error,\n                    )}`,\n                  );\n                });\n\n              await this.processingDatabase.transaction(async tx => {\n                await this.processingDatabase.updateProcessedEntityErrors(tx, {\n                  id,\n                  errors: errorsString,\n                  resultHash,\n                });\n              });\n\n              await this.stitcher.stitch({\n                entityRefs: [stringifyEntityRef(unprocessedEntity)],\n              });\n\n              track.markSuccessfulWithErrors();\n              return;\n            }\n\n            result.completedEntity.metadata.uid = id;\n            let oldRelationSources: Map<string, string>;\n            await this.processingDatabase.transaction(async tx => {\n              const { previous } =\n                await this.processingDatabase.updateProcessedEntity(tx, {\n                  id,\n                  processedEntity: result.completedEntity,\n                  resultHash,\n                  errors: errorsString,\n                  relations: result.relations,\n                  deferredEntities: result.deferredEntities,\n                  locationKey,\n                  refreshKeys: result.refreshKeys,\n                });\n              oldRelationSources = new Map(\n                previous.relations.map(r => [\n                  `${r.source_entity_ref}:${r.type}->${r.target_entity_ref}`,\n                  r.source_entity_ref,\n                ]),\n              );\n            });\n\n            const newRelationSources = new Map<string, string>(\n              result.relations.map(relation => {\n                const sourceEntityRef = stringifyEntityRef(relation.source);\n                const targetEntityRef = stringifyEntityRef(relation.target);\n                return [\n                  `${sourceEntityRef}:${relation.type}->${targetEntityRef}`,\n                  sourceEntityRef,\n                ];\n              }),\n            );\n\n            const setOfThingsToStitch = new Set<string>([\n              stringifyEntityRef(result.completedEntity),\n            ]);\n            newRelationSources.forEach((sourceEntityRef, uniqueKey) => {\n              if (!oldRelationSources.has(uniqueKey)) {\n                setOfThingsToStitch.add(sourceEntityRef);\n              }\n            });\n            oldRelationSources!.forEach((sourceEntityRef, uniqueKey) => {\n              if (!newRelationSources.has(uniqueKey)) {\n                setOfThingsToStitch.add(sourceEntityRef);\n              }\n            });\n\n            await this.stitcher.stitch({\n              entityRefs: setOfThingsToStitch,\n            });\n\n            track.markSuccessfulWithChanges();\n          } catch (error) {\n            assertError(error);\n            track.markFailed(error);\n          }\n        });\n      },\n    });\n  }\n\n  private startOrphanCleanup(): () => void {\n    const orphanStrategy =\n      this.config.getOptionalString('catalog.orphanStrategy') ?? 'keep';\n    if (orphanStrategy !== 'delete') {\n      return () => {};\n    }\n\n    const stitchingStrategy = stitchingStrategyFromConfig(this.config);\n\n    const runOnce = async () => {\n      try {\n        const n = await deleteOrphanedEntities({\n          knex: this.knex,\n          strategy: stitchingStrategy,\n        });\n        if (n > 0) {\n          this.logger.info(`Deleted ${n} orphaned entities`);\n        }\n      } catch (error) {\n        this.logger.warn(`Failed to delete orphaned entities`, error);\n      }\n    };\n\n    if (this.scheduler) {\n      const abortController = new AbortController();\n\n      this.scheduler.scheduleTask({\n        id: 'catalog_orphan_cleanup',\n        frequency: { milliseconds: this.orphanCleanupIntervalMs },\n        timeout: { milliseconds: this.orphanCleanupIntervalMs * 0.8 },\n        fn: runOnce,\n        signal: abortController.signal,\n      });\n\n      return () => {\n        abortController.abort();\n      };\n    }\n\n    const intervalKey = setInterval(runOnce, this.orphanCleanupIntervalMs);\n    return () => {\n      clearInterval(intervalKey);\n    };\n  }\n}\n\n// Helps wrap the timing and logging behaviors\nfunction progressTracker() {\n  // prom-client metrics are deprecated in favour of OpenTelemetry metrics.\n  const promProcessedEntities = createCounterMetric({\n    name: 'catalog_processed_entities_count',\n    help: 'Amount of entities processed, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingDuration = createSummaryMetric({\n    name: 'catalog_processing_duration_seconds',\n    help: 'Time spent executing the full processing flow, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessorsDuration = createSummaryMetric({\n    name: 'catalog_processors_duration_seconds',\n    help: 'Time spent executing catalog processors, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingQueueDelay = createSummaryMetric({\n    name: 'catalog_processing_queue_delay_seconds',\n    help: 'The amount of delay between being scheduled for processing, and the start of actually being processed, DEPRECATED, use OpenTelemetry metrics instead',\n  });\n\n  const meter = metrics.getMeter('default');\n  const processedEntities = meter.createCounter(\n    'catalog.processed.entities.count',\n    { description: 'Amount of entities processed' },\n  );\n\n  const processingDuration = meter.createHistogram(\n    'catalog.processing.duration',\n    {\n      description: 'Time spent executing the full processing flow',\n      unit: 'seconds',\n    },\n  );\n\n  const processorsDuration = meter.createHistogram(\n    'catalog.processors.duration',\n    {\n      description: 'Time spent executing catalog processors',\n      unit: 'seconds',\n    },\n  );\n\n  const processingQueueDelay = meter.createHistogram(\n    'catalog.processing.queue.delay',\n    {\n      description:\n        'The amount of delay between being scheduled for processing, and the start of actually being processed',\n      unit: 'seconds',\n    },\n  );\n\n  function processStart(item: RefreshStateItem, logger: LoggerService) {\n    const startTime = process.hrtime();\n    const endOverallTimer = promProcessingDuration.startTimer();\n    const endProcessorsTimer = promProcessorsDuration.startTimer();\n\n    logger.debug(`Processing ${item.entityRef}`);\n\n    if (item.nextUpdateAt) {\n      const seconds = -item.nextUpdateAt.diffNow().as('seconds');\n      promProcessingQueueDelay.observe(seconds);\n      processingQueueDelay.record(seconds);\n    }\n\n    function endTime() {\n      const delta = process.hrtime(startTime);\n      return delta[0] + delta[1] / 1e9;\n    }\n\n    function markProcessorsCompleted(result: EntityProcessingResult) {\n      endProcessorsTimer({ result: result.ok ? 'ok' : 'failed' });\n      processorsDuration.record(endTime(), {\n        result: result.ok ? 'ok' : 'failed',\n      });\n    }\n\n    function markSuccessfulWithNoChanges() {\n      endOverallTimer({ result: 'unchanged' });\n      promProcessedEntities.inc({ result: 'unchanged' }, 1);\n\n      processingDuration.record(endTime(), { result: 'unchanged' });\n      processedEntities.add(1, { result: 'unchanged' });\n    }\n\n    function markSuccessfulWithErrors() {\n      endOverallTimer({ result: 'errors' });\n      promProcessedEntities.inc({ result: 'errors' }, 1);\n\n      processingDuration.record(endTime(), { result: 'errors' });\n      processedEntities.add(1, { result: 'errors' });\n    }\n\n    function markSuccessfulWithChanges() {\n      endOverallTimer({ result: 'changed' });\n      promProcessedEntities.inc({ result: 'changed' }, 1);\n\n      processingDuration.record(endTime(), { result: 'changed' });\n      processedEntities.add(1, { result: 'changed' });\n    }\n\n    function markFailed(error: Error) {\n      promProcessedEntities.inc({ result: 'failed' }, 1);\n      processedEntities.add(1, { result: 'failed' });\n      logger.warn(`Processing of ${item.entityRef} failed`, error);\n    }\n\n    return {\n      markProcessorsCompleted,\n      markSuccessfulWithNoChanges,\n      markSuccessfulWithErrors,\n      markSuccessfulWithChanges,\n      markFailed,\n    };\n  }\n\n  return { processStart };\n}\n"],"names":["trace","TRACER_ID","startTaskPipeline","withActiveSpan","addEntityAttributes","stableStringify","ANNOTATION_LOCATION","CATALOG_ERRORS_TOPIC","serializeError","stringifyError","stringifyEntityRef","assertError","stitchingStrategyFromConfig","deleteOrphanedEntities","createCounterMetric","createSummaryMetric","metrics"],"mappings":";;;;;;;;;;;;;;;;;AA0CA,MAAM,SAAY,GAAA,CAAA;AAElB,MAAM,MAAA,GAASA,SAAM,CAAA,SAAA,CAAUC,uBAAS,CAAA;AAUjC,MAAM,8BAA+B,CAAA;AAAA,EACzB,MAAA;AAAA,EACA,SAAA;AAAA,EACA,MAAA;AAAA,EACA,IAAA;AAAA,EACA,kBAAA;AAAA,EACA,YAAA;AAAA,EACA,QAAA;AAAA,EACA,UAAA;AAAA,EACA,iBAAA;AAAA,EACA,uBAAA;AAAA,EACA,iBAAA;AAAA,EAIA,OAAA;AAAA,EACA,WAAA;AAAA,EAET,QAAA;AAAA,EAER,YAAY,OAiBT,EAAA;AACD,IAAA,IAAA,CAAK,SAAS,OAAQ,CAAA,MAAA;AACtB,IAAA,IAAA,CAAK,YAAY,OAAQ,CAAA,SAAA;AACzB,IAAA,IAAA,CAAK,SAAS,OAAQ,CAAA,MAAA;AACtB,IAAA,IAAA,CAAK,OAAO,OAAQ,CAAA,IAAA;AACpB,IAAA,IAAA,CAAK,qBAAqB,OAAQ,CAAA,kBAAA;AAClC,IAAA,IAAA,CAAK,eAAe,OAAQ,CAAA,YAAA;AAC5B,IAAA,IAAA,CAAK,WAAW,OAAQ,CAAA,QAAA;AACxB,IAAA,IAAA,CAAK,aAAa,OAAQ,CAAA,UAAA;AAC1B,IAAK,IAAA,CAAA,iBAAA,GAAoB,QAAQ,iBAAqB,IAAA,GAAA;AACtD,IAAK,IAAA,CAAA,uBAAA,GAA0B,QAAQ,uBAA2B,IAAA,GAAA;AAClE,IAAA,IAAA,CAAK,oBAAoB,OAAQ,CAAA,iBAAA;AACjC,IAAK,IAAA,CAAA,OAAA,GAAU,OAAQ,CAAA,OAAA,IAAW,eAAgB,EAAA;AAClD,IAAA,IAAA,CAAK,cAAc,OAAQ,CAAA,WAAA;AAE3B,IAAA,IAAA,CAAK,QAAW,GAAA,KAAA,CAAA;AAAA;AAClB,EAEA,MAAM,KAAQ,GAAA;AACZ,IAAA,IAAI,KAAK,QAAU,EAAA;AACjB,MAAM,MAAA,IAAI,MAAM,sCAAsC,CAAA;AAAA;AAGxD,IAAM,MAAA,YAAA,GAAe,KAAK,aAAc,EAAA;AACxC,IAAM,MAAA,WAAA,GAAc,KAAK,kBAAmB,EAAA;AAE5C,IAAA,IAAA,CAAK,WAAW,MAAM;AACpB,MAAa,YAAA,EAAA;AACb,MAAY,WAAA,EAAA;AAAA,KACd;AAAA;AACF,EAEA,MAAM,IAAO,GAAA;AACX,IAAA,IAAI,KAAK,QAAU,EAAA;AACjB,MAAA,IAAA,CAAK,QAAS,EAAA;AACd,MAAA,IAAA,CAAK,QAAW,GAAA,KAAA,CAAA;AAAA;AAClB;AACF,EAEQ,aAA4B,GAAA;AAClC,IAAA,OAAOC,8BAAoC,CAAA;AAAA,MACzC,YAAc,EAAA,CAAA;AAAA,MACd,aAAe,EAAA,EAAA;AAAA,MACf,mBAAmB,IAAK,CAAA,iBAAA;AAAA,MACxB,SAAA,EAAW,OAAM,KAAS,KAAA;AACxB,QAAI,IAAA;AACF,UAAM,MAAA,EAAE,OACN,GAAA,MAAM,KAAK,kBAAmB,CAAA,sBAAA,CAAuB,KAAK,IAAM,EAAA;AAAA,YAC9D,gBAAkB,EAAA;AAAA,WACnB,CAAA;AACH,UAAO,OAAA,KAAA;AAAA,iBACA,KAAO,EAAA;AACd,UAAK,IAAA,CAAA,MAAA,CAAO,IAAK,CAAA,iCAAA,EAAmC,KAAK,CAAA;AACzD,UAAA,OAAO,EAAC;AAAA;AACV,OACF;AAAA,MACA,WAAA,EAAa,OAAM,IAAQ,KAAA;AACzB,QAAA,MAAMC,4BAAe,CAAA,MAAA,EAAQ,eAAiB,EAAA,OAAM,IAAQ,KAAA;AAC1D,UAAA,MAAM,QAAQ,IAAK,CAAA,OAAA,CAAQ,YAAa,CAAA,IAAA,EAAM,KAAK,MAAM,CAAA;AACzD,UAAoBC,iCAAA,CAAA,IAAA,EAAM,KAAK,iBAAiB,CAAA;AAEhD,UAAI,IAAA;AACF,YAAM,MAAA;AAAA,cACJ,EAAA;AAAA,cACA,KAAA;AAAA,cACA,iBAAA;AAAA,cACA,SAAA;AAAA,cACA,WAAA;AAAA,cACA,UAAY,EAAA;AAAA,aACV,GAAA,IAAA;AACJ,YAAA,MAAM,MAAS,GAAA,MAAM,IAAK,CAAA,YAAA,CAAa,OAAQ,CAAA;AAAA,cAC7C,MAAQ,EAAA,iBAAA;AAAA,cACR;AAAA,aACD,CAAA;AAED,YAAA,KAAA,CAAM,wBAAwB,MAAM,CAAA;AAEpC,YAAA,IAAI,OAAO,EAAI,EAAA;AACb,cAAA,MAAM,EAAE,GAAK,EAAA,CAAA,EAAG,GAAG,eAAgB,EAAA,GAAI,SAAS,EAAC;AACjD,cAAA,IACEC,iCAAgB,eAAe,CAAA,KAC/BA,gCAAgB,CAAA,MAAA,CAAO,KAAK,CAC5B,EAAA;AACA,gBAAA,MAAM,IAAK,CAAA,kBAAA,CAAmB,WAAY,CAAA,OAAM,EAAM,KAAA;AACpD,kBAAM,MAAA,IAAA,CAAK,kBAAmB,CAAA,iBAAA,CAAkB,EAAI,EAAA;AAAA,oBAClD,EAAA;AAAA,oBACA,KAAO,EAAA;AAAA,sBACL,GAAK,EAAA,SAAA;AAAA,sBACL,GAAG,MAAO,CAAA;AAAA;AACZ,mBACD,CAAA;AAAA,iBACF,CAAA;AAAA;AACH,aACK,MAAA;AACL,cAAA,MAAM,WAAW,KAAO,EAAA,GAAA;AACxB,cAAA,MAAM,GAAM,GAAA,MAAA,CAAO,SAAU,CAAA,QAAQ,IAAK,QAAsB,GAAA,CAAA;AAChE,cAAA,MAAM,IAAK,CAAA,kBAAA,CAAmB,WAAY,CAAA,OAAM,EAAM,KAAA;AACpD,gBAAM,MAAA,IAAA,CAAK,kBAAmB,CAAA,iBAAA,CAAkB,EAAI,EAAA;AAAA,kBAClD,EAAA;AAAA,kBACA,KAAA,EAAO,GAAM,GAAA,CAAA,GAAI,EAAE,GAAG,OAAO,GAAK,EAAA,GAAA,GAAM,CAAE,EAAA,GAAI;AAAC,iBAChD,CAAA;AAAA,eACF,CAAA;AAAA;AAGH,YAAA,MAAM,QACJ,GAAA,iBAAA,EAAmB,QAAU,EAAA,WAAA,GAAcC,gCAAmB,CAAA;AAChE,YAAI,IAAA,MAAA,CAAO,OAAO,MAAQ,EAAA;AACxB,cAAA,IAAA,CAAK,aAAa,OAAQ,CAAA;AAAA,gBACxB,KAAO,EAAAC,8BAAA;AAAA,gBACP,YAAc,EAAA;AAAA,kBACZ,MAAQ,EAAA,SAAA;AAAA,kBACR,QAAA;AAAA,kBACA,QAAQ,MAAO,CAAA;AAAA;AACjB,eACD,CAAA;AAAA;AAEH,YAAA,MAAM,eAAe,IAAK,CAAA,SAAA;AAAA,cACxB,OAAO,MAAO,CAAA,GAAA,CAAI,CAAK,CAAA,KAAAC,qBAAA,CAAe,CAAC,CAAC;AAAA,aAC1C;AAEA,YAAA,IAAI,WAAc,GAAA,IAAA,CAAK,UAAW,EAAA,CAAE,OAAO,YAAY,CAAA;AAEvD,YAAA,IAAI,OAAO,EAAI,EAAA;AACb,cAAA,MAAM,EAAE,UAAY,EAAA,OAAA,EAClB,GAAA,MAAM,KAAK,kBAAmB,CAAA,WAAA;AAAA,gBAAY,CACxC,EAAA,KAAA,IAAA,CAAK,kBAAmB,CAAA,WAAA,CAAY,EAAI,EAAA;AAAA,kBACtC;AAAA,iBACD;AAAA,eACH;AAEF,cAAA,WAAA,GAAc,YACX,MAAO,CAAAH,gCAAA,CAAgB,EAAE,GAAG,MAAA,CAAO,iBAAiB,CAAC,EACrD,MAAO,CAAAA,gCAAA,CAAgB,CAAC,GAAG,MAAA,CAAO,gBAAgB,CAAC,CAAC,EACpD,MAAO,CAAAA,gCAAA,CAAgB,CAAC,GAAG,MAAA,CAAO,SAAS,CAAC,CAAC,EAC7C,MAAO,CAAAA,gCAAA,CAAgB,CAAC,GAAG,MAAA,CAAO,WAAW,CAAC,CAAC,EAC/C,MAAO,CAAAA,gCAAA,CAAgB,CAAC,GAAG,OAAO,CAAC,CAAC,CAAA;AAAA;AAGzC,YAAM,MAAA,UAAA,GAAa,WAAY,CAAA,MAAA,CAAO,KAAK,CAAA;AAC3C,YAAA,IAAI,eAAe,kBAAoB,EAAA;AAIrC,cAAA,KAAA,CAAM,2BAA4B,EAAA;AAClC,cAAA;AAAA;AAUF,YAAI,IAAA,CAAC,OAAO,EAAI,EAAA;AAEd,cAAQ,OAAA,CAAA,OAAA,CAAQ,MAAS,CACtB,CAAA,IAAA;AAAA,gBAAK,MACJ,KAAK,iBAAoB,GAAA;AAAA,kBACvB,iBAAA;AAAA,kBACA,QAAQ,MAAO,CAAA;AAAA,iBAChB;AAAA,eACH,CACC,MAAM,CAAS,KAAA,KAAA;AACd,gBAAA,IAAA,CAAK,MAAO,CAAA,KAAA;AAAA,kBACV,CAAiD,8CAAA,EAAAI,qBAAA;AAAA,oBAC/C;AAAA,mBACD,CAAA;AAAA,iBACH;AAAA,eACD,CAAA;AAEH,cAAA,MAAM,IAAK,CAAA,kBAAA,CAAmB,WAAY,CAAA,OAAM,EAAM,KAAA;AACpD,gBAAM,MAAA,IAAA,CAAK,kBAAmB,CAAA,2BAAA,CAA4B,EAAI,EAAA;AAAA,kBAC5D,EAAA;AAAA,kBACA,MAAQ,EAAA,YAAA;AAAA,kBACR;AAAA,iBACD,CAAA;AAAA,eACF,CAAA;AAED,cAAM,MAAA,IAAA,CAAK,SAAS,MAAO,CAAA;AAAA,gBACzB,UAAY,EAAA,CAACC,+BAAmB,CAAA,iBAAiB,CAAC;AAAA,eACnD,CAAA;AAED,cAAA,KAAA,CAAM,wBAAyB,EAAA;AAC/B,cAAA;AAAA;AAGF,YAAO,MAAA,CAAA,eAAA,CAAgB,SAAS,GAAM,GAAA,EAAA;AACtC,YAAI,IAAA,kBAAA;AACJ,YAAA,MAAM,IAAK,CAAA,kBAAA,CAAmB,WAAY,CAAA,OAAM,EAAM,KAAA;AACpD,cAAA,MAAM,EAAE,QAAS,EAAA,GACf,MAAM,IAAK,CAAA,kBAAA,CAAmB,sBAAsB,EAAI,EAAA;AAAA,gBACtD,EAAA;AAAA,gBACA,iBAAiB,MAAO,CAAA,eAAA;AAAA,gBACxB,UAAA;AAAA,gBACA,MAAQ,EAAA,YAAA;AAAA,gBACR,WAAW,MAAO,CAAA,SAAA;AAAA,gBAClB,kBAAkB,MAAO,CAAA,gBAAA;AAAA,gBACzB,WAAA;AAAA,gBACA,aAAa,MAAO,CAAA;AAAA,eACrB,CAAA;AACH,cAAA,kBAAA,GAAqB,IAAI,GAAA;AAAA,gBACvB,QAAA,CAAS,SAAU,CAAA,GAAA,CAAI,CAAK,CAAA,KAAA;AAAA,kBAC1B,CAAA,EAAG,EAAE,iBAAiB,CAAA,CAAA,EAAI,EAAE,IAAI,CAAA,EAAA,EAAK,EAAE,iBAAiB,CAAA,CAAA;AAAA,kBACxD,CAAE,CAAA;AAAA,iBACH;AAAA,eACH;AAAA,aACD,CAAA;AAED,YAAA,MAAM,qBAAqB,IAAI,GAAA;AAAA,cAC7B,MAAA,CAAO,SAAU,CAAA,GAAA,CAAI,CAAY,QAAA,KAAA;AAC/B,gBAAM,MAAA,eAAA,GAAkBA,+BAAmB,CAAA,QAAA,CAAS,MAAM,CAAA;AAC1D,gBAAM,MAAA,eAAA,GAAkBA,+BAAmB,CAAA,QAAA,CAAS,MAAM,CAAA;AAC1D,gBAAO,OAAA;AAAA,kBACL,GAAG,eAAe,CAAA,CAAA,EAAI,QAAS,CAAA,IAAI,KAAK,eAAe,CAAA,CAAA;AAAA,kBACvD;AAAA,iBACF;AAAA,eACD;AAAA,aACH;AAEA,YAAM,MAAA,mBAAA,uBAA0B,GAAY,CAAA;AAAA,cAC1CA,+BAAA,CAAmB,OAAO,eAAe;AAAA,aAC1C,CAAA;AACD,YAAmB,kBAAA,CAAA,OAAA,CAAQ,CAAC,eAAA,EAAiB,SAAc,KAAA;AACzD,cAAA,IAAI,CAAC,kBAAA,CAAmB,GAAI,CAAA,SAAS,CAAG,EAAA;AACtC,gBAAA,mBAAA,CAAoB,IAAI,eAAe,CAAA;AAAA;AACzC,aACD,CAAA;AACD,YAAoB,kBAAA,CAAA,OAAA,CAAQ,CAAC,eAAA,EAAiB,SAAc,KAAA;AAC1D,cAAA,IAAI,CAAC,kBAAA,CAAmB,GAAI,CAAA,SAAS,CAAG,EAAA;AACtC,gBAAA,mBAAA,CAAoB,IAAI,eAAe,CAAA;AAAA;AACzC,aACD,CAAA;AAED,YAAM,MAAA,IAAA,CAAK,SAAS,MAAO,CAAA;AAAA,cACzB,UAAY,EAAA;AAAA,aACb,CAAA;AAED,YAAA,KAAA,CAAM,yBAA0B,EAAA;AAAA,mBACzB,KAAO,EAAA;AACd,YAAAC,kBAAA,CAAY,KAAK,CAAA;AACjB,YAAA,KAAA,CAAM,WAAW,KAAK,CAAA;AAAA;AACxB,SACD,CAAA;AAAA;AACH,KACD,CAAA;AAAA;AACH,EAEQ,kBAAiC,GAAA;AACvC,IAAA,MAAM,cACJ,GAAA,IAAA,CAAK,MAAO,CAAA,iBAAA,CAAkB,wBAAwB,CAAK,IAAA,MAAA;AAC7D,IAAA,IAAI,mBAAmB,QAAU,EAAA;AAC/B,MAAA,OAAO,MAAM;AAAA,OAAC;AAAA;AAGhB,IAAM,MAAA,iBAAA,GAAoBC,iCAA4B,CAAA,IAAA,CAAK,MAAM,CAAA;AAEjE,IAAA,MAAM,UAAU,YAAY;AAC1B,MAAI,IAAA;AACF,QAAM,MAAA,CAAA,GAAI,MAAMC,6CAAuB,CAAA;AAAA,UACrC,MAAM,IAAK,CAAA,IAAA;AAAA,UACX,QAAU,EAAA;AAAA,SACX,CAAA;AACD,QAAA,IAAI,IAAI,CAAG,EAAA;AACT,UAAA,IAAA,CAAK,MAAO,CAAA,IAAA,CAAK,CAAW,QAAA,EAAA,CAAC,CAAoB,kBAAA,CAAA,CAAA;AAAA;AACnD,eACO,KAAO,EAAA;AACd,QAAK,IAAA,CAAA,MAAA,CAAO,IAAK,CAAA,CAAA,kCAAA,CAAA,EAAsC,KAAK,CAAA;AAAA;AAC9D,KACF;AAEA,IAAA,IAAI,KAAK,SAAW,EAAA;AAClB,MAAM,MAAA,eAAA,GAAkB,IAAI,eAAgB,EAAA;AAE5C,MAAA,IAAA,CAAK,UAAU,YAAa,CAAA;AAAA,QAC1B,EAAI,EAAA,wBAAA;AAAA,QACJ,SAAW,EAAA,EAAE,YAAc,EAAA,IAAA,CAAK,uBAAwB,EAAA;AAAA,QACxD,OAAS,EAAA,EAAE,YAAc,EAAA,IAAA,CAAK,0BAA0B,GAAI,EAAA;AAAA,QAC5D,EAAI,EAAA,OAAA;AAAA,QACJ,QAAQ,eAAgB,CAAA;AAAA,OACzB,CAAA;AAED,MAAA,OAAO,MAAM;AACX,QAAA,eAAA,CAAgB,KAAM,EAAA;AAAA,OACxB;AAAA;AAGF,IAAA,MAAM,WAAc,GAAA,WAAA,CAAY,OAAS,EAAA,IAAA,CAAK,uBAAuB,CAAA;AACrE,IAAA,OAAO,MAAM;AACX,MAAA,aAAA,CAAc,WAAW,CAAA;AAAA,KAC3B;AAAA;AAEJ;AAGA,SAAS,eAAkB,GAAA;AAEzB,EAAA,MAAM,wBAAwBC,2BAAoB,CAAA;AAAA,IAChD,IAAM,EAAA,kCAAA;AAAA,IACN,IAAM,EAAA,6EAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBC,2BAAoB,CAAA;AAAA,IACjD,IAAM,EAAA,qCAAA;AAAA,IACN,IAAM,EAAA,8FAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBA,2BAAoB,CAAA;AAAA,IACjD,IAAM,EAAA,qCAAA;AAAA,IACN,IAAM,EAAA,wFAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,2BAA2BA,2BAAoB,CAAA;AAAA,IACnD,IAAM,EAAA,wCAAA;AAAA,IACN,IAAM,EAAA;AAAA,GACP,CAAA;AAED,EAAM,MAAA,KAAA,GAAQC,WAAQ,CAAA,QAAA,CAAS,SAAS,CAAA;AACxC,EAAA,MAAM,oBAAoB,KAAM,CAAA,aAAA;AAAA,IAC9B,kCAAA;AAAA,IACA,EAAE,aAAa,8BAA+B;AAAA,GAChD;AAEA,EAAA,MAAM,qBAAqB,KAAM,CAAA,eAAA;AAAA,IAC/B,6BAAA;AAAA,IACA;AAAA,MACE,WAAa,EAAA,+CAAA;AAAA,MACb,IAAM,EAAA;AAAA;AACR,GACF;AAEA,EAAA,MAAM,qBAAqB,KAAM,CAAA,eAAA;AAAA,IAC/B,6BAAA;AAAA,IACA;AAAA,MACE,WAAa,EAAA,yCAAA;AAAA,MACb,IAAM,EAAA;AAAA;AACR,GACF;AAEA,EAAA,MAAM,uBAAuB,KAAM,CAAA,eAAA;AAAA,IACjC,gCAAA;AAAA,IACA;AAAA,MACE,WACE,EAAA,uGAAA;AAAA,MACF,IAAM,EAAA;AAAA;AACR,GACF;AAEA,EAAS,SAAA,YAAA,CAAa,MAAwB,MAAuB,EAAA;AACnE,IAAM,MAAA,SAAA,GAAY,QAAQ,MAAO,EAAA;AACjC,IAAM,MAAA,eAAA,GAAkB,uBAAuB,UAAW,EAAA;AAC1D,IAAM,MAAA,kBAAA,GAAqB,uBAAuB,UAAW,EAAA;AAE7D,IAAA,MAAA,CAAO,KAAM,CAAA,CAAA,WAAA,EAAc,IAAK,CAAA,SAAS,CAAE,CAAA,CAAA;AAE3C,IAAA,IAAI,KAAK,YAAc,EAAA;AACrB,MAAA,MAAM,UAAU,CAAC,IAAA,CAAK,aAAa,OAAQ,EAAA,CAAE,GAAG,SAAS,CAAA;AACzD,MAAA,wBAAA,CAAyB,QAAQ,OAAO,CAAA;AACxC,MAAA,oBAAA,CAAqB,OAAO,OAAO,CAAA;AAAA;AAGrC,IAAA,SAAS,OAAU,GAAA;AACjB,MAAM,MAAA,KAAA,GAAQ,OAAQ,CAAA,MAAA,CAAO,SAAS,CAAA;AACtC,MAAA,OAAO,KAAM,CAAA,CAAC,CAAI,GAAA,KAAA,CAAM,CAAC,CAAI,GAAA,GAAA;AAAA;AAG/B,IAAA,SAAS,wBAAwB,MAAgC,EAAA;AAC/D,MAAA,kBAAA,CAAmB,EAAE,MAAQ,EAAA,MAAA,CAAO,EAAK,GAAA,IAAA,GAAO,UAAU,CAAA;AAC1D,MAAmB,kBAAA,CAAA,MAAA,CAAO,SAAW,EAAA;AAAA,QACnC,MAAA,EAAQ,MAAO,CAAA,EAAA,GAAK,IAAO,GAAA;AAAA,OAC5B,CAAA;AAAA;AAGH,IAAA,SAAS,2BAA8B,GAAA;AACrC,MAAgB,eAAA,CAAA,EAAE,MAAQ,EAAA,WAAA,EAAa,CAAA;AACvC,MAAA,qBAAA,CAAsB,GAAI,CAAA,EAAE,MAAQ,EAAA,WAAA,IAAe,CAAC,CAAA;AAEpD,MAAA,kBAAA,CAAmB,OAAO,OAAQ,EAAA,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAC5D,MAAA,iBAAA,CAAkB,GAAI,CAAA,CAAA,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAAA;AAGlD,IAAA,SAAS,wBAA2B,GAAA;AAClC,MAAgB,eAAA,CAAA,EAAE,MAAQ,EAAA,QAAA,EAAU,CAAA;AACpC,MAAA,qBAAA,CAAsB,GAAI,CAAA,EAAE,MAAQ,EAAA,QAAA,IAAY,CAAC,CAAA;AAEjD,MAAA,kBAAA,CAAmB,OAAO,OAAQ,EAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AACzD,MAAA,iBAAA,CAAkB,GAAI,CAAA,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAAA;AAG/C,IAAA,SAAS,yBAA4B,GAAA;AACnC,MAAgB,eAAA,CAAA,EAAE,MAAQ,EAAA,SAAA,EAAW,CAAA;AACrC,MAAA,qBAAA,CAAsB,GAAI,CAAA,EAAE,MAAQ,EAAA,SAAA,IAAa,CAAC,CAAA;AAElD,MAAA,kBAAA,CAAmB,OAAO,OAAQ,EAAA,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAC1D,MAAA,iBAAA,CAAkB,GAAI,CAAA,CAAA,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAAA;AAGhD,IAAA,SAAS,WAAW,KAAc,EAAA;AAChC,MAAA,qBAAA,CAAsB,GAAI,CAAA,EAAE,MAAQ,EAAA,QAAA,IAAY,CAAC,CAAA;AACjD,MAAA,iBAAA,CAAkB,GAAI,CAAA,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAC7C,MAAA,MAAA,CAAO,IAAK,CAAA,CAAA,cAAA,EAAiB,IAAK,CAAA,SAAS,WAAW,KAAK,CAAA;AAAA;AAG7D,IAAO,OAAA;AAAA,MACL,uBAAA;AAAA,MACA,2BAAA;AAAA,MACA,wBAAA;AAAA,MACA,yBAAA;AAAA,MACA;AAAA,KACF;AAAA;AAGF,EAAA,OAAO,EAAE,YAAa,EAAA;AACxB;;;;"}

package/dist/service/DefaultEntitiesCatalog.cjs.js

@@ -136,11 +136,7 @@ class DefaultEntitiesCatalog {
       }
     });
     if (!request?.order) {
-      entitiesQuery = entitiesQuery.leftOuterJoin(
-        "refresh_state",
-        "refresh_state.entity_id",
-        "final_entities.entity_id"
-      ).orderBy("refresh_state.entity_ref", "asc");
+      entitiesQuery = entitiesQuery.orderBy("final_entities.entity_ref", "asc");
     } else {
       entitiesQuery.orderBy("final_entities.entity_id", "asc");
     }
@@ -188,21 +184,17 @@ class DefaultEntitiesCatalog {
   async entitiesBatch(request) {
     const lookup = /* @__PURE__ */ new Map();
     for (const chunk of lodash.chunk(request.entityRefs, 200)) {
-      let query = this.database("final_entities").innerJoin(
-        "refresh_state",
-        "refresh_state.entity_id",
-        "final_entities.entity_id"
-      ).select({
-        entityRef: "refresh_state.entity_ref",
+      let query = this.database("final_entities").select({
+        entityRef: "final_entities.entity_ref",
         entity: "final_entities.final_entity"
-      }).whereIn("refresh_state.entity_ref", chunk);
+      }).whereIn("final_entities.entity_ref", chunk);
       if (request?.filter) {
         query = parseFilter(
           request.filter,
           query,
           this.database,
           false,
-          "refresh_state.entity_id"
+          "final_entities.entity_id"
         );
       }
       for (const row of await query) {
@@ -254,7 +246,10 @@ class DefaultEntitiesCatalog {
           `%${normalizedFullTextFilterTerm.toLocaleLowerCase("en-US")}%`
         );
       } else {
-        const matchQuery = db("search").select("search.entity_id").whereIn("key", textFilterFields).andWhere(function keyFilter() {
+        const matchQuery = db("search").select("search.entity_id").whereIn(
+          "key",
+          textFilterFields.map((field) => field.toLocaleLowerCase("en-US"))
+        ).andWhere(function keyFilter() {
           this.andWhereRaw(
             "value like ?",
             `%${normalizedFullTextFilterTerm.toLocaleLowerCase("en-US")}%`
@@ -403,9 +398,7 @@ class DefaultEntitiesCatalog {
     });
   }
   async entityAncestry(rootRef) {
-    const [rootRow] = await this.database("refresh_state").leftJoin("final_entities", {
-      "refresh_state.entity_id": "final_entities.entity_id"
-    }).where("refresh_state.entity_ref", "=", rootRef).select({
+    const [rootRow] = await this.database("final_entities").where("final_entities.entity_ref", "=", rootRef).select({
       entityJson: "final_entities.final_entity"
     });
     if (!rootRow) {
@@ -420,12 +413,10 @@ class DefaultEntitiesCatalog {
       seenEntityRefs.add(currentRef);
       const parentRows = await this.database(
         "refresh_state_references"
-      ).innerJoin("refresh_state", {
-        "refresh_state_references.source_entity_ref": "refresh_state.entity_ref"
-      }).innerJoin("final_entities", {
-        "refresh_state.entity_id": "final_entities.entity_id"
+      ).innerJoin("final_entities", {
+        "refresh_state_references.source_entity_ref": "final_entities.entity_ref"
       }).where("refresh_state_references.target_entity_ref", "=", currentRef).select({
-        parentEntityRef: "refresh_state.entity_ref",
+        parentEntityRef: "final_entities.entity_ref",
         parentEntityJson: "final_entities.final_entity"
       });
       const parentRefs = [];

package/dist/service/DefaultEntitiesCatalog.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultEntitiesCatalog.cjs.js","sources":["../../src/service/DefaultEntitiesCatalog.ts"],"sourcesContent":["/*\n * Copyright 2020 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  Entity,\n  parseEntityRef,\n  stringifyEntityRef,\n} from '@backstage/catalog-model';\nimport { InputError, NotFoundError } from '@backstage/errors';\nimport { Knex } from 'knex';\nimport { chunk as lodashChunk, isEqual } from 'lodash';\nimport { z } from 'zod';\nimport {\n  Cursor,\n  EntitiesBatchRequest,\n  EntitiesBatchResponse,\n  EntitiesCatalog,\n  EntitiesRequest,\n  EntitiesResponse,\n  EntityAncestryResponse,\n  EntityFacetsRequest,\n  EntityFacetsResponse,\n  EntityOrder,\n  EntityPagination,\n  QueryEntitiesRequest,\n  QueryEntitiesResponse,\n} from '../catalog/types';\nimport {\n  DbFinalEntitiesRow,\n  DbPageInfo,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n  DbSearchRow,\n} from '../database/tables';\nimport { Stitcher } from '../stitching/types';\n\nimport {\n  isQueryEntitiesCursorRequest,\n  isQueryEntitiesInitialRequest,\n} from './util';\nimport {\n  EntitiesSearchFilter,\n  EntityFilter,\n} from '@backstage/plugin-catalog-node';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\nconst defaultSortField: EntityOrder = {\n  field: 'metadata.uid',\n  order: 'asc',\n};\n\nconst DEFAULT_LIMIT = 20;\n\nfunction parsePagination(input?: EntityPagination): EntityPagination {\n  if (!input) {\n    return {};\n  }\n\n  let { limit, offset } = input;\n\n  if (input.after === undefined) {\n    return { limit, offset };\n  }\n\n  let cursor;\n  try {\n    const json = Buffer.from(input.after, 'base64').toString('utf8');\n    cursor = JSON.parse(json);\n  } catch {\n    throw new InputError('Malformed after cursor, could not be parsed');\n  }\n\n  if (cursor.limit !== undefined) {\n    if (!Number.isInteger(cursor.limit)) {\n      throw new InputError('Malformed after cursor, limit was not an number');\n    }\n    limit = cursor.limit;\n  }\n\n  if (cursor.offset !== undefined) {\n    if (!Number.isInteger(cursor.offset)) {\n      throw new InputError('Malformed after cursor, offset was not a number');\n    }\n    offset = cursor.offset;\n  }\n\n  return { limit, offset };\n}\n\nfunction stringifyPagination(\n  input: Required<Omit<EntityPagination, 'after'>>,\n): string {\n  const { limit, offset } = input;\n  const json = JSON.stringify({ limit, offset });\n  const base64 = Buffer.from(json, 'utf8').toString('base64');\n  return base64;\n}\n\nfunction addCondition(\n  queryBuilder: Knex.QueryBuilder,\n  db: Knex,\n  filter: EntitiesSearchFilter,\n  negate: boolean = false,\n  entityIdField = 'entity_id',\n): void {\n  const key = filter.key.toLowerCase();\n  const values = filter.values?.map(v => v.toLowerCase());\n\n  // NOTE(freben): This used to be a set of OUTER JOIN, which may seem to\n  // make a lot of sense. However, it had abysmal performance on sqlite\n  // when datasets grew large, so we're using IN instead.\n  const matchQuery = db<DbSearchRow>('search')\n    .select('search.entity_id')\n    .where({ key })\n    .andWhere(function keyFilter() {\n      if (values?.length === 1) {\n        this.where({ value: values.at(0) });\n      } else if (values) {\n        this.andWhere('value', 'in', values);\n      }\n    });\n  queryBuilder.andWhere(entityIdField, negate ? 'not in' : 'in', matchQuery);\n}\n\nfunction isEntitiesSearchFilter(\n  filter: EntitiesSearchFilter | EntityFilter,\n): filter is EntitiesSearchFilter {\n  return filter.hasOwnProperty('key');\n}\n\nfunction isOrEntityFilter(\n  filter: { anyOf: EntityFilter[] } | EntityFilter,\n): filter is { anyOf: EntityFilter[] } {\n  return filter.hasOwnProperty('anyOf');\n}\n\nfunction isNegationEntityFilter(\n  filter: { not: EntityFilter } | EntityFilter,\n): filter is { not: EntityFilter } {\n  return filter.hasOwnProperty('not');\n}\n\nfunction parseFilter(\n  filter: EntityFilter,\n  query: Knex.QueryBuilder,\n  db: Knex,\n  negate: boolean = false,\n  entityIdField = 'entity_id',\n): Knex.QueryBuilder {\n  if (isNegationEntityFilter(filter)) {\n    return parseFilter(filter.not, query, db, !negate, entityIdField);\n  }\n\n  if (isEntitiesSearchFilter(filter)) {\n    return query.andWhere(function filterFunction() {\n      addCondition(this, db, filter, negate, entityIdField);\n    });\n  }\n\n  return query[negate ? 'andWhereNot' : 'andWhere'](function filterFunction() {\n    if (isOrEntityFilter(filter)) {\n      for (const subFilter of filter.anyOf ?? []) {\n        this.orWhere(subQuery =>\n          parseFilter(subFilter, subQuery, db, false, entityIdField),\n        );\n      }\n    } else {\n      for (const subFilter of filter.allOf ?? []) {\n        this.andWhere(subQuery =>\n          parseFilter(subFilter, subQuery, db, false, entityIdField),\n        );\n      }\n    }\n  });\n}\n\nexport class DefaultEntitiesCatalog implements EntitiesCatalog {\n  private readonly database: Knex;\n  private readonly logger: LoggerService;\n  private readonly stitcher: Stitcher;\n\n  constructor(options: {\n    database: Knex;\n    logger: LoggerService;\n    stitcher: Stitcher;\n  }) {\n    this.database = options.database;\n    this.logger = options.logger;\n    this.stitcher = options.stitcher;\n  }\n\n  async entities(request?: EntitiesRequest): Promise<EntitiesResponse> {\n    const db = this.database;\n\n    let entitiesQuery =\n      db<DbFinalEntitiesRow>('final_entities').select('final_entities.*');\n\n    request?.order?.forEach(({ field }, index) => {\n      const alias = `order_${index}`;\n      entitiesQuery = entitiesQuery.leftOuterJoin(\n        { [alias]: 'search' },\n        function search(inner) {\n          inner\n            .on(`${alias}.entity_id`, 'final_entities.entity_id')\n            .andOn(`${alias}.key`, db.raw('?', [field]));\n        },\n      );\n    });\n\n    entitiesQuery = entitiesQuery.whereNotNull('final_entities.final_entity');\n\n    if (request?.filter) {\n      entitiesQuery = parseFilter(\n        request.filter,\n        entitiesQuery,\n        db,\n        false,\n        'final_entities.entity_id',\n      );\n    }\n\n    request?.order?.forEach(({ order }, index) => {\n      if (db.client.config.client === 'pg') {\n        // pg correctly orders by the column value and handling nulls in one go\n        entitiesQuery = entitiesQuery.orderBy([\n          { column: `order_${index}.value`, order, nulls: 'last' },\n        ]);\n      } else {\n        // sqlite and mysql translate the above statement ONLY into \"order by (value is null) asc\"\n        // no matter what the order is, for some reason, so we have to manually add back the statement\n        // that translates to \"order by value <order>\" while avoiding to give an order\n        entitiesQuery = entitiesQuery.orderBy([\n          { column: `order_${index}.value`, order: undefined, nulls: 'last' },\n          { column: `order_${index}.value`, order },\n        ]);\n      }\n    });\n\n    if (!request?.order) {\n      entitiesQuery = entitiesQuery\n        .leftOuterJoin(\n          'refresh_state',\n          'refresh_state.entity_id',\n          'final_entities.entity_id',\n        )\n        .orderBy('refresh_state.entity_ref', 'asc'); // default sort\n    } else {\n      entitiesQuery.orderBy('final_entities.entity_id', 'asc'); // stable sort\n    }\n\n    const { limit, offset } = parsePagination(request?.pagination);\n    if (limit !== undefined) {\n      entitiesQuery = entitiesQuery.limit(limit + 1);\n    }\n    if (offset !== undefined) {\n      entitiesQuery = entitiesQuery.offset(offset);\n    }\n\n    let rows = await entitiesQuery;\n    let pageInfo: DbPageInfo;\n    if (limit === undefined || rows.length <= limit) {\n      pageInfo = { hasNextPage: false };\n    } else {\n      rows = rows.slice(0, -1);\n      pageInfo = {\n        hasNextPage: true,\n        endCursor: stringifyPagination({\n          limit,\n          offset: (offset ?? 0) + limit,\n        }),\n      };\n    }\n\n    let entities: Entity[] = rows.map(e => JSON.parse(e.final_entity!));\n\n    if (request?.fields) {\n      entities = entities.map(e => request.fields!(e));\n    }\n\n    // TODO(freben): This is added as a compatibility guarantee, until we can be\n    // sure that all adopters have re-stitched their entities so that the new\n    // targetRef field is present on them, and that they have stopped consuming\n    // the now-removed old field\n    // TODO(jhaals): Remove this in April 2022\n    for (const entity of entities) {\n      if (entity.relations) {\n        for (const relation of entity.relations as any) {\n          if (!relation.targetRef && relation.target) {\n            // This is the case where an old-form entity, not yet stitched with\n            // the updated code, was in the database\n            relation.targetRef = stringifyEntityRef(relation.target);\n          } else if (!relation.target && relation.targetRef) {\n            // This is the case where a new-form entity, stitched with the\n            // updated code, was in the database but we still want to produce\n            // the old data shape as well for compatibility reasons\n            relation.target = parseEntityRef(relation.targetRef);\n          }\n        }\n      }\n    }\n\n    return {\n      entities,\n      pageInfo,\n    };\n  }\n\n  async entitiesBatch(\n    request: EntitiesBatchRequest,\n  ): Promise<EntitiesBatchResponse> {\n    const lookup = new Map<string, Entity>();\n\n    for (const chunk of lodashChunk(request.entityRefs, 200)) {\n      let query = this.database<DbFinalEntitiesRow>('final_entities')\n        .innerJoin<DbRefreshStateRow>(\n          'refresh_state',\n          'refresh_state.entity_id',\n          'final_entities.entity_id',\n        )\n        .select({\n          entityRef: 'refresh_state.entity_ref',\n          entity: 'final_entities.final_entity',\n        })\n        .whereIn('refresh_state.entity_ref', chunk);\n\n      if (request?.filter) {\n        query = parseFilter(\n          request.filter,\n          query,\n          this.database,\n          false,\n          'refresh_state.entity_id',\n        );\n      }\n\n      for (const row of await query) {\n        lookup.set(row.entityRef, row.entity ? JSON.parse(row.entity) : null);\n      }\n    }\n\n    let items = request.entityRefs.map(ref => lookup.get(ref) ?? null);\n\n    if (request.fields) {\n      items = items.map(e => e && request.fields!(e));\n    }\n\n    return { items };\n  }\n\n  async queryEntities(\n    request: QueryEntitiesRequest,\n  ): Promise<QueryEntitiesResponse> {\n    const db = this.database;\n\n    const limit = request.limit ?? DEFAULT_LIMIT;\n\n    const cursor: Omit<Cursor, 'orderFieldValues'> & {\n      orderFieldValues?: (string | null)[];\n    } = {\n      orderFields: [defaultSortField],\n      isPrevious: false,\n      ...parseCursorFromRequest(request),\n    };\n\n    const isFetchingBackwards = cursor.isPrevious;\n\n    if (cursor.orderFields.length > 1) {\n      this.logger.warn(`Only one sort field is supported, ignoring the rest`);\n    }\n\n    const sortField: EntityOrder = {\n      ...defaultSortField,\n      ...cursor.orderFields[0],\n    };\n\n    const [prevItemOrderFieldValue, prevItemUid] =\n      cursor.orderFieldValues || [];\n\n    const dbQuery = db('final_entities').leftOuterJoin('search', qb =>\n      qb\n        .on('search.entity_id', 'final_entities.entity_id')\n        .andOnVal('search.key', sortField.field),\n    );\n\n    if (cursor.filter) {\n      parseFilter(\n        cursor.filter,\n        dbQuery,\n        db,\n        false,\n        'final_entities.entity_id',\n      );\n    }\n\n    const normalizedFullTextFilterTerm = cursor.fullTextFilter?.term?.trim();\n    const textFilterFields = cursor.fullTextFilter?.fields ?? [sortField.field];\n    if (normalizedFullTextFilterTerm) {\n      if (\n        textFilterFields.length === 1 &&\n        textFilterFields[0] === sortField.field\n      ) {\n        // If there is one item, apply the like query to the top level query which is already\n        //   filtered based on the singular sortField.\n        dbQuery.andWhereRaw(\n          'value like ?',\n          `%${normalizedFullTextFilterTerm.toLocaleLowerCase('en-US')}%`,\n        );\n      } else {\n        const matchQuery = db<DbSearchRow>('search')\n          .select('search.entity_id')\n          .whereIn('key', textFilterFields)\n          .andWhere(function keyFilter() {\n            this.andWhereRaw(\n              'value like ?',\n              `%${normalizedFullTextFilterTerm.toLocaleLowerCase('en-US')}%`,\n            );\n          });\n        dbQuery.andWhere('final_entities.entity_id', 'in', matchQuery);\n      }\n    }\n\n    const countQuery = dbQuery.clone();\n\n    const isOrderingDescending = sortField.order === 'desc';\n\n    if (prevItemOrderFieldValue) {\n      dbQuery.andWhere(function nested() {\n        this.where(\n          'value',\n          isFetchingBackwards !== isOrderingDescending ? '<' : '>',\n          prevItemOrderFieldValue,\n        )\n          .orWhere('value', '=', prevItemOrderFieldValue)\n          .andWhere(\n            'final_entities.entity_id',\n            isFetchingBackwards !== isOrderingDescending ? '<' : '>',\n            prevItemUid,\n          );\n      });\n    }\n\n    if (db.client.config.client === 'pg') {\n      // pg correctly orders by the column value and handling nulls in one go\n      dbQuery.orderBy([\n        {\n          column: 'search.value',\n          order: isFetchingBackwards\n            ? invertOrder(sortField.order)\n            : sortField.order,\n          nulls: 'last',\n        },\n        {\n          column: 'final_entities.entity_id',\n          order: isFetchingBackwards\n            ? invertOrder(sortField.order)\n            : sortField.order,\n        },\n      ]);\n    } else {\n      // sqlite and mysql translate the above statement ONLY into \"order by (value is null) asc\"\n      // no matter what the order is, for some reason, so we have to manually add back the statement\n      // that translates to \"order by value <order>\" while avoiding to give an order\n      dbQuery.orderBy([\n        {\n          column: 'search.value',\n          order: undefined,\n          nulls: 'last',\n        },\n        {\n          column: 'search.value',\n          order: isFetchingBackwards\n            ? invertOrder(sortField.order)\n            : sortField.order,\n        },\n        {\n          column: 'final_entities.entity_id',\n          order: isFetchingBackwards\n            ? invertOrder(sortField.order)\n            : sortField.order,\n        },\n      ]);\n    }\n\n    if (\n      isQueryEntitiesInitialRequest(request) &&\n      request.offset !== undefined\n    ) {\n      dbQuery.offset(request.offset);\n    }\n    // fetch an extra item to check if there are more items.\n    dbQuery.limit(isFetchingBackwards ? limit : limit + 1);\n\n    countQuery.count('final_entities.entity_id', { as: 'count' });\n\n    const [rows, [{ count }]] = await Promise.all([\n      limit > 0 ? dbQuery : [],\n      // for performance reasons we invoke the countQuery\n      // only on the first request.\n      // The result is then embedded into the cursor\n      // for subsequent requests.\n      typeof cursor.totalItems === 'undefined'\n        ? countQuery\n        : [{ count: cursor.totalItems }],\n    ]);\n\n    const totalItems = Number(count);\n\n    if (isFetchingBackwards) {\n      rows.reverse();\n    }\n    const hasMoreResults =\n      limit > 0 && (isFetchingBackwards || rows.length > limit);\n\n    // discard the extra item only when fetching forward.\n    if (rows.length > limit) {\n      rows.length -= 1;\n    }\n\n    const isInitialRequest = cursor.firstSortFieldValues === undefined;\n\n    const firstRow = rows[0];\n    const lastRow = rows[rows.length - 1];\n\n    const firstSortFieldValues = cursor.firstSortFieldValues || [\n      firstRow?.value,\n      firstRow?.entity_id,\n    ];\n\n    const nextCursor: Cursor | undefined = hasMoreResults\n      ? {\n          ...cursor,\n          orderFieldValues: sortFieldsFromRow(lastRow),\n          firstSortFieldValues,\n          isPrevious: false,\n          totalItems,\n        }\n      : undefined;\n\n    const prevCursor: Cursor | undefined =\n      !isInitialRequest &&\n      rows.length > 0 &&\n      !isEqual(sortFieldsFromRow(firstRow), cursor.firstSortFieldValues)\n        ? {\n            ...cursor,\n            orderFieldValues: sortFieldsFromRow(firstRow),\n            firstSortFieldValues: cursor.firstSortFieldValues,\n            isPrevious: true,\n            totalItems,\n          }\n        : undefined;\n\n    const items = rows\n      .map(e => JSON.parse(e.final_entity!))\n      .map(e => (request.fields ? request.fields(e) : e));\n\n    return {\n      items,\n      pageInfo: {\n        ...(!!prevCursor && { prevCursor }),\n        ...(!!nextCursor && { nextCursor }),\n      },\n      totalItems,\n    };\n  }\n\n  async removeEntityByUid(uid: string): Promise<void> {\n    const dbConfig = this.database.client.config;\n\n    // Clear the hashed state of the immediate parents of the deleted entity.\n    // This makes sure that when they get reprocessed, their output is written\n    // down again. The reason for wanting to do this, is that if the user\n    // deletes entities that ARE still emitted by the parent, the parent\n    // processing will still generate the same output hash as always, which\n    // means it'll never try to write down the children again (it assumes that\n    // they already exist). This means that without the code below, the database\n    // never \"heals\" from accidental deletes.\n    if (dbConfig.client.includes('mysql')) {\n      // MySQL doesn't support the syntax we need to do this in a single query,\n      // http://dev.mysql.com/doc/refman/5.6/en/update.html\n      const results = await this.database<DbRefreshStateRow>('refresh_state')\n        .select('entity_id')\n        .whereIn('entity_ref', function parents(builder) {\n          return builder\n            .from<DbRefreshStateRow>('refresh_state')\n            .innerJoin<DbRefreshStateReferencesRow>(\n              'refresh_state_references',\n              {\n                'refresh_state_references.target_entity_ref':\n                  'refresh_state.entity_ref',\n              },\n            )\n            .where('refresh_state.entity_id', '=', uid)\n            .select('refresh_state_references.source_entity_ref');\n        });\n      await this.database<DbRefreshStateRow>('refresh_state')\n        .update({\n          result_hash: 'child-was-deleted',\n          next_update_at: this.database.fn.now(),\n        })\n        .whereIn(\n          'entity_id',\n          results.map(key => key.entity_id),\n        );\n    } else {\n      await this.database<DbRefreshStateRow>('refresh_state')\n        .update({\n          result_hash: 'child-was-deleted',\n          next_update_at: this.database.fn.now(),\n        })\n        .whereIn('entity_ref', function parents(builder) {\n          return builder\n            .from<DbRefreshStateRow>('refresh_state')\n            .innerJoin<DbRefreshStateReferencesRow>(\n              'refresh_state_references',\n              {\n                'refresh_state_references.target_entity_ref':\n                  'refresh_state.entity_ref',\n              },\n            )\n            .where('refresh_state.entity_id', '=', uid)\n            .select('refresh_state_references.source_entity_ref');\n        });\n    }\n\n    // Stitch the entities that the deleted one had relations to. If we do not\n    // do this, the entities in the other end of the relations will still look\n    // like they have a relation to the entity that was deleted, despite not\n    // having any corresponding rows in the relations table.\n    const relationPeers = await this.database\n      .from<DbRelationsRow>('relations')\n      .innerJoin<DbRefreshStateReferencesRow>('refresh_state', {\n        'refresh_state.entity_ref': 'relations.target_entity_ref',\n      })\n      .where('relations.originating_entity_id', '=', uid)\n      .andWhere('refresh_state.entity_id', '!=', uid)\n      .select({ ref: 'relations.target_entity_ref' })\n      .union(other =>\n        other\n          .from<DbRelationsRow>('relations')\n          .innerJoin<DbRefreshStateReferencesRow>('refresh_state', {\n            'refresh_state.entity_ref': 'relations.source_entity_ref',\n          })\n          .where('relations.originating_entity_id', '=', uid)\n          .andWhere('refresh_state.entity_id', '!=', uid)\n          .select({ ref: 'relations.source_entity_ref' }),\n      );\n\n    await this.database<DbRefreshStateRow>('refresh_state')\n      .where('entity_id', uid)\n      .delete();\n\n    await this.stitcher.stitch({\n      entityRefs: new Set(relationPeers.map(p => p.ref)),\n    });\n  }\n\n  async entityAncestry(rootRef: string): Promise<EntityAncestryResponse> {\n    const [rootRow] = await this.database<DbRefreshStateRow>('refresh_state')\n      .leftJoin<DbFinalEntitiesRow>('final_entities', {\n        'refresh_state.entity_id': 'final_entities.entity_id',\n      })\n      .where('refresh_state.entity_ref', '=', rootRef)\n      .select({\n        entityJson: 'final_entities.final_entity',\n      });\n\n    if (!rootRow) {\n      throw new NotFoundError(`No such entity ${rootRef}`);\n    }\n\n    const rootEntity = JSON.parse(rootRow.entityJson) as Entity;\n    const seenEntityRefs = new Set<string>();\n    const todo = new Array<Entity>();\n    const items = new Array<{ entity: Entity; parentEntityRefs: string[] }>();\n\n    for (\n      let current: Entity | undefined = rootEntity;\n      current;\n      current = todo.pop()\n    ) {\n      const currentRef = stringifyEntityRef(current);\n      seenEntityRefs.add(currentRef);\n\n      const parentRows = await this.database<DbRefreshStateReferencesRow>(\n        'refresh_state_references',\n      )\n        .innerJoin<DbRefreshStateRow>('refresh_state', {\n          'refresh_state_references.source_entity_ref':\n            'refresh_state.entity_ref',\n        })\n        .innerJoin<DbFinalEntitiesRow>('final_entities', {\n          'refresh_state.entity_id': 'final_entities.entity_id',\n        })\n        .where('refresh_state_references.target_entity_ref', '=', currentRef)\n        .select({\n          parentEntityRef: 'refresh_state.entity_ref',\n          parentEntityJson: 'final_entities.final_entity',\n        });\n\n      const parentRefs: string[] = [];\n      for (const { parentEntityRef, parentEntityJson } of parentRows) {\n        parentRefs.push(parentEntityRef);\n        if (!seenEntityRefs.has(parentEntityRef)) {\n          seenEntityRefs.add(parentEntityRef);\n          todo.push(JSON.parse(parentEntityJson));\n        }\n      }\n\n      items.push({\n        entity: current,\n        parentEntityRefs: parentRefs,\n      });\n    }\n\n    return {\n      rootEntityRef: stringifyEntityRef(rootEntity),\n      items,\n    };\n  }\n\n  async facets(request: EntityFacetsRequest): Promise<EntityFacetsResponse> {\n    const facets: EntityFacetsResponse['facets'] = {};\n    const db = this.database;\n\n    for (const facet of request.facets) {\n      const dbQuery = db<DbSearchRow>('search')\n        .where('search.key', facet.toLocaleLowerCase('en-US'))\n        .whereNotNull('search.original_value')\n        .select({ value: 'search.original_value', count: db.raw('count(*)') })\n        .groupBy('search.original_value');\n\n      if (request?.filter) {\n        parseFilter(request.filter, dbQuery, db, false, 'search.entity_id');\n      }\n\n      const result = await dbQuery;\n\n      facets[facet] = result.map(data => ({\n        value: String(data.value),\n        count: Number(data.count),\n      }));\n    }\n\n    return { facets };\n  }\n}\n\nconst entityFilterParser: z.ZodSchema<EntityFilter> = z.lazy(() =>\n  z\n    .object({\n      key: z.string(),\n      values: z.array(z.string()).optional(),\n    })\n    .or(z.object({ not: entityFilterParser }))\n    .or(z.object({ anyOf: z.array(entityFilterParser) }))\n    .or(z.object({ allOf: z.array(entityFilterParser) })),\n);\n\nexport const cursorParser: z.ZodSchema<Cursor> = z.object({\n  orderFields: z.array(\n    z.object({ field: z.string(), order: z.enum(['asc', 'desc']) }),\n  ),\n  orderFieldValues: z.array(z.string().or(z.null())),\n  filter: entityFilterParser.optional(),\n  isPrevious: z.boolean(),\n  query: z.string().optional(),\n  firstSortFieldValues: z.array(z.string().or(z.null())).optional(),\n  totalItems: z.number().optional(),\n});\n\nfunction parseCursorFromRequest(\n  request?: QueryEntitiesRequest,\n): Partial<Cursor> {\n  if (isQueryEntitiesInitialRequest(request)) {\n    const {\n      filter,\n      orderFields: sortFields = [defaultSortField],\n      fullTextFilter,\n    } = request;\n    return { filter, orderFields: sortFields, fullTextFilter };\n  }\n  if (isQueryEntitiesCursorRequest(request)) {\n    return request.cursor;\n  }\n  return {};\n}\n\nfunction invertOrder(order: EntityOrder['order']) {\n  return order === 'asc' ? 'desc' : 'asc';\n}\n\nfunction sortFieldsFromRow(row: DbSearchRow) {\n  return [row.value, row.entity_id];\n}\n"],"names":["InputError","stringifyEntityRef","parseEntityRef","lodashChunk","isQueryEntitiesInitialRequest","isEqual","NotFoundError","z","isQueryEntitiesCursorRequest"],"mappings":";;;;;;;;AA4DA,MAAM,gBAAgC,GAAA;AAAA,EACpC,KAAO,EAAA,cAAA;AAAA,EACP,KAAO,EAAA;AACT,CAAA;AAEA,MAAM,aAAgB,GAAA,EAAA;AAEtB,SAAS,gBAAgB,KAA4C,EAAA;AACnE,EAAA,IAAI,CAAC,KAAO,EAAA;AACV,IAAA,OAAO,EAAC;AAAA;AAGV,EAAI,IAAA,EAAE,KAAO,EAAA,MAAA,EAAW,GAAA,KAAA;AAExB,EAAI,IAAA,KAAA,CAAM,UAAU,KAAW,CAAA,EAAA;AAC7B,IAAO,OAAA,EAAE,OAAO,MAAO,EAAA;AAAA;AAGzB,EAAI,IAAA,MAAA;AACJ,EAAI,IAAA;AACF,IAAM,MAAA,IAAA,GAAO,OAAO,IAAK,CAAA,KAAA,CAAM,OAAO,QAAQ,CAAA,CAAE,SAAS,MAAM,CAAA;AAC/D,IAAS,MAAA,GAAA,IAAA,CAAK,MAAM,IAAI,CAAA;AAAA,GAClB,CAAA,MAAA;AACN,IAAM,MAAA,IAAIA,kBAAW,6CAA6C,CAAA;AAAA;AAGpE,EAAI,IAAA,MAAA,CAAO,UAAU,KAAW,CAAA,EAAA;AAC9B,IAAA,IAAI,CAAC,MAAA,CAAO,SAAU,CAAA,MAAA,CAAO,KAAK,CAAG,EAAA;AACnC,MAAM,MAAA,IAAIA,kBAAW,iDAAiD,CAAA;AAAA;AAExE,IAAA,KAAA,GAAQ,MAAO,CAAA,KAAA;AAAA;AAGjB,EAAI,IAAA,MAAA,CAAO,WAAW,KAAW,CAAA,EAAA;AAC/B,IAAA,IAAI,CAAC,MAAA,CAAO,SAAU,CAAA,MAAA,CAAO,MAAM,CAAG,EAAA;AACpC,MAAM,MAAA,IAAIA,kBAAW,iDAAiD,CAAA;AAAA;AAExE,IAAA,MAAA,GAAS,MAAO,CAAA,MAAA;AAAA;AAGlB,EAAO,OAAA,EAAE,OAAO,MAAO,EAAA;AACzB;AAEA,SAAS,oBACP,KACQ,EAAA;AACR,EAAM,MAAA,EAAE,KAAO,EAAA,MAAA,EAAW,GAAA,KAAA;AAC1B,EAAA,MAAM,OAAO,IAAK,CAAA,SAAA,CAAU,EAAE,KAAA,EAAO,QAAQ,CAAA;AAC7C,EAAA,MAAM,SAAS,MAAO,CAAA,IAAA,CAAK,MAAM,MAAM,CAAA,CAAE,SAAS,QAAQ,CAAA;AAC1D,EAAO,OAAA,MAAA;AACT;AAEA,SAAS,aACP,YACA,EAAA,EAAA,EACA,QACA,MAAkB,GAAA,KAAA,EAClB,gBAAgB,WACV,EAAA;AACN,EAAM,MAAA,GAAA,GAAM,MAAO,CAAA,GAAA,CAAI,WAAY,EAAA;AACnC,EAAA,MAAM,SAAS,MAAO,CAAA,MAAA,EAAQ,IAAI,CAAK,CAAA,KAAA,CAAA,CAAE,aAAa,CAAA;AAKtD,EAAA,MAAM,UAAa,GAAA,EAAA,CAAgB,QAAQ,CAAA,CACxC,OAAO,kBAAkB,CAAA,CACzB,KAAM,CAAA,EAAE,GAAI,EAAC,CACb,CAAA,QAAA,CAAS,SAAS,SAAY,GAAA;AAC7B,IAAI,IAAA,MAAA,EAAQ,WAAW,CAAG,EAAA;AACxB,MAAA,IAAA,CAAK,MAAM,EAAE,KAAA,EAAO,OAAO,EAAG,CAAA,CAAC,GAAG,CAAA;AAAA,eACzB,MAAQ,EAAA;AACjB,MAAK,IAAA,CAAA,QAAA,CAAS,OAAS,EAAA,IAAA,EAAM,MAAM,CAAA;AAAA;AACrC,GACD,CAAA;AACH,EAAA,YAAA,CAAa,QAAS,CAAA,aAAA,EAAe,MAAS,GAAA,QAAA,GAAW,MAAM,UAAU,CAAA;AAC3E;AAEA,SAAS,uBACP,MACgC,EAAA;AAChC,EAAO,OAAA,MAAA,CAAO,eAAe,KAAK,CAAA;AACpC;AAEA,SAAS,iBACP,MACqC,EAAA;AACrC,EAAO,OAAA,MAAA,CAAO,eAAe,OAAO,CAAA;AACtC;AAEA,SAAS,uBACP,MACiC,EAAA;AACjC,EAAO,OAAA,MAAA,CAAO,eAAe,KAAK,CAAA;AACpC;AAEA,SAAS,YACP,MACA,EAAA,KAAA,EACA,IACA,MAAkB,GAAA,KAAA,EAClB,gBAAgB,WACG,EAAA;AACnB,EAAI,IAAA,sBAAA,CAAuB,MAAM,CAAG,EAAA;AAClC,IAAA,OAAO,YAAY,MAAO,CAAA,GAAA,EAAK,OAAO,EAAI,EAAA,CAAC,QAAQ,aAAa,CAAA;AAAA;AAGlE,EAAI,IAAA,sBAAA,CAAuB,MAAM,CAAG,EAAA;AAClC,IAAO,OAAA,KAAA,CAAM,QAAS,CAAA,SAAS,cAAiB,GAAA;AAC9C,MAAA,YAAA,CAAa,IAAM,EAAA,EAAA,EAAI,MAAQ,EAAA,MAAA,EAAQ,aAAa,CAAA;AAAA,KACrD,CAAA;AAAA;AAGH,EAAA,OAAO,MAAM,MAAS,GAAA,aAAA,GAAgB,UAAU,CAAA,CAAE,SAAS,cAAiB,GAAA;AAC1E,IAAI,IAAA,gBAAA,CAAiB,MAAM,CAAG,EAAA;AAC5B,MAAA,KAAA,MAAW,SAAa,IAAA,MAAA,CAAO,KAAS,IAAA,EAAI,EAAA;AAC1C,QAAK,IAAA,CAAA,OAAA;AAAA,UAAQ,cACX,WAAY,CAAA,SAAA,EAAW,QAAU,EAAA,EAAA,EAAI,OAAO,aAAa;AAAA,SAC3D;AAAA;AACF,KACK,MAAA;AACL,MAAA,KAAA,MAAW,SAAa,IAAA,MAAA,CAAO,KAAS,IAAA,EAAI,EAAA;AAC1C,QAAK,IAAA,CAAA,QAAA;AAAA,UAAS,cACZ,WAAY,CAAA,SAAA,EAAW,QAAU,EAAA,EAAA,EAAI,OAAO,aAAa;AAAA,SAC3D;AAAA;AACF;AACF,GACD,CAAA;AACH;AAEO,MAAM,sBAAkD,CAAA;AAAA,EAC5C,QAAA;AAAA,EACA,MAAA;AAAA,EACA,QAAA;AAAA,EAEjB,YAAY,OAIT,EAAA;AACD,IAAA,IAAA,CAAK,WAAW,OAAQ,CAAA,QAAA;AACxB,IAAA,IAAA,CAAK,SAAS,OAAQ,CAAA,MAAA;AACtB,IAAA,IAAA,CAAK,WAAW,OAAQ,CAAA,QAAA;AAAA;AAC1B,EAEA,MAAM,SAAS,OAAsD,EAAA;AACnE,IAAA,MAAM,KAAK,IAAK,CAAA,QAAA;AAEhB,IAAA,IAAI,aACF,GAAA,EAAA,CAAuB,gBAAgB,CAAA,CAAE,OAAO,kBAAkB,CAAA;AAEpE,IAAA,OAAA,EAAS,OAAO,OAAQ,CAAA,CAAC,EAAE,KAAA,IAAS,KAAU,KAAA;AAC5C,MAAM,MAAA,KAAA,GAAQ,SAAS,KAAK,CAAA,CAAA;AAC5B,MAAA,aAAA,GAAgB,aAAc,CAAA,aAAA;AAAA,QAC5B,EAAE,CAAC,KAAK,GAAG,QAAS,EAAA;AAAA,QACpB,SAAS,OAAO,KAAO,EAAA;AACrB,UAAA,KAAA,CACG,GAAG,CAAG,EAAA,KAAK,CAAc,UAAA,CAAA,EAAA,0BAA0B,EACnD,KAAM,CAAA,CAAA,EAAG,KAAK,CAAA,IAAA,CAAA,EAAQ,GAAG,GAAI,CAAA,GAAA,EAAK,CAAC,KAAK,CAAC,CAAC,CAAA;AAAA;AAC/C,OACF;AAAA,KACD,CAAA;AAED,IAAgB,aAAA,GAAA,aAAA,CAAc,aAAa,6BAA6B,CAAA;AAExE,IAAA,IAAI,SAAS,MAAQ,EAAA;AACnB,MAAgB,aAAA,GAAA,WAAA;AAAA,QACd,OAAQ,CAAA,MAAA;AAAA,QACR,aAAA;AAAA,QACA,EAAA;AAAA,QACA,KAAA;AAAA,QACA;AAAA,OACF;AAAA;AAGF,IAAA,OAAA,EAAS,OAAO,OAAQ,CAAA,CAAC,EAAE,KAAA,IAAS,KAAU,KAAA;AAC5C,MAAA,IAAI,EAAG,CAAA,MAAA,CAAO,MAAO,CAAA,MAAA,KAAW,IAAM,EAAA;AAEpC,QAAA,aAAA,GAAgB,cAAc,OAAQ,CAAA;AAAA,UACpC,EAAE,MAAQ,EAAA,CAAA,MAAA,EAAS,KAAK,CAAU,MAAA,CAAA,EAAA,KAAA,EAAO,OAAO,MAAO;AAAA,SACxD,CAAA;AAAA,OACI,MAAA;AAIL,QAAA,aAAA,GAAgB,cAAc,OAAQ,CAAA;AAAA,UACpC,EAAE,QAAQ,CAAS,MAAA,EAAA,KAAK,UAAU,KAAO,EAAA,KAAA,CAAA,EAAW,OAAO,MAAO,EAAA;AAAA,UAClE,EAAE,MAAA,EAAQ,CAAS,MAAA,EAAA,KAAK,UAAU,KAAM;AAAA,SACzC,CAAA;AAAA;AACH,KACD,CAAA;AAED,IAAI,IAAA,CAAC,SAAS,KAAO,EAAA;AACnB,MAAA,aAAA,GAAgB,aACb,CAAA,aAAA;AAAA,QACC,eAAA;AAAA,QACA,yBAAA;AAAA,QACA;AAAA,OACF,CACC,OAAQ,CAAA,0BAAA,EAA4B,KAAK,CAAA;AAAA,KACvC,MAAA;AACL,MAAc,aAAA,CAAA,OAAA,CAAQ,4BAA4B,KAAK,CAAA;AAAA;AAGzD,IAAA,MAAM,EAAE,KAAO,EAAA,MAAA,EAAW,GAAA,eAAA,CAAgB,SAAS,UAAU,CAAA;AAC7D,IAAA,IAAI,UAAU,KAAW,CAAA,EAAA;AACvB,MAAgB,aAAA,GAAA,aAAA,CAAc,KAAM,CAAA,KAAA,GAAQ,CAAC,CAAA;AAAA;AAE/C,IAAA,IAAI,WAAW,KAAW,CAAA,EAAA;AACxB,MAAgB,aAAA,GAAA,aAAA,CAAc,OAAO,MAAM,CAAA;AAAA;AAG7C,IAAA,IAAI,OAAO,MAAM,aAAA;AACjB,IAAI,IAAA,QAAA;AACJ,IAAA,IAAI,KAAU,KAAA,KAAA,CAAA,IAAa,IAAK,CAAA,MAAA,IAAU,KAAO,EAAA;AAC/C,MAAW,QAAA,GAAA,EAAE,aAAa,KAAM,EAAA;AAAA,KAC3B,MAAA;AACL,MAAO,IAAA,GAAA,IAAA,CAAK,KAAM,CAAA,CAAA,EAAG,CAAE,CAAA,CAAA;AACvB,MAAW,QAAA,GAAA;AAAA,QACT,WAAa,EAAA,IAAA;AAAA,QACb,WAAW,mBAAoB,CAAA;AAAA,UAC7B,KAAA;AAAA,UACA,MAAA,EAAA,CAAS,UAAU,CAAK,IAAA;AAAA,SACzB;AAAA,OACH;AAAA;AAGF,IAAI,IAAA,QAAA,GAAqB,KAAK,GAAI,CAAA,CAAA,CAAA,KAAK,KAAK,KAAM,CAAA,CAAA,CAAE,YAAa,CAAC,CAAA;AAElE,IAAA,IAAI,SAAS,MAAQ,EAAA;AACnB,MAAA,QAAA,GAAW,SAAS,GAAI,CAAA,CAAA,CAAA,KAAK,OAAQ,CAAA,MAAA,CAAQ,CAAC,CAAC,CAAA;AAAA;AAQjD,IAAA,KAAA,MAAW,UAAU,QAAU,EAAA;AAC7B,MAAA,IAAI,OAAO,SAAW,EAAA;AACpB,QAAW,KAAA,MAAA,QAAA,IAAY,OAAO,SAAkB,EAAA;AAC9C,UAAA,IAAI,CAAC,QAAA,CAAS,SAAa,IAAA,QAAA,CAAS,MAAQ,EAAA;AAG1C,YAAS,QAAA,CAAA,SAAA,GAAYC,+BAAmB,CAAA,QAAA,CAAS,MAAM,CAAA;AAAA,WAC9C,MAAA,IAAA,CAAC,QAAS,CAAA,MAAA,IAAU,SAAS,SAAW,EAAA;AAIjD,YAAS,QAAA,CAAA,MAAA,GAASC,2BAAe,CAAA,QAAA,CAAS,SAAS,CAAA;AAAA;AACrD;AACF;AACF;AAGF,IAAO,OAAA;AAAA,MACL,QAAA;AAAA,MACA;AAAA,KACF;AAAA;AACF,EAEA,MAAM,cACJ,OACgC,EAAA;AAChC,IAAM,MAAA,MAAA,uBAAa,GAAoB,EAAA;AAEvC,IAAA,KAAA,MAAW,KAAS,IAAAC,YAAA,CAAY,OAAQ,CAAA,UAAA,EAAY,GAAG,CAAG,EAAA;AACxD,MAAA,IAAI,KAAQ,GAAA,IAAA,CAAK,QAA6B,CAAA,gBAAgB,CAC3D,CAAA,SAAA;AAAA,QACC,eAAA;AAAA,QACA,yBAAA;AAAA,QACA;AAAA,QAED,MAAO,CAAA;AAAA,QACN,SAAW,EAAA,0BAAA;AAAA,QACX,MAAQ,EAAA;AAAA,OACT,CAAA,CACA,OAAQ,CAAA,0BAAA,EAA4B,KAAK,CAAA;AAE5C,MAAA,IAAI,SAAS,MAAQ,EAAA;AACnB,QAAQ,KAAA,GAAA,WAAA;AAAA,UACN,OAAQ,CAAA,MAAA;AAAA,UACR,KAAA;AAAA,UACA,IAAK,CAAA,QAAA;AAAA,UACL,KAAA;AAAA,UACA;AAAA,SACF;AAAA;AAGF,MAAW,KAAA,MAAA,GAAA,IAAO,MAAM,KAAO,EAAA;AAC7B,QAAO,MAAA,CAAA,GAAA,CAAI,GAAI,CAAA,SAAA,EAAW,GAAI,CAAA,MAAA,GAAS,KAAK,KAAM,CAAA,GAAA,CAAI,MAAM,CAAA,GAAI,IAAI,CAAA;AAAA;AACtE;AAGF,IAAI,IAAA,KAAA,GAAQ,QAAQ,UAAW,CAAA,GAAA,CAAI,SAAO,MAAO,CAAA,GAAA,CAAI,GAAG,CAAA,IAAK,IAAI,CAAA;AAEjE,IAAA,IAAI,QAAQ,MAAQ,EAAA;AAClB,MAAA,KAAA,GAAQ,MAAM,GAAI,CAAA,CAAA,CAAA,KAAK,KAAK,OAAQ,CAAA,MAAA,CAAQ,CAAC,CAAC,CAAA;AAAA;AAGhD,IAAA,OAAO,EAAE,KAAM,EAAA;AAAA;AACjB,EAEA,MAAM,cACJ,OACgC,EAAA;AAChC,IAAA,MAAM,KAAK,IAAK,CAAA,QAAA;AAEhB,IAAM,MAAA,KAAA,GAAQ,QAAQ,KAAS,IAAA,aAAA;AAE/B,IAAA,MAAM,MAEF,GAAA;AAAA,MACF,WAAA,EAAa,CAAC,gBAAgB,CAAA;AAAA,MAC9B,UAAY,EAAA,KAAA;AAAA,MACZ,GAAG,uBAAuB,OAAO;AAAA,KACnC;AAEA,IAAA,MAAM,sBAAsB,MAAO,CAAA,UAAA;AAEnC,IAAI,IAAA,MAAA,CAAO,WAAY,CAAA,MAAA,GAAS,CAAG,EAAA;AACjC,MAAK,IAAA,CAAA,MAAA,CAAO,KAAK,CAAqD,mDAAA,CAAA,CAAA;AAAA;AAGxE,IAAA,MAAM,SAAyB,GAAA;AAAA,MAC7B,GAAG,gBAAA;AAAA,MACH,GAAG,MAAO,CAAA,WAAA,CAAY,CAAC;AAAA,KACzB;AAEA,IAAA,MAAM,CAAC,uBAAyB,EAAA,WAAW,CACzC,GAAA,MAAA,CAAO,oBAAoB,EAAC;AAE9B,IAAM,MAAA,OAAA,GAAU,EAAG,CAAA,gBAAgB,CAAE,CAAA,aAAA;AAAA,MAAc,QAAA;AAAA,MAAU,CAAA,EAAA,KAC3D,GACG,EAAG,CAAA,kBAAA,EAAoB,0BAA0B,CACjD,CAAA,QAAA,CAAS,YAAc,EAAA,SAAA,CAAU,KAAK;AAAA,KAC3C;AAEA,IAAA,IAAI,OAAO,MAAQ,EAAA;AACjB,MAAA,WAAA;AAAA,QACE,MAAO,CAAA,MAAA;AAAA,QACP,OAAA;AAAA,QACA,EAAA;AAAA,QACA,KAAA;AAAA,QACA;AAAA,OACF;AAAA;AAGF,IAAA,MAAM,4BAA+B,GAAA,MAAA,CAAO,cAAgB,EAAA,IAAA,EAAM,IAAK,EAAA;AACvE,IAAA,MAAM,mBAAmB,MAAO,CAAA,cAAA,EAAgB,MAAU,IAAA,CAAC,UAAU,KAAK,CAAA;AAC1E,IAAA,IAAI,4BAA8B,EAAA;AAChC,MAAA,IACE,iBAAiB,MAAW,KAAA,CAAA,IAC5B,iBAAiB,CAAC,CAAA,KAAM,UAAU,KAClC,EAAA;AAGA,QAAQ,OAAA,CAAA,WAAA;AAAA,UACN,cAAA;AAAA,UACA,CAAI,CAAA,EAAA,4BAAA,CAA6B,iBAAkB,CAAA,OAAO,CAAC,CAAA,CAAA;AAAA,SAC7D;AAAA,OACK,MAAA;AACL,QAAA,MAAM,UAAa,GAAA,EAAA,CAAgB,QAAQ,CAAA,CACxC,MAAO,CAAA,kBAAkB,CACzB,CAAA,OAAA,CAAQ,KAAO,EAAA,gBAAgB,CAC/B,CAAA,QAAA,CAAS,SAAS,SAAY,GAAA;AAC7B,UAAK,IAAA,CAAA,WAAA;AAAA,YACH,cAAA;AAAA,YACA,CAAI,CAAA,EAAA,4BAAA,CAA6B,iBAAkB,CAAA,OAAO,CAAC,CAAA,CAAA;AAAA,WAC7D;AAAA,SACD,CAAA;AACH,QAAQ,OAAA,CAAA,QAAA,CAAS,0BAA4B,EAAA,IAAA,EAAM,UAAU,CAAA;AAAA;AAC/D;AAGF,IAAM,MAAA,UAAA,GAAa,QAAQ,KAAM,EAAA;AAEjC,IAAM,MAAA,oBAAA,GAAuB,UAAU,KAAU,KAAA,MAAA;AAEjD,IAAA,IAAI,uBAAyB,EAAA;AAC3B,MAAQ,OAAA,CAAA,QAAA,CAAS,SAAS,MAAS,GAAA;AACjC,QAAK,IAAA,CAAA,KAAA;AAAA,UACH,OAAA;AAAA,UACA,mBAAA,KAAwB,uBAAuB,GAAM,GAAA,GAAA;AAAA,UACrD;AAAA,SAEC,CAAA,OAAA,CAAQ,OAAS,EAAA,GAAA,EAAK,uBAAuB,CAC7C,CAAA,QAAA;AAAA,UACC,0BAAA;AAAA,UACA,mBAAA,KAAwB,uBAAuB,GAAM,GAAA,GAAA;AAAA,UACrD;AAAA,SACF;AAAA,OACH,CAAA;AAAA;AAGH,IAAA,IAAI,EAAG,CAAA,MAAA,CAAO,MAAO,CAAA,MAAA,KAAW,IAAM,EAAA;AAEpC,MAAA,OAAA,CAAQ,OAAQ,CAAA;AAAA,QACd;AAAA,UACE,MAAQ,EAAA,cAAA;AAAA,UACR,OAAO,mBACH,GAAA,WAAA,CAAY,SAAU,CAAA,KAAK,IAC3B,SAAU,CAAA,KAAA;AAAA,UACd,KAAO,EAAA;AAAA,SACT;AAAA,QACA;AAAA,UACE,MAAQ,EAAA,0BAAA;AAAA,UACR,OAAO,mBACH,GAAA,WAAA,CAAY,SAAU,CAAA,KAAK,IAC3B,SAAU,CAAA;AAAA;AAChB,OACD,CAAA;AAAA,KACI,MAAA;AAIL,MAAA,OAAA,CAAQ,OAAQ,CAAA;AAAA,QACd;AAAA,UACE,MAAQ,EAAA,cAAA;AAAA,UACR,KAAO,EAAA,KAAA,CAAA;AAAA,UACP,KAAO,EAAA;AAAA,SACT;AAAA,QACA;AAAA,UACE,MAAQ,EAAA,cAAA;AAAA,UACR,OAAO,mBACH,GAAA,WAAA,CAAY,SAAU,CAAA,KAAK,IAC3B,SAAU,CAAA;AAAA,SAChB;AAAA,QACA;AAAA,UACE,MAAQ,EAAA,0BAAA;AAAA,UACR,OAAO,mBACH,GAAA,WAAA,CAAY,SAAU,CAAA,KAAK,IAC3B,SAAU,CAAA;AAAA;AAChB,OACD,CAAA;AAAA;AAGH,IAAA,IACEC,kCAA8B,CAAA,OAAO,CACrC,IAAA,OAAA,CAAQ,WAAW,KACnB,CAAA,EAAA;AACA,MAAQ,OAAA,CAAA,MAAA,CAAO,QAAQ,MAAM,CAAA;AAAA;AAG/B,IAAA,OAAA,CAAQ,KAAM,CAAA,mBAAA,GAAsB,KAAQ,GAAA,KAAA,GAAQ,CAAC,CAAA;AAErD,IAAA,UAAA,CAAW,KAAM,CAAA,0BAAA,EAA4B,EAAE,EAAA,EAAI,SAAS,CAAA;AAE5D,IAAM,MAAA,CAAC,IAAM,EAAA,CAAC,EAAE,KAAA,EAAO,CAAC,CAAA,GAAI,MAAM,OAAA,CAAQ,GAAI,CAAA;AAAA,MAC5C,KAAA,GAAQ,CAAI,GAAA,OAAA,GAAU,EAAC;AAAA;AAAA;AAAA;AAAA;AAAA,MAKvB,OAAO,MAAO,CAAA,UAAA,KAAe,WACzB,GAAA,UAAA,GACA,CAAC,EAAE,KAAA,EAAO,MAAO,CAAA,UAAA,EAAY;AAAA,KAClC,CAAA;AAED,IAAM,MAAA,UAAA,GAAa,OAAO,KAAK,CAAA;AAE/B,IAAA,IAAI,mBAAqB,EAAA;AACvB,MAAA,IAAA,CAAK,OAAQ,EAAA;AAAA;AAEf,IAAA,MAAM,cACJ,GAAA,KAAA,GAAQ,CAAM,KAAA,mBAAA,IAAuB,KAAK,MAAS,GAAA,KAAA,CAAA;AAGrD,IAAI,IAAA,IAAA,CAAK,SAAS,KAAO,EAAA;AACvB,MAAA,IAAA,CAAK,MAAU,IAAA,CAAA;AAAA;AAGjB,IAAM,MAAA,gBAAA,GAAmB,OAAO,oBAAyB,KAAA,KAAA,CAAA;AAEzD,IAAM,MAAA,QAAA,GAAW,KAAK,CAAC,CAAA;AACvB,IAAA,MAAM,OAAU,GAAA,IAAA,CAAK,IAAK,CAAA,MAAA,GAAS,CAAC,CAAA;AAEpC,IAAM,MAAA,oBAAA,GAAuB,OAAO,oBAAwB,IAAA;AAAA,MAC1D,QAAU,EAAA,KAAA;AAAA,MACV,QAAU,EAAA;AAAA,KACZ;AAEA,IAAA,MAAM,aAAiC,cACnC,GAAA;AAAA,MACE,GAAG,MAAA;AAAA,MACH,gBAAA,EAAkB,kBAAkB,OAAO,CAAA;AAAA,MAC3C,oBAAA;AAAA,MACA,UAAY,EAAA,KAAA;AAAA,MACZ;AAAA,KAEF,GAAA,KAAA,CAAA;AAEJ,IAAA,MAAM,UACJ,GAAA,CAAC,gBACD,IAAA,IAAA,CAAK,MAAS,GAAA,CAAA,IACd,CAACC,cAAA,CAAQ,iBAAkB,CAAA,QAAQ,CAAG,EAAA,MAAA,CAAO,oBAAoB,CAC7D,GAAA;AAAA,MACE,GAAG,MAAA;AAAA,MACH,gBAAA,EAAkB,kBAAkB,QAAQ,CAAA;AAAA,MAC5C,sBAAsB,MAAO,CAAA,oBAAA;AAAA,MAC7B,UAAY,EAAA,IAAA;AAAA,MACZ;AAAA,KAEF,GAAA,KAAA,CAAA;AAEN,IAAA,MAAM,QAAQ,IACX,CAAA,GAAA,CAAI,OAAK,IAAK,CAAA,KAAA,CAAM,EAAE,YAAa,CAAC,CACpC,CAAA,GAAA,CAAI,OAAM,OAAQ,CAAA,MAAA,GAAS,QAAQ,MAAO,CAAA,CAAC,IAAI,CAAE,CAAA;AAEpD,IAAO,OAAA;AAAA,MACL,KAAA;AAAA,MACA,QAAU,EAAA;AAAA,QACR,GAAI,CAAC,CAAC,UAAA,IAAc,EAAE,UAAW,EAAA;AAAA,QACjC,GAAI,CAAC,CAAC,UAAA,IAAc,EAAE,UAAW;AAAA,OACnC;AAAA,MACA;AAAA,KACF;AAAA;AACF,EAEA,MAAM,kBAAkB,GAA4B,EAAA;AAClD,IAAM,MAAA,QAAA,GAAW,IAAK,CAAA,QAAA,CAAS,MAAO,CAAA,MAAA;AAUtC,IAAA,IAAI,QAAS,CAAA,MAAA,CAAO,QAAS,CAAA,OAAO,CAAG,EAAA;AAGrC,MAAA,MAAM,OAAU,GAAA,MAAM,IAAK,CAAA,QAAA,CAA4B,eAAe,CAAA,CACnE,MAAO,CAAA,WAAW,CAClB,CAAA,OAAA,CAAQ,YAAc,EAAA,SAAS,QAAQ,OAAS,EAAA;AAC/C,QAAO,OAAA,OAAA,CACJ,IAAwB,CAAA,eAAe,CACvC,CAAA,SAAA;AAAA,UACC,0BAAA;AAAA,UACA;AAAA,YACE,4CACE,EAAA;AAAA;AACJ,UAED,KAAM,CAAA,yBAAA,EAA2B,KAAK,GAAG,CAAA,CACzC,OAAO,4CAA4C,CAAA;AAAA,OACvD,CAAA;AACH,MAAA,MAAM,IAAK,CAAA,QAAA,CAA4B,eAAe,CAAA,CACnD,MAAO,CAAA;AAAA,QACN,WAAa,EAAA,mBAAA;AAAA,QACb,cAAgB,EAAA,IAAA,CAAK,QAAS,CAAA,EAAA,CAAG,GAAI;AAAA,OACtC,CACA,CAAA,OAAA;AAAA,QACC,WAAA;AAAA,QACA,OAAQ,CAAA,GAAA,CAAI,CAAO,GAAA,KAAA,GAAA,CAAI,SAAS;AAAA,OAClC;AAAA,KACG,MAAA;AACL,MAAA,MAAM,IAAK,CAAA,QAAA,CAA4B,eAAe,CAAA,CACnD,MAAO,CAAA;AAAA,QACN,WAAa,EAAA,mBAAA;AAAA,QACb,cAAgB,EAAA,IAAA,CAAK,QAAS,CAAA,EAAA,CAAG,GAAI;AAAA,OACtC,CACA,CAAA,OAAA,CAAQ,YAAc,EAAA,SAAS,QAAQ,OAAS,EAAA;AAC/C,QAAO,OAAA,OAAA,CACJ,IAAwB,CAAA,eAAe,CACvC,CAAA,SAAA;AAAA,UACC,0BAAA;AAAA,UACA;AAAA,YACE,4CACE,EAAA;AAAA;AACJ,UAED,KAAM,CAAA,yBAAA,EAA2B,KAAK,GAAG,CAAA,CACzC,OAAO,4CAA4C,CAAA;AAAA,OACvD,CAAA;AAAA;AAOL,IAAM,MAAA,aAAA,GAAgB,MAAM,IAAK,CAAA,QAAA,CAC9B,KAAqB,WAAW,CAAA,CAChC,UAAuC,eAAiB,EAAA;AAAA,MACvD,0BAA4B,EAAA;AAAA,KAC7B,CACA,CAAA,KAAA,CAAM,iCAAmC,EAAA,GAAA,EAAK,GAAG,CACjD,CAAA,QAAA,CAAS,yBAA2B,EAAA,IAAA,EAAM,GAAG,CAC7C,CAAA,MAAA,CAAO,EAAE,GAAK,EAAA,6BAAA,EAA+B,CAC7C,CAAA,KAAA;AAAA,MAAM,WACL,KACG,CAAA,IAAA,CAAqB,WAAW,CAAA,CAChC,UAAuC,eAAiB,EAAA;AAAA,QACvD,0BAA4B,EAAA;AAAA,OAC7B,CACA,CAAA,KAAA,CAAM,iCAAmC,EAAA,GAAA,EAAK,GAAG,CACjD,CAAA,QAAA,CAAS,yBAA2B,EAAA,IAAA,EAAM,GAAG,CAC7C,CAAA,MAAA,CAAO,EAAE,GAAA,EAAK,+BAA+B;AAAA,KAClD;AAEF,IAAM,MAAA,IAAA,CAAK,SAA4B,eAAe,CAAA,CACnD,MAAM,WAAa,EAAA,GAAG,EACtB,MAAO,EAAA;AAEV,IAAM,MAAA,IAAA,CAAK,SAAS,MAAO,CAAA;AAAA,MACzB,UAAA,EAAY,IAAI,GAAI,CAAA,aAAA,CAAc,IAAI,CAAK,CAAA,KAAA,CAAA,CAAE,GAAG,CAAC;AAAA,KAClD,CAAA;AAAA;AACH,EAEA,MAAM,eAAe,OAAkD,EAAA;AACrE,IAAM,MAAA,CAAC,OAAO,CAAI,GAAA,MAAM,KAAK,QAA4B,CAAA,eAAe,CACrE,CAAA,QAAA,CAA6B,gBAAkB,EAAA;AAAA,MAC9C,yBAA2B,EAAA;AAAA,KAC5B,CACA,CAAA,KAAA,CAAM,4BAA4B,GAAK,EAAA,OAAO,EAC9C,MAAO,CAAA;AAAA,MACN,UAAY,EAAA;AAAA,KACb,CAAA;AAEH,IAAA,IAAI,CAAC,OAAS,EAAA;AACZ,MAAA,MAAM,IAAIC,oBAAA,CAAc,CAAkB,eAAA,EAAA,OAAO,CAAE,CAAA,CAAA;AAAA;AAGrD,IAAA,MAAM,UAAa,GAAA,IAAA,CAAK,KAAM,CAAA,OAAA,CAAQ,UAAU,CAAA;AAChD,IAAM,MAAA,cAAA,uBAAqB,GAAY,EAAA;AACvC,IAAM,MAAA,IAAA,GAAO,IAAI,KAAc,EAAA;AAC/B,IAAM,MAAA,KAAA,GAAQ,IAAI,KAAsD,EAAA;AAExE,IAAA,KAAA,IACM,UAA8B,UAClC,EAAA,OAAA,EACA,OAAU,GAAA,IAAA,CAAK,KACf,EAAA;AACA,MAAM,MAAA,UAAA,GAAaL,gCAAmB,OAAO,CAAA;AAC7C,MAAA,cAAA,CAAe,IAAI,UAAU,CAAA;AAE7B,MAAM,MAAA,UAAA,GAAa,MAAM,IAAK,CAAA,QAAA;AAAA,QAC5B;AAAA,OACF,CACG,UAA6B,eAAiB,EAAA;AAAA,QAC7C,4CACE,EAAA;AAAA,OACH,CACA,CAAA,SAAA,CAA8B,gBAAkB,EAAA;AAAA,QAC/C,yBAA2B,EAAA;AAAA,OAC5B,CACA,CAAA,KAAA,CAAM,8CAA8C,GAAK,EAAA,UAAU,EACnE,MAAO,CAAA;AAAA,QACN,eAAiB,EAAA,0BAAA;AAAA,QACjB,gBAAkB,EAAA;AAAA,OACnB,CAAA;AAEH,MAAA,MAAM,aAAuB,EAAC;AAC9B,MAAA,KAAA,MAAW,EAAE,eAAA,EAAiB,gBAAiB,EAAA,IAAK,UAAY,EAAA;AAC9D,QAAA,UAAA,CAAW,KAAK,eAAe,CAAA;AAC/B,QAAA,IAAI,CAAC,cAAA,CAAe,GAAI,CAAA,eAAe,CAAG,EAAA;AACxC,UAAA,cAAA,CAAe,IAAI,eAAe,CAAA;AAClC,UAAA,IAAA,CAAK,IAAK,CAAA,IAAA,CAAK,KAAM,CAAA,gBAAgB,CAAC,CAAA;AAAA;AACxC;AAGF,MAAA,KAAA,CAAM,IAAK,CAAA;AAAA,QACT,MAAQ,EAAA,OAAA;AAAA,QACR,gBAAkB,EAAA;AAAA,OACnB,CAAA;AAAA;AAGH,IAAO,OAAA;AAAA,MACL,aAAA,EAAeA,gCAAmB,UAAU,CAAA;AAAA,MAC5C;AAAA,KACF;AAAA;AACF,EAEA,MAAM,OAAO,OAA6D,EAAA;AACxE,IAAA,MAAM,SAAyC,EAAC;AAChD,IAAA,MAAM,KAAK,IAAK,CAAA,QAAA;AAEhB,IAAW,KAAA,MAAA,KAAA,IAAS,QAAQ,MAAQ,EAAA;AAClC,MAAM,MAAA,OAAA,GAAU,EAAgB,CAAA,QAAQ,CACrC,CAAA,KAAA,CAAM,YAAc,EAAA,KAAA,CAAM,iBAAkB,CAAA,OAAO,CAAC,CAAA,CACpD,YAAa,CAAA,uBAAuB,EACpC,MAAO,CAAA,EAAE,KAAO,EAAA,uBAAA,EAAyB,KAAO,EAAA,EAAA,CAAG,GAAI,CAAA,UAAU,CAAE,EAAC,CACpE,CAAA,OAAA,CAAQ,uBAAuB,CAAA;AAElC,MAAA,IAAI,SAAS,MAAQ,EAAA;AACnB,QAAA,WAAA,CAAY,OAAQ,CAAA,MAAA,EAAQ,OAAS,EAAA,EAAA,EAAI,OAAO,kBAAkB,CAAA;AAAA;AAGpE,MAAA,MAAM,SAAS,MAAM,OAAA;AAErB,MAAA,MAAA,CAAO,KAAK,CAAA,GAAI,MAAO,CAAA,GAAA,CAAI,CAAS,IAAA,MAAA;AAAA,QAClC,KAAA,EAAO,MAAO,CAAA,IAAA,CAAK,KAAK,CAAA;AAAA,QACxB,KAAA,EAAO,MAAO,CAAA,IAAA,CAAK,KAAK;AAAA,OACxB,CAAA,CAAA;AAAA;AAGJ,IAAA,OAAO,EAAE,MAAO,EAAA;AAAA;AAEpB;AAEA,MAAM,qBAAgDM,KAAE,CAAA,IAAA;AAAA,EAAK,MAC3DA,MACG,MAAO,CAAA;AAAA,IACN,GAAA,EAAKA,MAAE,MAAO,EAAA;AAAA,IACd,QAAQA,KAAE,CAAA,KAAA,CAAMA,MAAE,MAAO,EAAC,EAAE,QAAS;AAAA,GACtC,CAAA,CACA,EAAG,CAAAA,KAAA,CAAE,OAAO,EAAE,GAAA,EAAK,kBAAmB,EAAC,CAAC,CAAA,CACxC,EAAG,CAAAA,KAAA,CAAE,OAAO,EAAE,KAAA,EAAOA,KAAE,CAAA,KAAA,CAAM,kBAAkB,CAAA,EAAG,CAAC,EACnD,EAAG,CAAAA,KAAA,CAAE,MAAO,CAAA,EAAE,OAAOA,KAAE,CAAA,KAAA,CAAM,kBAAkB,CAAA,EAAG,CAAC;AACxD,CAAA;AAEiDA,MAAE,MAAO,CAAA;AAAA,EACxD,aAAaA,KAAE,CAAA,KAAA;AAAA,IACbA,KAAE,CAAA,MAAA,CAAO,EAAE,KAAA,EAAOA,MAAE,MAAO,EAAA,EAAG,KAAO,EAAAA,KAAA,CAAE,KAAK,CAAC,KAAA,EAAO,MAAM,CAAC,GAAG;AAAA,GAChE;AAAA,EACA,gBAAA,EAAkBA,KAAE,CAAA,KAAA,CAAMA,KAAE,CAAA,MAAA,GAAS,EAAG,CAAAA,KAAA,CAAE,IAAK,EAAC,CAAC,CAAA;AAAA,EACjD,MAAA,EAAQ,mBAAmB,QAAS,EAAA;AAAA,EACpC,UAAA,EAAYA,MAAE,OAAQ,EAAA;AAAA,EACtB,KAAO,EAAAA,KAAA,CAAE,MAAO,EAAA,CAAE,QAAS,EAAA;AAAA,EAC3B,oBAAsB,EAAAA,KAAA,CAAE,KAAM,CAAAA,KAAA,CAAE,MAAO,EAAA,CAAE,EAAG,CAAAA,KAAA,CAAE,IAAK,EAAC,CAAC,CAAA,CAAE,QAAS,EAAA;AAAA,EAChE,UAAY,EAAAA,KAAA,CAAE,MAAO,EAAA,CAAE,QAAS;AAClC,CAAC;AAED,SAAS,uBACP,OACiB,EAAA;AACjB,EAAI,IAAAH,kCAAA,CAA8B,OAAO,CAAG,EAAA;AAC1C,IAAM,MAAA;AAAA,MACJ,MAAA;AAAA,MACA,WAAA,EAAa,UAAa,GAAA,CAAC,gBAAgB,CAAA;AAAA,MAC3C;AAAA,KACE,GAAA,OAAA;AACJ,IAAA,OAAO,EAAE,MAAA,EAAQ,WAAa,EAAA,UAAA,EAAY,cAAe,EAAA;AAAA;AAE3D,EAAI,IAAAI,iCAAA,CAA6B,OAAO,CAAG,EAAA;AACzC,IAAA,OAAO,OAAQ,CAAA,MAAA;AAAA;AAEjB,EAAA,OAAO,EAAC;AACV;AAEA,SAAS,YAAY,KAA6B,EAAA;AAChD,EAAO,OAAA,KAAA,KAAU,QAAQ,MAAS,GAAA,KAAA;AACpC;AAEA,SAAS,kBAAkB,GAAkB,EAAA;AAC3C,EAAA,OAAO,CAAC,GAAA,CAAI,KAAO,EAAA,GAAA,CAAI,SAAS,CAAA;AAClC;;;;"}
+{"version":3,"file":"DefaultEntitiesCatalog.cjs.js","sources":["../../src/service/DefaultEntitiesCatalog.ts"],"sourcesContent":["/*\n * Copyright 2020 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  Entity,\n  parseEntityRef,\n  stringifyEntityRef,\n} from '@backstage/catalog-model';\nimport { InputError, NotFoundError } from '@backstage/errors';\nimport { Knex } from 'knex';\nimport { chunk as lodashChunk, isEqual } from 'lodash';\nimport { z } from 'zod';\nimport {\n  Cursor,\n  EntitiesBatchRequest,\n  EntitiesBatchResponse,\n  EntitiesCatalog,\n  EntitiesRequest,\n  EntitiesResponse,\n  EntityAncestryResponse,\n  EntityFacetsRequest,\n  EntityFacetsResponse,\n  EntityOrder,\n  EntityPagination,\n  QueryEntitiesRequest,\n  QueryEntitiesResponse,\n} from '../catalog/types';\nimport {\n  DbFinalEntitiesRow,\n  DbPageInfo,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n  DbSearchRow,\n} from '../database/tables';\nimport { Stitcher } from '../stitching/types';\n\nimport {\n  isQueryEntitiesCursorRequest,\n  isQueryEntitiesInitialRequest,\n} from './util';\nimport {\n  EntitiesSearchFilter,\n  EntityFilter,\n} from '@backstage/plugin-catalog-node';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\nconst defaultSortField: EntityOrder = {\n  field: 'metadata.uid',\n  order: 'asc',\n};\n\nconst DEFAULT_LIMIT = 20;\n\nfunction parsePagination(input?: EntityPagination): EntityPagination {\n  if (!input) {\n    return {};\n  }\n\n  let { limit, offset } = input;\n\n  if (input.after === undefined) {\n    return { limit, offset };\n  }\n\n  let cursor;\n  try {\n    const json = Buffer.from(input.after, 'base64').toString('utf8');\n    cursor = JSON.parse(json);\n  } catch {\n    throw new InputError('Malformed after cursor, could not be parsed');\n  }\n\n  if (cursor.limit !== undefined) {\n    if (!Number.isInteger(cursor.limit)) {\n      throw new InputError('Malformed after cursor, limit was not an number');\n    }\n    limit = cursor.limit;\n  }\n\n  if (cursor.offset !== undefined) {\n    if (!Number.isInteger(cursor.offset)) {\n      throw new InputError('Malformed after cursor, offset was not a number');\n    }\n    offset = cursor.offset;\n  }\n\n  return { limit, offset };\n}\n\nfunction stringifyPagination(\n  input: Required<Omit<EntityPagination, 'after'>>,\n): string {\n  const { limit, offset } = input;\n  const json = JSON.stringify({ limit, offset });\n  const base64 = Buffer.from(json, 'utf8').toString('base64');\n  return base64;\n}\n\nfunction addCondition(\n  queryBuilder: Knex.QueryBuilder,\n  db: Knex,\n  filter: EntitiesSearchFilter,\n  negate: boolean = false,\n  entityIdField = 'entity_id',\n): void {\n  const key = filter.key.toLowerCase();\n  const values = filter.values?.map(v => v.toLowerCase());\n\n  // NOTE(freben): This used to be a set of OUTER JOIN, which may seem to\n  // make a lot of sense. However, it had abysmal performance on sqlite\n  // when datasets grew large, so we're using IN instead.\n  const matchQuery = db<DbSearchRow>('search')\n    .select('search.entity_id')\n    .where({ key })\n    .andWhere(function keyFilter() {\n      if (values?.length === 1) {\n        this.where({ value: values.at(0) });\n      } else if (values) {\n        this.andWhere('value', 'in', values);\n      }\n    });\n  queryBuilder.andWhere(entityIdField, negate ? 'not in' : 'in', matchQuery);\n}\n\nfunction isEntitiesSearchFilter(\n  filter: EntitiesSearchFilter | EntityFilter,\n): filter is EntitiesSearchFilter {\n  return filter.hasOwnProperty('key');\n}\n\nfunction isOrEntityFilter(\n  filter: { anyOf: EntityFilter[] } | EntityFilter,\n): filter is { anyOf: EntityFilter[] } {\n  return filter.hasOwnProperty('anyOf');\n}\n\nfunction isNegationEntityFilter(\n  filter: { not: EntityFilter } | EntityFilter,\n): filter is { not: EntityFilter } {\n  return filter.hasOwnProperty('not');\n}\n\nfunction parseFilter(\n  filter: EntityFilter,\n  query: Knex.QueryBuilder,\n  db: Knex,\n  negate: boolean = false,\n  entityIdField = 'entity_id',\n): Knex.QueryBuilder {\n  if (isNegationEntityFilter(filter)) {\n    return parseFilter(filter.not, query, db, !negate, entityIdField);\n  }\n\n  if (isEntitiesSearchFilter(filter)) {\n    return query.andWhere(function filterFunction() {\n      addCondition(this, db, filter, negate, entityIdField);\n    });\n  }\n\n  return query[negate ? 'andWhereNot' : 'andWhere'](function filterFunction() {\n    if (isOrEntityFilter(filter)) {\n      for (const subFilter of filter.anyOf ?? []) {\n        this.orWhere(subQuery =>\n          parseFilter(subFilter, subQuery, db, false, entityIdField),\n        );\n      }\n    } else {\n      for (const subFilter of filter.allOf ?? []) {\n        this.andWhere(subQuery =>\n          parseFilter(subFilter, subQuery, db, false, entityIdField),\n        );\n      }\n    }\n  });\n}\n\nexport class DefaultEntitiesCatalog implements EntitiesCatalog {\n  private readonly database: Knex;\n  private readonly logger: LoggerService;\n  private readonly stitcher: Stitcher;\n\n  constructor(options: {\n    database: Knex;\n    logger: LoggerService;\n    stitcher: Stitcher;\n  }) {\n    this.database = options.database;\n    this.logger = options.logger;\n    this.stitcher = options.stitcher;\n  }\n\n  async entities(request?: EntitiesRequest): Promise<EntitiesResponse> {\n    const db = this.database;\n\n    let entitiesQuery =\n      db<DbFinalEntitiesRow>('final_entities').select('final_entities.*');\n\n    request?.order?.forEach(({ field }, index) => {\n      const alias = `order_${index}`;\n      entitiesQuery = entitiesQuery.leftOuterJoin(\n        { [alias]: 'search' },\n        function search(inner) {\n          inner\n            .on(`${alias}.entity_id`, 'final_entities.entity_id')\n            .andOn(`${alias}.key`, db.raw('?', [field]));\n        },\n      );\n    });\n\n    entitiesQuery = entitiesQuery.whereNotNull('final_entities.final_entity');\n\n    if (request?.filter) {\n      entitiesQuery = parseFilter(\n        request.filter,\n        entitiesQuery,\n        db,\n        false,\n        'final_entities.entity_id',\n      );\n    }\n\n    request?.order?.forEach(({ order }, index) => {\n      if (db.client.config.client === 'pg') {\n        // pg correctly orders by the column value and handling nulls in one go\n        entitiesQuery = entitiesQuery.orderBy([\n          { column: `order_${index}.value`, order, nulls: 'last' },\n        ]);\n      } else {\n        // sqlite and mysql translate the above statement ONLY into \"order by (value is null) asc\"\n        // no matter what the order is, for some reason, so we have to manually add back the statement\n        // that translates to \"order by value <order>\" while avoiding to give an order\n        entitiesQuery = entitiesQuery.orderBy([\n          { column: `order_${index}.value`, order: undefined, nulls: 'last' },\n          { column: `order_${index}.value`, order },\n        ]);\n      }\n    });\n\n    if (!request?.order) {\n      entitiesQuery = entitiesQuery.orderBy('final_entities.entity_ref', 'asc'); // default sort\n    } else {\n      entitiesQuery.orderBy('final_entities.entity_id', 'asc'); // stable sort\n    }\n\n    const { limit, offset } = parsePagination(request?.pagination);\n    if (limit !== undefined) {\n      entitiesQuery = entitiesQuery.limit(limit + 1);\n    }\n    if (offset !== undefined) {\n      entitiesQuery = entitiesQuery.offset(offset);\n    }\n\n    let rows = await entitiesQuery;\n    let pageInfo: DbPageInfo;\n    if (limit === undefined || rows.length <= limit) {\n      pageInfo = { hasNextPage: false };\n    } else {\n      rows = rows.slice(0, -1);\n      pageInfo = {\n        hasNextPage: true,\n        endCursor: stringifyPagination({\n          limit,\n          offset: (offset ?? 0) + limit,\n        }),\n      };\n    }\n\n    let entities: Entity[] = rows.map(e => JSON.parse(e.final_entity!));\n\n    if (request?.fields) {\n      entities = entities.map(e => request.fields!(e));\n    }\n\n    // TODO(freben): This is added as a compatibility guarantee, until we can be\n    // sure that all adopters have re-stitched their entities so that the new\n    // targetRef field is present on them, and that they have stopped consuming\n    // the now-removed old field\n    // TODO(jhaals): Remove this in April 2022\n    for (const entity of entities) {\n      if (entity.relations) {\n        for (const relation of entity.relations as any) {\n          if (!relation.targetRef && relation.target) {\n            // This is the case where an old-form entity, not yet stitched with\n            // the updated code, was in the database\n            relation.targetRef = stringifyEntityRef(relation.target);\n          } else if (!relation.target && relation.targetRef) {\n            // This is the case where a new-form entity, stitched with the\n            // updated code, was in the database but we still want to produce\n            // the old data shape as well for compatibility reasons\n            relation.target = parseEntityRef(relation.targetRef);\n          }\n        }\n      }\n    }\n\n    return {\n      entities,\n      pageInfo,\n    };\n  }\n\n  async entitiesBatch(\n    request: EntitiesBatchRequest,\n  ): Promise<EntitiesBatchResponse> {\n    const lookup = new Map<string, Entity>();\n\n    for (const chunk of lodashChunk(request.entityRefs, 200)) {\n      let query = this.database<DbFinalEntitiesRow>('final_entities')\n        .select({\n          entityRef: 'final_entities.entity_ref',\n          entity: 'final_entities.final_entity',\n        })\n        .whereIn('final_entities.entity_ref', chunk);\n\n      if (request?.filter) {\n        query = parseFilter(\n          request.filter,\n          query,\n          this.database,\n          false,\n          'final_entities.entity_id',\n        );\n      }\n\n      for (const row of await query) {\n        lookup.set(row.entityRef, row.entity ? JSON.parse(row.entity) : null);\n      }\n    }\n\n    let items = request.entityRefs.map(ref => lookup.get(ref) ?? null);\n\n    if (request.fields) {\n      items = items.map(e => e && request.fields!(e));\n    }\n\n    return { items };\n  }\n\n  async queryEntities(\n    request: QueryEntitiesRequest,\n  ): Promise<QueryEntitiesResponse> {\n    const db = this.database;\n\n    const limit = request.limit ?? DEFAULT_LIMIT;\n\n    const cursor: Omit<Cursor, 'orderFieldValues'> & {\n      orderFieldValues?: (string | null)[];\n    } = {\n      orderFields: [defaultSortField],\n      isPrevious: false,\n      ...parseCursorFromRequest(request),\n    };\n\n    const isFetchingBackwards = cursor.isPrevious;\n\n    if (cursor.orderFields.length > 1) {\n      this.logger.warn(`Only one sort field is supported, ignoring the rest`);\n    }\n\n    const sortField: EntityOrder = {\n      ...defaultSortField,\n      ...cursor.orderFields[0],\n    };\n\n    const [prevItemOrderFieldValue, prevItemUid] =\n      cursor.orderFieldValues || [];\n\n    const dbQuery = db('final_entities').leftOuterJoin('search', qb =>\n      qb\n        .on('search.entity_id', 'final_entities.entity_id')\n        .andOnVal('search.key', sortField.field),\n    );\n\n    if (cursor.filter) {\n      parseFilter(\n        cursor.filter,\n        dbQuery,\n        db,\n        false,\n        'final_entities.entity_id',\n      );\n    }\n\n    const normalizedFullTextFilterTerm = cursor.fullTextFilter?.term?.trim();\n    const textFilterFields = cursor.fullTextFilter?.fields ?? [sortField.field];\n    if (normalizedFullTextFilterTerm) {\n      if (\n        textFilterFields.length === 1 &&\n        textFilterFields[0] === sortField.field\n      ) {\n        // If there is one item, apply the like query to the top level query which is already\n        //   filtered based on the singular sortField.\n        dbQuery.andWhereRaw(\n          'value like ?',\n          `%${normalizedFullTextFilterTerm.toLocaleLowerCase('en-US')}%`,\n        );\n      } else {\n        const matchQuery = db<DbSearchRow>('search')\n          .select('search.entity_id')\n          // textFilterFields must be lowercased to match searchable keys in database, i.e. spec.profile.displayName -> spec.profile.displayname\n          .whereIn(\n            'key',\n            textFilterFields.map(field => field.toLocaleLowerCase('en-US')),\n          )\n          .andWhere(function keyFilter() {\n            this.andWhereRaw(\n              'value like ?',\n              `%${normalizedFullTextFilterTerm.toLocaleLowerCase('en-US')}%`,\n            );\n          });\n        dbQuery.andWhere('final_entities.entity_id', 'in', matchQuery);\n      }\n    }\n\n    const countQuery = dbQuery.clone();\n\n    const isOrderingDescending = sortField.order === 'desc';\n\n    if (prevItemOrderFieldValue) {\n      dbQuery.andWhere(function nested() {\n        this.where(\n          'value',\n          isFetchingBackwards !== isOrderingDescending ? '<' : '>',\n          prevItemOrderFieldValue,\n        )\n          .orWhere('value', '=', prevItemOrderFieldValue)\n          .andWhere(\n            'final_entities.entity_id',\n            isFetchingBackwards !== isOrderingDescending ? '<' : '>',\n            prevItemUid,\n          );\n      });\n    }\n\n    if (db.client.config.client === 'pg') {\n      // pg correctly orders by the column value and handling nulls in one go\n      dbQuery.orderBy([\n        {\n          column: 'search.value',\n          order: isFetchingBackwards\n            ? invertOrder(sortField.order)\n            : sortField.order,\n          nulls: 'last',\n        },\n        {\n          column: 'final_entities.entity_id',\n          order: isFetchingBackwards\n            ? invertOrder(sortField.order)\n            : sortField.order,\n        },\n      ]);\n    } else {\n      // sqlite and mysql translate the above statement ONLY into \"order by (value is null) asc\"\n      // no matter what the order is, for some reason, so we have to manually add back the statement\n      // that translates to \"order by value <order>\" while avoiding to give an order\n      dbQuery.orderBy([\n        {\n          column: 'search.value',\n          order: undefined,\n          nulls: 'last',\n        },\n        {\n          column: 'search.value',\n          order: isFetchingBackwards\n            ? invertOrder(sortField.order)\n            : sortField.order,\n        },\n        {\n          column: 'final_entities.entity_id',\n          order: isFetchingBackwards\n            ? invertOrder(sortField.order)\n            : sortField.order,\n        },\n      ]);\n    }\n\n    if (\n      isQueryEntitiesInitialRequest(request) &&\n      request.offset !== undefined\n    ) {\n      dbQuery.offset(request.offset);\n    }\n    // fetch an extra item to check if there are more items.\n    dbQuery.limit(isFetchingBackwards ? limit : limit + 1);\n\n    countQuery.count('final_entities.entity_id', { as: 'count' });\n\n    const [rows, [{ count }]] = await Promise.all([\n      limit > 0 ? dbQuery : [],\n      // for performance reasons we invoke the countQuery\n      // only on the first request.\n      // The result is then embedded into the cursor\n      // for subsequent requests.\n      typeof cursor.totalItems === 'undefined'\n        ? countQuery\n        : [{ count: cursor.totalItems }],\n    ]);\n\n    const totalItems = Number(count);\n\n    if (isFetchingBackwards) {\n      rows.reverse();\n    }\n    const hasMoreResults =\n      limit > 0 && (isFetchingBackwards || rows.length > limit);\n\n    // discard the extra item only when fetching forward.\n    if (rows.length > limit) {\n      rows.length -= 1;\n    }\n\n    const isInitialRequest = cursor.firstSortFieldValues === undefined;\n\n    const firstRow = rows[0];\n    const lastRow = rows[rows.length - 1];\n\n    const firstSortFieldValues = cursor.firstSortFieldValues || [\n      firstRow?.value,\n      firstRow?.entity_id,\n    ];\n\n    const nextCursor: Cursor | undefined = hasMoreResults\n      ? {\n          ...cursor,\n          orderFieldValues: sortFieldsFromRow(lastRow),\n          firstSortFieldValues,\n          isPrevious: false,\n          totalItems,\n        }\n      : undefined;\n\n    const prevCursor: Cursor | undefined =\n      !isInitialRequest &&\n      rows.length > 0 &&\n      !isEqual(sortFieldsFromRow(firstRow), cursor.firstSortFieldValues)\n        ? {\n            ...cursor,\n            orderFieldValues: sortFieldsFromRow(firstRow),\n            firstSortFieldValues: cursor.firstSortFieldValues,\n            isPrevious: true,\n            totalItems,\n          }\n        : undefined;\n\n    const items = rows\n      .map(e => JSON.parse(e.final_entity!))\n      .map(e => (request.fields ? request.fields(e) : e));\n\n    return {\n      items,\n      pageInfo: {\n        ...(!!prevCursor && { prevCursor }),\n        ...(!!nextCursor && { nextCursor }),\n      },\n      totalItems,\n    };\n  }\n\n  async removeEntityByUid(uid: string): Promise<void> {\n    const dbConfig = this.database.client.config;\n\n    // Clear the hashed state of the immediate parents of the deleted entity.\n    // This makes sure that when they get reprocessed, their output is written\n    // down again. The reason for wanting to do this, is that if the user\n    // deletes entities that ARE still emitted by the parent, the parent\n    // processing will still generate the same output hash as always, which\n    // means it'll never try to write down the children again (it assumes that\n    // they already exist). This means that without the code below, the database\n    // never \"heals\" from accidental deletes.\n    if (dbConfig.client.includes('mysql')) {\n      // MySQL doesn't support the syntax we need to do this in a single query,\n      // http://dev.mysql.com/doc/refman/5.6/en/update.html\n      const results = await this.database<DbRefreshStateRow>('refresh_state')\n        .select('entity_id')\n        .whereIn('entity_ref', function parents(builder) {\n          return builder\n            .from<DbRefreshStateRow>('refresh_state')\n            .innerJoin<DbRefreshStateReferencesRow>(\n              'refresh_state_references',\n              {\n                'refresh_state_references.target_entity_ref':\n                  'refresh_state.entity_ref',\n              },\n            )\n            .where('refresh_state.entity_id', '=', uid)\n            .select('refresh_state_references.source_entity_ref');\n        });\n      await this.database<DbRefreshStateRow>('refresh_state')\n        .update({\n          result_hash: 'child-was-deleted',\n          next_update_at: this.database.fn.now(),\n        })\n        .whereIn(\n          'entity_id',\n          results.map(key => key.entity_id),\n        );\n    } else {\n      await this.database<DbRefreshStateRow>('refresh_state')\n        .update({\n          result_hash: 'child-was-deleted',\n          next_update_at: this.database.fn.now(),\n        })\n        .whereIn('entity_ref', function parents(builder) {\n          return builder\n            .from<DbRefreshStateRow>('refresh_state')\n            .innerJoin<DbRefreshStateReferencesRow>(\n              'refresh_state_references',\n              {\n                'refresh_state_references.target_entity_ref':\n                  'refresh_state.entity_ref',\n              },\n            )\n            .where('refresh_state.entity_id', '=', uid)\n            .select('refresh_state_references.source_entity_ref');\n        });\n    }\n\n    // Stitch the entities that the deleted one had relations to. If we do not\n    // do this, the entities in the other end of the relations will still look\n    // like they have a relation to the entity that was deleted, despite not\n    // having any corresponding rows in the relations table.\n    const relationPeers = await this.database\n      .from<DbRelationsRow>('relations')\n      .innerJoin<DbRefreshStateReferencesRow>('refresh_state', {\n        'refresh_state.entity_ref': 'relations.target_entity_ref',\n      })\n      .where('relations.originating_entity_id', '=', uid)\n      .andWhere('refresh_state.entity_id', '!=', uid)\n      .select({ ref: 'relations.target_entity_ref' })\n      .union(other =>\n        other\n          .from<DbRelationsRow>('relations')\n          .innerJoin<DbRefreshStateReferencesRow>('refresh_state', {\n            'refresh_state.entity_ref': 'relations.source_entity_ref',\n          })\n          .where('relations.originating_entity_id', '=', uid)\n          .andWhere('refresh_state.entity_id', '!=', uid)\n          .select({ ref: 'relations.source_entity_ref' }),\n      );\n\n    await this.database<DbRefreshStateRow>('refresh_state')\n      .where('entity_id', uid)\n      .delete();\n\n    await this.stitcher.stitch({\n      entityRefs: new Set(relationPeers.map(p => p.ref)),\n    });\n  }\n\n  async entityAncestry(rootRef: string): Promise<EntityAncestryResponse> {\n    const [rootRow] = await this.database<DbFinalEntitiesRow>('final_entities')\n      .where('final_entities.entity_ref', '=', rootRef)\n      .select({\n        entityJson: 'final_entities.final_entity',\n      });\n\n    if (!rootRow) {\n      throw new NotFoundError(`No such entity ${rootRef}`);\n    }\n\n    const rootEntity = JSON.parse(rootRow.entityJson) as Entity;\n    const seenEntityRefs = new Set<string>();\n    const todo = new Array<Entity>();\n    const items = new Array<{ entity: Entity; parentEntityRefs: string[] }>();\n\n    for (\n      let current: Entity | undefined = rootEntity;\n      current;\n      current = todo.pop()\n    ) {\n      const currentRef = stringifyEntityRef(current);\n      seenEntityRefs.add(currentRef);\n\n      const parentRows = await this.database<DbRefreshStateReferencesRow>(\n        'refresh_state_references',\n      )\n        .innerJoin<DbFinalEntitiesRow>('final_entities', {\n          'refresh_state_references.source_entity_ref':\n            'final_entities.entity_ref',\n        })\n        .where('refresh_state_references.target_entity_ref', '=', currentRef)\n        .select({\n          parentEntityRef: 'final_entities.entity_ref',\n          parentEntityJson: 'final_entities.final_entity',\n        });\n\n      const parentRefs: string[] = [];\n      for (const { parentEntityRef, parentEntityJson } of parentRows) {\n        parentRefs.push(parentEntityRef);\n        if (!seenEntityRefs.has(parentEntityRef)) {\n          seenEntityRefs.add(parentEntityRef);\n          todo.push(JSON.parse(parentEntityJson));\n        }\n      }\n\n      items.push({\n        entity: current,\n        parentEntityRefs: parentRefs,\n      });\n    }\n\n    return {\n      rootEntityRef: stringifyEntityRef(rootEntity),\n      items,\n    };\n  }\n\n  async facets(request: EntityFacetsRequest): Promise<EntityFacetsResponse> {\n    const facets: EntityFacetsResponse['facets'] = {};\n    const db = this.database;\n\n    for (const facet of request.facets) {\n      const dbQuery = db<DbSearchRow>('search')\n        .where('search.key', facet.toLocaleLowerCase('en-US'))\n        .whereNotNull('search.original_value')\n        .select({ value: 'search.original_value', count: db.raw('count(*)') })\n        .groupBy('search.original_value');\n\n      if (request?.filter) {\n        parseFilter(request.filter, dbQuery, db, false, 'search.entity_id');\n      }\n\n      const result = await dbQuery;\n\n      facets[facet] = result.map(data => ({\n        value: String(data.value),\n        count: Number(data.count),\n      }));\n    }\n\n    return { facets };\n  }\n}\n\nconst entityFilterParser: z.ZodSchema<EntityFilter> = z.lazy(() =>\n  z\n    .object({\n      key: z.string(),\n      values: z.array(z.string()).optional(),\n    })\n    .or(z.object({ not: entityFilterParser }))\n    .or(z.object({ anyOf: z.array(entityFilterParser) }))\n    .or(z.object({ allOf: z.array(entityFilterParser) })),\n);\n\nexport const cursorParser: z.ZodSchema<Cursor> = z.object({\n  orderFields: z.array(\n    z.object({ field: z.string(), order: z.enum(['asc', 'desc']) }),\n  ),\n  orderFieldValues: z.array(z.string().or(z.null())),\n  filter: entityFilterParser.optional(),\n  isPrevious: z.boolean(),\n  query: z.string().optional(),\n  firstSortFieldValues: z.array(z.string().or(z.null())).optional(),\n  totalItems: z.number().optional(),\n});\n\nfunction parseCursorFromRequest(\n  request?: QueryEntitiesRequest,\n): Partial<Cursor> {\n  if (isQueryEntitiesInitialRequest(request)) {\n    const {\n      filter,\n      orderFields: sortFields = [defaultSortField],\n      fullTextFilter,\n    } = request;\n    return { filter, orderFields: sortFields, fullTextFilter };\n  }\n  if (isQueryEntitiesCursorRequest(request)) {\n    return request.cursor;\n  }\n  return {};\n}\n\nfunction invertOrder(order: EntityOrder['order']) {\n  return order === 'asc' ? 'desc' : 'asc';\n}\n\nfunction sortFieldsFromRow(row: DbSearchRow) {\n  return [row.value, row.entity_id];\n}\n"],"names":["InputError","stringifyEntityRef","parseEntityRef","lodashChunk","isQueryEntitiesInitialRequest","isEqual","NotFoundError","z","isQueryEntitiesCursorRequest"],"mappings":";;;;;;;;AA4DA,MAAM,gBAAgC,GAAA;AAAA,EACpC,KAAO,EAAA,cAAA;AAAA,EACP,KAAO,EAAA;AACT,CAAA;AAEA,MAAM,aAAgB,GAAA,EAAA;AAEtB,SAAS,gBAAgB,KAA4C,EAAA;AACnE,EAAA,IAAI,CAAC,KAAO,EAAA;AACV,IAAA,OAAO,EAAC;AAAA;AAGV,EAAI,IAAA,EAAE,KAAO,EAAA,MAAA,EAAW,GAAA,KAAA;AAExB,EAAI,IAAA,KAAA,CAAM,UAAU,KAAW,CAAA,EAAA;AAC7B,IAAO,OAAA,EAAE,OAAO,MAAO,EAAA;AAAA;AAGzB,EAAI,IAAA,MAAA;AACJ,EAAI,IAAA;AACF,IAAM,MAAA,IAAA,GAAO,OAAO,IAAK,CAAA,KAAA,CAAM,OAAO,QAAQ,CAAA,CAAE,SAAS,MAAM,CAAA;AAC/D,IAAS,MAAA,GAAA,IAAA,CAAK,MAAM,IAAI,CAAA;AAAA,GAClB,CAAA,MAAA;AACN,IAAM,MAAA,IAAIA,kBAAW,6CAA6C,CAAA;AAAA;AAGpE,EAAI,IAAA,MAAA,CAAO,UAAU,KAAW,CAAA,EAAA;AAC9B,IAAA,IAAI,CAAC,MAAA,CAAO,SAAU,CAAA,MAAA,CAAO,KAAK,CAAG,EAAA;AACnC,MAAM,MAAA,IAAIA,kBAAW,iDAAiD,CAAA;AAAA;AAExE,IAAA,KAAA,GAAQ,MAAO,CAAA,KAAA;AAAA;AAGjB,EAAI,IAAA,MAAA,CAAO,WAAW,KAAW,CAAA,EAAA;AAC/B,IAAA,IAAI,CAAC,MAAA,CAAO,SAAU,CAAA,MAAA,CAAO,MAAM,CAAG,EAAA;AACpC,MAAM,MAAA,IAAIA,kBAAW,iDAAiD,CAAA;AAAA;AAExE,IAAA,MAAA,GAAS,MAAO,CAAA,MAAA;AAAA;AAGlB,EAAO,OAAA,EAAE,OAAO,MAAO,EAAA;AACzB;AAEA,SAAS,oBACP,KACQ,EAAA;AACR,EAAM,MAAA,EAAE,KAAO,EAAA,MAAA,EAAW,GAAA,KAAA;AAC1B,EAAA,MAAM,OAAO,IAAK,CAAA,SAAA,CAAU,EAAE,KAAA,EAAO,QAAQ,CAAA;AAC7C,EAAA,MAAM,SAAS,MAAO,CAAA,IAAA,CAAK,MAAM,MAAM,CAAA,CAAE,SAAS,QAAQ,CAAA;AAC1D,EAAO,OAAA,MAAA;AACT;AAEA,SAAS,aACP,YACA,EAAA,EAAA,EACA,QACA,MAAkB,GAAA,KAAA,EAClB,gBAAgB,WACV,EAAA;AACN,EAAM,MAAA,GAAA,GAAM,MAAO,CAAA,GAAA,CAAI,WAAY,EAAA;AACnC,EAAA,MAAM,SAAS,MAAO,CAAA,MAAA,EAAQ,IAAI,CAAK,CAAA,KAAA,CAAA,CAAE,aAAa,CAAA;AAKtD,EAAA,MAAM,UAAa,GAAA,EAAA,CAAgB,QAAQ,CAAA,CACxC,OAAO,kBAAkB,CAAA,CACzB,KAAM,CAAA,EAAE,GAAI,EAAC,CACb,CAAA,QAAA,CAAS,SAAS,SAAY,GAAA;AAC7B,IAAI,IAAA,MAAA,EAAQ,WAAW,CAAG,EAAA;AACxB,MAAA,IAAA,CAAK,MAAM,EAAE,KAAA,EAAO,OAAO,EAAG,CAAA,CAAC,GAAG,CAAA;AAAA,eACzB,MAAQ,EAAA;AACjB,MAAK,IAAA,CAAA,QAAA,CAAS,OAAS,EAAA,IAAA,EAAM,MAAM,CAAA;AAAA;AACrC,GACD,CAAA;AACH,EAAA,YAAA,CAAa,QAAS,CAAA,aAAA,EAAe,MAAS,GAAA,QAAA,GAAW,MAAM,UAAU,CAAA;AAC3E;AAEA,SAAS,uBACP,MACgC,EAAA;AAChC,EAAO,OAAA,MAAA,CAAO,eAAe,KAAK,CAAA;AACpC;AAEA,SAAS,iBACP,MACqC,EAAA;AACrC,EAAO,OAAA,MAAA,CAAO,eAAe,OAAO,CAAA;AACtC;AAEA,SAAS,uBACP,MACiC,EAAA;AACjC,EAAO,OAAA,MAAA,CAAO,eAAe,KAAK,CAAA;AACpC;AAEA,SAAS,YACP,MACA,EAAA,KAAA,EACA,IACA,MAAkB,GAAA,KAAA,EAClB,gBAAgB,WACG,EAAA;AACnB,EAAI,IAAA,sBAAA,CAAuB,MAAM,CAAG,EAAA;AAClC,IAAA,OAAO,YAAY,MAAO,CAAA,GAAA,EAAK,OAAO,EAAI,EAAA,CAAC,QAAQ,aAAa,CAAA;AAAA;AAGlE,EAAI,IAAA,sBAAA,CAAuB,MAAM,CAAG,EAAA;AAClC,IAAO,OAAA,KAAA,CAAM,QAAS,CAAA,SAAS,cAAiB,GAAA;AAC9C,MAAA,YAAA,CAAa,IAAM,EAAA,EAAA,EAAI,MAAQ,EAAA,MAAA,EAAQ,aAAa,CAAA;AAAA,KACrD,CAAA;AAAA;AAGH,EAAA,OAAO,MAAM,MAAS,GAAA,aAAA,GAAgB,UAAU,CAAA,CAAE,SAAS,cAAiB,GAAA;AAC1E,IAAI,IAAA,gBAAA,CAAiB,MAAM,CAAG,EAAA;AAC5B,MAAA,KAAA,MAAW,SAAa,IAAA,MAAA,CAAO,KAAS,IAAA,EAAI,EAAA;AAC1C,QAAK,IAAA,CAAA,OAAA;AAAA,UAAQ,cACX,WAAY,CAAA,SAAA,EAAW,QAAU,EAAA,EAAA,EAAI,OAAO,aAAa;AAAA,SAC3D;AAAA;AACF,KACK,MAAA;AACL,MAAA,KAAA,MAAW,SAAa,IAAA,MAAA,CAAO,KAAS,IAAA,EAAI,EAAA;AAC1C,QAAK,IAAA,CAAA,QAAA;AAAA,UAAS,cACZ,WAAY,CAAA,SAAA,EAAW,QAAU,EAAA,EAAA,EAAI,OAAO,aAAa;AAAA,SAC3D;AAAA;AACF;AACF,GACD,CAAA;AACH;AAEO,MAAM,sBAAkD,CAAA;AAAA,EAC5C,QAAA;AAAA,EACA,MAAA;AAAA,EACA,QAAA;AAAA,EAEjB,YAAY,OAIT,EAAA;AACD,IAAA,IAAA,CAAK,WAAW,OAAQ,CAAA,QAAA;AACxB,IAAA,IAAA,CAAK,SAAS,OAAQ,CAAA,MAAA;AACtB,IAAA,IAAA,CAAK,WAAW,OAAQ,CAAA,QAAA;AAAA;AAC1B,EAEA,MAAM,SAAS,OAAsD,EAAA;AACnE,IAAA,MAAM,KAAK,IAAK,CAAA,QAAA;AAEhB,IAAA,IAAI,aACF,GAAA,EAAA,CAAuB,gBAAgB,CAAA,CAAE,OAAO,kBAAkB,CAAA;AAEpE,IAAA,OAAA,EAAS,OAAO,OAAQ,CAAA,CAAC,EAAE,KAAA,IAAS,KAAU,KAAA;AAC5C,MAAM,MAAA,KAAA,GAAQ,SAAS,KAAK,CAAA,CAAA;AAC5B,MAAA,aAAA,GAAgB,aAAc,CAAA,aAAA;AAAA,QAC5B,EAAE,CAAC,KAAK,GAAG,QAAS,EAAA;AAAA,QACpB,SAAS,OAAO,KAAO,EAAA;AACrB,UAAA,KAAA,CACG,GAAG,CAAG,EAAA,KAAK,CAAc,UAAA,CAAA,EAAA,0BAA0B,EACnD,KAAM,CAAA,CAAA,EAAG,KAAK,CAAA,IAAA,CAAA,EAAQ,GAAG,GAAI,CAAA,GAAA,EAAK,CAAC,KAAK,CAAC,CAAC,CAAA;AAAA;AAC/C,OACF;AAAA,KACD,CAAA;AAED,IAAgB,aAAA,GAAA,aAAA,CAAc,aAAa,6BAA6B,CAAA;AAExE,IAAA,IAAI,SAAS,MAAQ,EAAA;AACnB,MAAgB,aAAA,GAAA,WAAA;AAAA,QACd,OAAQ,CAAA,MAAA;AAAA,QACR,aAAA;AAAA,QACA,EAAA;AAAA,QACA,KAAA;AAAA,QACA;AAAA,OACF;AAAA;AAGF,IAAA,OAAA,EAAS,OAAO,OAAQ,CAAA,CAAC,EAAE,KAAA,IAAS,KAAU,KAAA;AAC5C,MAAA,IAAI,EAAG,CAAA,MAAA,CAAO,MAAO,CAAA,MAAA,KAAW,IAAM,EAAA;AAEpC,QAAA,aAAA,GAAgB,cAAc,OAAQ,CAAA;AAAA,UACpC,EAAE,MAAQ,EAAA,CAAA,MAAA,EAAS,KAAK,CAAU,MAAA,CAAA,EAAA,KAAA,EAAO,OAAO,MAAO;AAAA,SACxD,CAAA;AAAA,OACI,MAAA;AAIL,QAAA,aAAA,GAAgB,cAAc,OAAQ,CAAA;AAAA,UACpC,EAAE,QAAQ,CAAS,MAAA,EAAA,KAAK,UAAU,KAAO,EAAA,KAAA,CAAA,EAAW,OAAO,MAAO,EAAA;AAAA,UAClE,EAAE,MAAA,EAAQ,CAAS,MAAA,EAAA,KAAK,UAAU,KAAM;AAAA,SACzC,CAAA;AAAA;AACH,KACD,CAAA;AAED,IAAI,IAAA,CAAC,SAAS,KAAO,EAAA;AACnB,MAAgB,aAAA,GAAA,aAAA,CAAc,OAAQ,CAAA,2BAAA,EAA6B,KAAK,CAAA;AAAA,KACnE,MAAA;AACL,MAAc,aAAA,CAAA,OAAA,CAAQ,4BAA4B,KAAK,CAAA;AAAA;AAGzD,IAAA,MAAM,EAAE,KAAO,EAAA,MAAA,EAAW,GAAA,eAAA,CAAgB,SAAS,UAAU,CAAA;AAC7D,IAAA,IAAI,UAAU,KAAW,CAAA,EAAA;AACvB,MAAgB,aAAA,GAAA,aAAA,CAAc,KAAM,CAAA,KAAA,GAAQ,CAAC,CAAA;AAAA;AAE/C,IAAA,IAAI,WAAW,KAAW,CAAA,EAAA;AACxB,MAAgB,aAAA,GAAA,aAAA,CAAc,OAAO,MAAM,CAAA;AAAA;AAG7C,IAAA,IAAI,OAAO,MAAM,aAAA;AACjB,IAAI,IAAA,QAAA;AACJ,IAAA,IAAI,KAAU,KAAA,KAAA,CAAA,IAAa,IAAK,CAAA,MAAA,IAAU,KAAO,EAAA;AAC/C,MAAW,QAAA,GAAA,EAAE,aAAa,KAAM,EAAA;AAAA,KAC3B,MAAA;AACL,MAAO,IAAA,GAAA,IAAA,CAAK,KAAM,CAAA,CAAA,EAAG,CAAE,CAAA,CAAA;AACvB,MAAW,QAAA,GAAA;AAAA,QACT,WAAa,EAAA,IAAA;AAAA,QACb,WAAW,mBAAoB,CAAA;AAAA,UAC7B,KAAA;AAAA,UACA,MAAA,EAAA,CAAS,UAAU,CAAK,IAAA;AAAA,SACzB;AAAA,OACH;AAAA;AAGF,IAAI,IAAA,QAAA,GAAqB,KAAK,GAAI,CAAA,CAAA,CAAA,KAAK,KAAK,KAAM,CAAA,CAAA,CAAE,YAAa,CAAC,CAAA;AAElE,IAAA,IAAI,SAAS,MAAQ,EAAA;AACnB,MAAA,QAAA,GAAW,SAAS,GAAI,CAAA,CAAA,CAAA,KAAK,OAAQ,CAAA,MAAA,CAAQ,CAAC,CAAC,CAAA;AAAA;AAQjD,IAAA,KAAA,MAAW,UAAU,QAAU,EAAA;AAC7B,MAAA,IAAI,OAAO,SAAW,EAAA;AACpB,QAAW,KAAA,MAAA,QAAA,IAAY,OAAO,SAAkB,EAAA;AAC9C,UAAA,IAAI,CAAC,QAAA,CAAS,SAAa,IAAA,QAAA,CAAS,MAAQ,EAAA;AAG1C,YAAS,QAAA,CAAA,SAAA,GAAYC,+BAAmB,CAAA,QAAA,CAAS,MAAM,CAAA;AAAA,WAC9C,MAAA,IAAA,CAAC,QAAS,CAAA,MAAA,IAAU,SAAS,SAAW,EAAA;AAIjD,YAAS,QAAA,CAAA,MAAA,GAASC,2BAAe,CAAA,QAAA,CAAS,SAAS,CAAA;AAAA;AACrD;AACF;AACF;AAGF,IAAO,OAAA;AAAA,MACL,QAAA;AAAA,MACA;AAAA,KACF;AAAA;AACF,EAEA,MAAM,cACJ,OACgC,EAAA;AAChC,IAAM,MAAA,MAAA,uBAAa,GAAoB,EAAA;AAEvC,IAAA,KAAA,MAAW,KAAS,IAAAC,YAAA,CAAY,OAAQ,CAAA,UAAA,EAAY,GAAG,CAAG,EAAA;AACxD,MAAA,IAAI,KAAQ,GAAA,IAAA,CAAK,QAA6B,CAAA,gBAAgB,EAC3D,MAAO,CAAA;AAAA,QACN,SAAW,EAAA,2BAAA;AAAA,QACX,MAAQ,EAAA;AAAA,OACT,CAAA,CACA,OAAQ,CAAA,2BAAA,EAA6B,KAAK,CAAA;AAE7C,MAAA,IAAI,SAAS,MAAQ,EAAA;AACnB,QAAQ,KAAA,GAAA,WAAA;AAAA,UACN,OAAQ,CAAA,MAAA;AAAA,UACR,KAAA;AAAA,UACA,IAAK,CAAA,QAAA;AAAA,UACL,KAAA;AAAA,UACA;AAAA,SACF;AAAA;AAGF,MAAW,KAAA,MAAA,GAAA,IAAO,MAAM,KAAO,EAAA;AAC7B,QAAO,MAAA,CAAA,GAAA,CAAI,GAAI,CAAA,SAAA,EAAW,GAAI,CAAA,MAAA,GAAS,KAAK,KAAM,CAAA,GAAA,CAAI,MAAM,CAAA,GAAI,IAAI,CAAA;AAAA;AACtE;AAGF,IAAI,IAAA,KAAA,GAAQ,QAAQ,UAAW,CAAA,GAAA,CAAI,SAAO,MAAO,CAAA,GAAA,CAAI,GAAG,CAAA,IAAK,IAAI,CAAA;AAEjE,IAAA,IAAI,QAAQ,MAAQ,EAAA;AAClB,MAAA,KAAA,GAAQ,MAAM,GAAI,CAAA,CAAA,CAAA,KAAK,KAAK,OAAQ,CAAA,MAAA,CAAQ,CAAC,CAAC,CAAA;AAAA;AAGhD,IAAA,OAAO,EAAE,KAAM,EAAA;AAAA;AACjB,EAEA,MAAM,cACJ,OACgC,EAAA;AAChC,IAAA,MAAM,KAAK,IAAK,CAAA,QAAA;AAEhB,IAAM,MAAA,KAAA,GAAQ,QAAQ,KAAS,IAAA,aAAA;AAE/B,IAAA,MAAM,MAEF,GAAA;AAAA,MACF,WAAA,EAAa,CAAC,gBAAgB,CAAA;AAAA,MAC9B,UAAY,EAAA,KAAA;AAAA,MACZ,GAAG,uBAAuB,OAAO;AAAA,KACnC;AAEA,IAAA,MAAM,sBAAsB,MAAO,CAAA,UAAA;AAEnC,IAAI,IAAA,MAAA,CAAO,WAAY,CAAA,MAAA,GAAS,CAAG,EAAA;AACjC,MAAK,IAAA,CAAA,MAAA,CAAO,KAAK,CAAqD,mDAAA,CAAA,CAAA;AAAA;AAGxE,IAAA,MAAM,SAAyB,GAAA;AAAA,MAC7B,GAAG,gBAAA;AAAA,MACH,GAAG,MAAO,CAAA,WAAA,CAAY,CAAC;AAAA,KACzB;AAEA,IAAA,MAAM,CAAC,uBAAyB,EAAA,WAAW,CACzC,GAAA,MAAA,CAAO,oBAAoB,EAAC;AAE9B,IAAM,MAAA,OAAA,GAAU,EAAG,CAAA,gBAAgB,CAAE,CAAA,aAAA;AAAA,MAAc,QAAA;AAAA,MAAU,CAAA,EAAA,KAC3D,GACG,EAAG,CAAA,kBAAA,EAAoB,0BAA0B,CACjD,CAAA,QAAA,CAAS,YAAc,EAAA,SAAA,CAAU,KAAK;AAAA,KAC3C;AAEA,IAAA,IAAI,OAAO,MAAQ,EAAA;AACjB,MAAA,WAAA;AAAA,QACE,MAAO,CAAA,MAAA;AAAA,QACP,OAAA;AAAA,QACA,EAAA;AAAA,QACA,KAAA;AAAA,QACA;AAAA,OACF;AAAA;AAGF,IAAA,MAAM,4BAA+B,GAAA,MAAA,CAAO,cAAgB,EAAA,IAAA,EAAM,IAAK,EAAA;AACvE,IAAA,MAAM,mBAAmB,MAAO,CAAA,cAAA,EAAgB,MAAU,IAAA,CAAC,UAAU,KAAK,CAAA;AAC1E,IAAA,IAAI,4BAA8B,EAAA;AAChC,MAAA,IACE,iBAAiB,MAAW,KAAA,CAAA,IAC5B,iBAAiB,CAAC,CAAA,KAAM,UAAU,KAClC,EAAA;AAGA,QAAQ,OAAA,CAAA,WAAA;AAAA,UACN,cAAA;AAAA,UACA,CAAI,CAAA,EAAA,4BAAA,CAA6B,iBAAkB,CAAA,OAAO,CAAC,CAAA,CAAA;AAAA,SAC7D;AAAA,OACK,MAAA;AACL,QAAA,MAAM,aAAa,EAAgB,CAAA,QAAQ,CACxC,CAAA,MAAA,CAAO,kBAAkB,CAEzB,CAAA,OAAA;AAAA,UACC,KAAA;AAAA,UACA,iBAAiB,GAAI,CAAA,CAAA,KAAA,KAAS,KAAM,CAAA,iBAAA,CAAkB,OAAO,CAAC;AAAA,SAChE,CACC,QAAS,CAAA,SAAS,SAAY,GAAA;AAC7B,UAAK,IAAA,CAAA,WAAA;AAAA,YACH,cAAA;AAAA,YACA,CAAI,CAAA,EAAA,4BAAA,CAA6B,iBAAkB,CAAA,OAAO,CAAC,CAAA,CAAA;AAAA,WAC7D;AAAA,SACD,CAAA;AACH,QAAQ,OAAA,CAAA,QAAA,CAAS,0BAA4B,EAAA,IAAA,EAAM,UAAU,CAAA;AAAA;AAC/D;AAGF,IAAM,MAAA,UAAA,GAAa,QAAQ,KAAM,EAAA;AAEjC,IAAM,MAAA,oBAAA,GAAuB,UAAU,KAAU,KAAA,MAAA;AAEjD,IAAA,IAAI,uBAAyB,EAAA;AAC3B,MAAQ,OAAA,CAAA,QAAA,CAAS,SAAS,MAAS,GAAA;AACjC,QAAK,IAAA,CAAA,KAAA;AAAA,UACH,OAAA;AAAA,UACA,mBAAA,KAAwB,uBAAuB,GAAM,GAAA,GAAA;AAAA,UACrD;AAAA,SAEC,CAAA,OAAA,CAAQ,OAAS,EAAA,GAAA,EAAK,uBAAuB,CAC7C,CAAA,QAAA;AAAA,UACC,0BAAA;AAAA,UACA,mBAAA,KAAwB,uBAAuB,GAAM,GAAA,GAAA;AAAA,UACrD;AAAA,SACF;AAAA,OACH,CAAA;AAAA;AAGH,IAAA,IAAI,EAAG,CAAA,MAAA,CAAO,MAAO,CAAA,MAAA,KAAW,IAAM,EAAA;AAEpC,MAAA,OAAA,CAAQ,OAAQ,CAAA;AAAA,QACd;AAAA,UACE,MAAQ,EAAA,cAAA;AAAA,UACR,OAAO,mBACH,GAAA,WAAA,CAAY,SAAU,CAAA,KAAK,IAC3B,SAAU,CAAA,KAAA;AAAA,UACd,KAAO,EAAA;AAAA,SACT;AAAA,QACA;AAAA,UACE,MAAQ,EAAA,0BAAA;AAAA,UACR,OAAO,mBACH,GAAA,WAAA,CAAY,SAAU,CAAA,KAAK,IAC3B,SAAU,CAAA;AAAA;AAChB,OACD,CAAA;AAAA,KACI,MAAA;AAIL,MAAA,OAAA,CAAQ,OAAQ,CAAA;AAAA,QACd;AAAA,UACE,MAAQ,EAAA,cAAA;AAAA,UACR,KAAO,EAAA,KAAA,CAAA;AAAA,UACP,KAAO,EAAA;AAAA,SACT;AAAA,QACA;AAAA,UACE,MAAQ,EAAA,cAAA;AAAA,UACR,OAAO,mBACH,GAAA,WAAA,CAAY,SAAU,CAAA,KAAK,IAC3B,SAAU,CAAA;AAAA,SAChB;AAAA,QACA;AAAA,UACE,MAAQ,EAAA,0BAAA;AAAA,UACR,OAAO,mBACH,GAAA,WAAA,CAAY,SAAU,CAAA,KAAK,IAC3B,SAAU,CAAA;AAAA;AAChB,OACD,CAAA;AAAA;AAGH,IAAA,IACEC,kCAA8B,CAAA,OAAO,CACrC,IAAA,OAAA,CAAQ,WAAW,KACnB,CAAA,EAAA;AACA,MAAQ,OAAA,CAAA,MAAA,CAAO,QAAQ,MAAM,CAAA;AAAA;AAG/B,IAAA,OAAA,CAAQ,KAAM,CAAA,mBAAA,GAAsB,KAAQ,GAAA,KAAA,GAAQ,CAAC,CAAA;AAErD,IAAA,UAAA,CAAW,KAAM,CAAA,0BAAA,EAA4B,EAAE,EAAA,EAAI,SAAS,CAAA;AAE5D,IAAM,MAAA,CAAC,IAAM,EAAA,CAAC,EAAE,KAAA,EAAO,CAAC,CAAA,GAAI,MAAM,OAAA,CAAQ,GAAI,CAAA;AAAA,MAC5C,KAAA,GAAQ,CAAI,GAAA,OAAA,GAAU,EAAC;AAAA;AAAA;AAAA;AAAA;AAAA,MAKvB,OAAO,MAAO,CAAA,UAAA,KAAe,WACzB,GAAA,UAAA,GACA,CAAC,EAAE,KAAA,EAAO,MAAO,CAAA,UAAA,EAAY;AAAA,KAClC,CAAA;AAED,IAAM,MAAA,UAAA,GAAa,OAAO,KAAK,CAAA;AAE/B,IAAA,IAAI,mBAAqB,EAAA;AACvB,MAAA,IAAA,CAAK,OAAQ,EAAA;AAAA;AAEf,IAAA,MAAM,cACJ,GAAA,KAAA,GAAQ,CAAM,KAAA,mBAAA,IAAuB,KAAK,MAAS,GAAA,KAAA,CAAA;AAGrD,IAAI,IAAA,IAAA,CAAK,SAAS,KAAO,EAAA;AACvB,MAAA,IAAA,CAAK,MAAU,IAAA,CAAA;AAAA;AAGjB,IAAM,MAAA,gBAAA,GAAmB,OAAO,oBAAyB,KAAA,KAAA,CAAA;AAEzD,IAAM,MAAA,QAAA,GAAW,KAAK,CAAC,CAAA;AACvB,IAAA,MAAM,OAAU,GAAA,IAAA,CAAK,IAAK,CAAA,MAAA,GAAS,CAAC,CAAA;AAEpC,IAAM,MAAA,oBAAA,GAAuB,OAAO,oBAAwB,IAAA;AAAA,MAC1D,QAAU,EAAA,KAAA;AAAA,MACV,QAAU,EAAA;AAAA,KACZ;AAEA,IAAA,MAAM,aAAiC,cACnC,GAAA;AAAA,MACE,GAAG,MAAA;AAAA,MACH,gBAAA,EAAkB,kBAAkB,OAAO,CAAA;AAAA,MAC3C,oBAAA;AAAA,MACA,UAAY,EAAA,KAAA;AAAA,MACZ;AAAA,KAEF,GAAA,KAAA,CAAA;AAEJ,IAAA,MAAM,UACJ,GAAA,CAAC,gBACD,IAAA,IAAA,CAAK,MAAS,GAAA,CAAA,IACd,CAACC,cAAA,CAAQ,iBAAkB,CAAA,QAAQ,CAAG,EAAA,MAAA,CAAO,oBAAoB,CAC7D,GAAA;AAAA,MACE,GAAG,MAAA;AAAA,MACH,gBAAA,EAAkB,kBAAkB,QAAQ,CAAA;AAAA,MAC5C,sBAAsB,MAAO,CAAA,oBAAA;AAAA,MAC7B,UAAY,EAAA,IAAA;AAAA,MACZ;AAAA,KAEF,GAAA,KAAA,CAAA;AAEN,IAAA,MAAM,QAAQ,IACX,CAAA,GAAA,CAAI,OAAK,IAAK,CAAA,KAAA,CAAM,EAAE,YAAa,CAAC,CACpC,CAAA,GAAA,CAAI,OAAM,OAAQ,CAAA,MAAA,GAAS,QAAQ,MAAO,CAAA,CAAC,IAAI,CAAE,CAAA;AAEpD,IAAO,OAAA;AAAA,MACL,KAAA;AAAA,MACA,QAAU,EAAA;AAAA,QACR,GAAI,CAAC,CAAC,UAAA,IAAc,EAAE,UAAW,EAAA;AAAA,QACjC,GAAI,CAAC,CAAC,UAAA,IAAc,EAAE,UAAW;AAAA,OACnC;AAAA,MACA;AAAA,KACF;AAAA;AACF,EAEA,MAAM,kBAAkB,GAA4B,EAAA;AAClD,IAAM,MAAA,QAAA,GAAW,IAAK,CAAA,QAAA,CAAS,MAAO,CAAA,MAAA;AAUtC,IAAA,IAAI,QAAS,CAAA,MAAA,CAAO,QAAS,CAAA,OAAO,CAAG,EAAA;AAGrC,MAAA,MAAM,OAAU,GAAA,MAAM,IAAK,CAAA,QAAA,CAA4B,eAAe,CAAA,CACnE,MAAO,CAAA,WAAW,CAClB,CAAA,OAAA,CAAQ,YAAc,EAAA,SAAS,QAAQ,OAAS,EAAA;AAC/C,QAAO,OAAA,OAAA,CACJ,IAAwB,CAAA,eAAe,CACvC,CAAA,SAAA;AAAA,UACC,0BAAA;AAAA,UACA;AAAA,YACE,4CACE,EAAA;AAAA;AACJ,UAED,KAAM,CAAA,yBAAA,EAA2B,KAAK,GAAG,CAAA,CACzC,OAAO,4CAA4C,CAAA;AAAA,OACvD,CAAA;AACH,MAAA,MAAM,IAAK,CAAA,QAAA,CAA4B,eAAe,CAAA,CACnD,MAAO,CAAA;AAAA,QACN,WAAa,EAAA,mBAAA;AAAA,QACb,cAAgB,EAAA,IAAA,CAAK,QAAS,CAAA,EAAA,CAAG,GAAI;AAAA,OACtC,CACA,CAAA,OAAA;AAAA,QACC,WAAA;AAAA,QACA,OAAQ,CAAA,GAAA,CAAI,CAAO,GAAA,KAAA,GAAA,CAAI,SAAS;AAAA,OAClC;AAAA,KACG,MAAA;AACL,MAAA,MAAM,IAAK,CAAA,QAAA,CAA4B,eAAe,CAAA,CACnD,MAAO,CAAA;AAAA,QACN,WAAa,EAAA,mBAAA;AAAA,QACb,cAAgB,EAAA,IAAA,CAAK,QAAS,CAAA,EAAA,CAAG,GAAI;AAAA,OACtC,CACA,CAAA,OAAA,CAAQ,YAAc,EAAA,SAAS,QAAQ,OAAS,EAAA;AAC/C,QAAO,OAAA,OAAA,CACJ,IAAwB,CAAA,eAAe,CACvC,CAAA,SAAA;AAAA,UACC,0BAAA;AAAA,UACA;AAAA,YACE,4CACE,EAAA;AAAA;AACJ,UAED,KAAM,CAAA,yBAAA,EAA2B,KAAK,GAAG,CAAA,CACzC,OAAO,4CAA4C,CAAA;AAAA,OACvD,CAAA;AAAA;AAOL,IAAM,MAAA,aAAA,GAAgB,MAAM,IAAK,CAAA,QAAA,CAC9B,KAAqB,WAAW,CAAA,CAChC,UAAuC,eAAiB,EAAA;AAAA,MACvD,0BAA4B,EAAA;AAAA,KAC7B,CACA,CAAA,KAAA,CAAM,iCAAmC,EAAA,GAAA,EAAK,GAAG,CACjD,CAAA,QAAA,CAAS,yBAA2B,EAAA,IAAA,EAAM,GAAG,CAC7C,CAAA,MAAA,CAAO,EAAE,GAAK,EAAA,6BAAA,EAA+B,CAC7C,CAAA,KAAA;AAAA,MAAM,WACL,KACG,CAAA,IAAA,CAAqB,WAAW,CAAA,CAChC,UAAuC,eAAiB,EAAA;AAAA,QACvD,0BAA4B,EAAA;AAAA,OAC7B,CACA,CAAA,KAAA,CAAM,iCAAmC,EAAA,GAAA,EAAK,GAAG,CACjD,CAAA,QAAA,CAAS,yBAA2B,EAAA,IAAA,EAAM,GAAG,CAC7C,CAAA,MAAA,CAAO,EAAE,GAAA,EAAK,+BAA+B;AAAA,KAClD;AAEF,IAAM,MAAA,IAAA,CAAK,SAA4B,eAAe,CAAA,CACnD,MAAM,WAAa,EAAA,GAAG,EACtB,MAAO,EAAA;AAEV,IAAM,MAAA,IAAA,CAAK,SAAS,MAAO,CAAA;AAAA,MACzB,UAAA,EAAY,IAAI,GAAI,CAAA,aAAA,CAAc,IAAI,CAAK,CAAA,KAAA,CAAA,CAAE,GAAG,CAAC;AAAA,KAClD,CAAA;AAAA;AACH,EAEA,MAAM,eAAe,OAAkD,EAAA;AACrE,IAAA,MAAM,CAAC,OAAO,CAAI,GAAA,MAAM,IAAK,CAAA,QAAA,CAA6B,gBAAgB,CAAA,CACvE,KAAM,CAAA,2BAAA,EAA6B,GAAK,EAAA,OAAO,EAC/C,MAAO,CAAA;AAAA,MACN,UAAY,EAAA;AAAA,KACb,CAAA;AAEH,IAAA,IAAI,CAAC,OAAS,EAAA;AACZ,MAAA,MAAM,IAAIC,oBAAA,CAAc,CAAkB,eAAA,EAAA,OAAO,CAAE,CAAA,CAAA;AAAA;AAGrD,IAAA,MAAM,UAAa,GAAA,IAAA,CAAK,KAAM,CAAA,OAAA,CAAQ,UAAU,CAAA;AAChD,IAAM,MAAA,cAAA,uBAAqB,GAAY,EAAA;AACvC,IAAM,MAAA,IAAA,GAAO,IAAI,KAAc,EAAA;AAC/B,IAAM,MAAA,KAAA,GAAQ,IAAI,KAAsD,EAAA;AAExE,IAAA,KAAA,IACM,UAA8B,UAClC,EAAA,OAAA,EACA,OAAU,GAAA,IAAA,CAAK,KACf,EAAA;AACA,MAAM,MAAA,UAAA,GAAaL,gCAAmB,OAAO,CAAA;AAC7C,MAAA,cAAA,CAAe,IAAI,UAAU,CAAA;AAE7B,MAAM,MAAA,UAAA,GAAa,MAAM,IAAK,CAAA,QAAA;AAAA,QAC5B;AAAA,OACF,CACG,UAA8B,gBAAkB,EAAA;AAAA,QAC/C,4CACE,EAAA;AAAA,OACH,CACA,CAAA,KAAA,CAAM,8CAA8C,GAAK,EAAA,UAAU,EACnE,MAAO,CAAA;AAAA,QACN,eAAiB,EAAA,2BAAA;AAAA,QACjB,gBAAkB,EAAA;AAAA,OACnB,CAAA;AAEH,MAAA,MAAM,aAAuB,EAAC;AAC9B,MAAA,KAAA,MAAW,EAAE,eAAA,EAAiB,gBAAiB,EAAA,IAAK,UAAY,EAAA;AAC9D,QAAA,UAAA,CAAW,KAAK,eAAe,CAAA;AAC/B,QAAA,IAAI,CAAC,cAAA,CAAe,GAAI,CAAA,eAAe,CAAG,EAAA;AACxC,UAAA,cAAA,CAAe,IAAI,eAAe,CAAA;AAClC,UAAA,IAAA,CAAK,IAAK,CAAA,IAAA,CAAK,KAAM,CAAA,gBAAgB,CAAC,CAAA;AAAA;AACxC;AAGF,MAAA,KAAA,CAAM,IAAK,CAAA;AAAA,QACT,MAAQ,EAAA,OAAA;AAAA,QACR,gBAAkB,EAAA;AAAA,OACnB,CAAA;AAAA;AAGH,IAAO,OAAA;AAAA,MACL,aAAA,EAAeA,gCAAmB,UAAU,CAAA;AAAA,MAC5C;AAAA,KACF;AAAA;AACF,EAEA,MAAM,OAAO,OAA6D,EAAA;AACxE,IAAA,MAAM,SAAyC,EAAC;AAChD,IAAA,MAAM,KAAK,IAAK,CAAA,QAAA;AAEhB,IAAW,KAAA,MAAA,KAAA,IAAS,QAAQ,MAAQ,EAAA;AAClC,MAAM,MAAA,OAAA,GAAU,EAAgB,CAAA,QAAQ,CACrC,CAAA,KAAA,CAAM,YAAc,EAAA,KAAA,CAAM,iBAAkB,CAAA,OAAO,CAAC,CAAA,CACpD,YAAa,CAAA,uBAAuB,EACpC,MAAO,CAAA,EAAE,KAAO,EAAA,uBAAA,EAAyB,KAAO,EAAA,EAAA,CAAG,GAAI,CAAA,UAAU,CAAE,EAAC,CACpE,CAAA,OAAA,CAAQ,uBAAuB,CAAA;AAElC,MAAA,IAAI,SAAS,MAAQ,EAAA;AACnB,QAAA,WAAA,CAAY,OAAQ,CAAA,MAAA,EAAQ,OAAS,EAAA,EAAA,EAAI,OAAO,kBAAkB,CAAA;AAAA;AAGpE,MAAA,MAAM,SAAS,MAAM,OAAA;AAErB,MAAA,MAAA,CAAO,KAAK,CAAA,GAAI,MAAO,CAAA,GAAA,CAAI,CAAS,IAAA,MAAA;AAAA,QAClC,KAAA,EAAO,MAAO,CAAA,IAAA,CAAK,KAAK,CAAA;AAAA,QACxB,KAAA,EAAO,MAAO,CAAA,IAAA,CAAK,KAAK;AAAA,OACxB,CAAA,CAAA;AAAA;AAGJ,IAAA,OAAO,EAAE,MAAO,EAAA;AAAA;AAEpB;AAEA,MAAM,qBAAgDM,KAAE,CAAA,IAAA;AAAA,EAAK,MAC3DA,MACG,MAAO,CAAA;AAAA,IACN,GAAA,EAAKA,MAAE,MAAO,EAAA;AAAA,IACd,QAAQA,KAAE,CAAA,KAAA,CAAMA,MAAE,MAAO,EAAC,EAAE,QAAS;AAAA,GACtC,CAAA,CACA,EAAG,CAAAA,KAAA,CAAE,OAAO,EAAE,GAAA,EAAK,kBAAmB,EAAC,CAAC,CAAA,CACxC,EAAG,CAAAA,KAAA,CAAE,OAAO,EAAE,KAAA,EAAOA,KAAE,CAAA,KAAA,CAAM,kBAAkB,CAAA,EAAG,CAAC,EACnD,EAAG,CAAAA,KAAA,CAAE,MAAO,CAAA,EAAE,OAAOA,KAAE,CAAA,KAAA,CAAM,kBAAkB,CAAA,EAAG,CAAC;AACxD,CAAA;AAEiDA,MAAE,MAAO,CAAA;AAAA,EACxD,aAAaA,KAAE,CAAA,KAAA;AAAA,IACbA,KAAE,CAAA,MAAA,CAAO,EAAE,KAAA,EAAOA,MAAE,MAAO,EAAA,EAAG,KAAO,EAAAA,KAAA,CAAE,KAAK,CAAC,KAAA,EAAO,MAAM,CAAC,GAAG;AAAA,GAChE;AAAA,EACA,gBAAA,EAAkBA,KAAE,CAAA,KAAA,CAAMA,KAAE,CAAA,MAAA,GAAS,EAAG,CAAAA,KAAA,CAAE,IAAK,EAAC,CAAC,CAAA;AAAA,EACjD,MAAA,EAAQ,mBAAmB,QAAS,EAAA;AAAA,EACpC,UAAA,EAAYA,MAAE,OAAQ,EAAA;AAAA,EACtB,KAAO,EAAAA,KAAA,CAAE,MAAO,EAAA,CAAE,QAAS,EAAA;AAAA,EAC3B,oBAAsB,EAAAA,KAAA,CAAE,KAAM,CAAAA,KAAA,CAAE,MAAO,EAAA,CAAE,EAAG,CAAAA,KAAA,CAAE,IAAK,EAAC,CAAC,CAAA,CAAE,QAAS,EAAA;AAAA,EAChE,UAAY,EAAAA,KAAA,CAAE,MAAO,EAAA,CAAE,QAAS;AAClC,CAAC;AAED,SAAS,uBACP,OACiB,EAAA;AACjB,EAAI,IAAAH,kCAAA,CAA8B,OAAO,CAAG,EAAA;AAC1C,IAAM,MAAA;AAAA,MACJ,MAAA;AAAA,MACA,WAAA,EAAa,UAAa,GAAA,CAAC,gBAAgB,CAAA;AAAA,MAC3C;AAAA,KACE,GAAA,OAAA;AACJ,IAAA,OAAO,EAAE,MAAA,EAAQ,WAAa,EAAA,UAAA,EAAY,cAAe,EAAA;AAAA;AAE3D,EAAI,IAAAI,iCAAA,CAA6B,OAAO,CAAG,EAAA;AACzC,IAAA,OAAO,OAAQ,CAAA,MAAA;AAAA;AAEjB,EAAA,OAAO,EAAC;AACV;AAEA,SAAS,YAAY,KAA6B,EAAA;AAChD,EAAO,OAAA,KAAA,KAAU,QAAQ,MAAS,GAAA,KAAA;AACpC;AAEA,SAAS,kBAAkB,GAAkB,EAAA;AAC3C,EAAA,OAAO,CAAC,GAAA,CAAI,KAAO,EAAA,GAAA,CAAI,SAAS,CAAA;AAClC;;;;"}

package/dist/stitching/types.cjs.js

@@ -13,12 +13,8 @@ function stitchingStrategyFromConfig(config$1) {
   } else if (strategyMode === "deferred") {
     const pollingIntervalKey = "catalog.stitchingStrategy.pollingInterval";
     const stitchTimeoutKey = "catalog.stitchingStrategy.stitchTimeout";
-    const pollingInterval = config$1.has(pollingIntervalKey) ? config.readDurationFromConfig(config$1, {
-      key: pollingIntervalKey
-    }) : { seconds: 1 };
-    const stitchTimeout = config$1.has(stitchTimeoutKey) ? config.readDurationFromConfig(config$1, {
-      key: stitchTimeoutKey
-    }) : { seconds: 60 };
+    const pollingInterval = config$1.has(pollingIntervalKey) ? config.readDurationFromConfig(config$1, { key: pollingIntervalKey }) : { seconds: 1 };
+    const stitchTimeout = config$1.has(stitchTimeoutKey) ? config.readDurationFromConfig(config$1, { key: stitchTimeoutKey }) : { seconds: 60 };
     return {
       mode: "deferred",
       pollingInterval,

package/dist/stitching/types.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.cjs.js","sources":["../../src/stitching/types.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Config, readDurationFromConfig } from '@backstage/config';\nimport { HumanDuration } from '@backstage/types';\n\n/**\n * Performs the act of stitching - to take all of the various outputs from the\n * ingestion process, and stitching them together into the final entity JSON\n * shape.\n */\nexport interface Stitcher {\n  stitch(options: {\n    entityRefs?: Iterable<string>;\n    entityIds?: Iterable<string>;\n  }): Promise<void>;\n}\n\n/**\n * The strategies supported by the stitching process, in terms of when to\n * perform stitching.\n *\n * @remarks\n *\n * In immediate mode, stitching happens \"in-band\" (blocking) immediately when\n * each processing task finishes. When set to `'deferred'`, stitching is instead\n * deferred to happen on a separate asynchronous worker queue just like\n * processing.\n *\n * Deferred stitching should make performance smoother when ingesting large\n * amounts of entities, and reduce p99 processing times and repeated\n * over-stitching of hot spot entities when fan-out/fan-in in terms of relations\n * is very large. It does however also come with some performance cost due to\n * the queuing with how much wall-clock time some types of task take.\n */\nexport type StitchingStrategy =\n  | {\n      mode: 'immediate';\n    }\n  | {\n      mode: 'deferred';\n      pollingInterval: HumanDuration;\n      stitchTimeout: HumanDuration;\n    };\n\nexport function stitchingStrategyFromConfig(config: Config): StitchingStrategy {\n  const strategyMode = config.getOptionalString(\n    'catalog.stitchingStrategy.mode',\n  );\n\n  if (strategyMode === undefined || strategyMode === 'immediate') {\n    return {\n      mode: 'immediate',\n    };\n  } else if (strategyMode === 'deferred') {\n    const pollingIntervalKey = 'catalog.stitchingStrategy.pollingInterval';\n    const stitchTimeoutKey = 'catalog.stitchingStrategy.stitchTimeout';\n\n    const pollingInterval = config.has(pollingIntervalKey)\n      ? readDurationFromConfig(config, {\n          key: pollingIntervalKey,\n        })\n      : { seconds: 1 };\n    const stitchTimeout = config.has(stitchTimeoutKey)\n      ? readDurationFromConfig(config, {\n          key: stitchTimeoutKey,\n        })\n      : { seconds: 60 };\n    return {\n      mode: 'deferred',\n      pollingInterval: pollingInterval,\n      stitchTimeout: stitchTimeout,\n    };\n  }\n\n  throw new Error(\n    `Invalid stitching strategy mode '${strategyMode}', expected one of 'immediate' or 'deferred'`,\n  );\n}\n"],"names":["config","readDurationFromConfig"],"mappings":";;;;AA0DO,SAAS,4BAA4BA,QAAmC,EAAA;AAC7E,EAAA,MAAM,eAAeA,QAAO,CAAA,iBAAA;AAAA,IAC1B;AAAA,GACF;AAEA,EAAI,IAAA,YAAA,KAAiB,KAAa,CAAA,IAAA,YAAA,KAAiB,WAAa,EAAA;AAC9D,IAAO,OAAA;AAAA,MACL,IAAM,EAAA;AAAA,KACR;AAAA,GACF,MAAA,IAAW,iBAAiB,UAAY,EAAA;AACtC,IAAA,MAAM,kBAAqB,GAAA,2CAAA;AAC3B,IAAA,MAAM,gBAAmB,GAAA,yCAAA;AAEzB,IAAA,MAAM,kBAAkBA,QAAO,CAAA,GAAA,CAAI,kBAAkB,CAAA,GACjDC,8BAAuBD,QAAQ,EAAA;AAAA,MAC7B,GAAK,EAAA;AAAA,KACN,CAAA,GACD,EAAE,OAAA,EAAS,CAAE,EAAA;AACjB,IAAA,MAAM,gBAAgBA,QAAO,CAAA,GAAA,CAAI,gBAAgB,CAAA,GAC7CC,8BAAuBD,QAAQ,EAAA;AAAA,MAC7B,GAAK,EAAA;AAAA,KACN,CAAA,GACD,EAAE,OAAA,EAAS,EAAG,EAAA;AAClB,IAAO,OAAA;AAAA,MACL,IAAM,EAAA,UAAA;AAAA,MACN,eAAA;AAAA,MACA;AAAA,KACF;AAAA;AAGF,EAAA,MAAM,IAAI,KAAA;AAAA,IACR,oCAAoC,YAAY,CAAA,4CAAA;AAAA,GAClD;AACF;;;;"}
+{"version":3,"file":"types.cjs.js","sources":["../../src/stitching/types.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Config, readDurationFromConfig } from '@backstage/config';\nimport { HumanDuration } from '@backstage/types';\n\n/**\n * Performs the act of stitching - to take all of the various outputs from the\n * ingestion process, and stitching them together into the final entity JSON\n * shape.\n */\nexport interface Stitcher {\n  stitch(options: {\n    entityRefs?: Iterable<string>;\n    entityIds?: Iterable<string>;\n  }): Promise<void>;\n}\n\n/**\n * The strategies supported by the stitching process, in terms of when to\n * perform stitching.\n *\n * @remarks\n *\n * In immediate mode, stitching happens \"in-band\" (blocking) immediately when\n * each processing task finishes. When set to `'deferred'`, stitching is instead\n * deferred to happen on a separate asynchronous worker queue just like\n * processing.\n *\n * Deferred stitching should make performance smoother when ingesting large\n * amounts of entities, and reduce p99 processing times and repeated\n * over-stitching of hot spot entities when fan-out/fan-in in terms of relations\n * is very large. It does however also come with some performance cost due to\n * the queuing with how much wall-clock time some types of task take.\n */\nexport type StitchingStrategy =\n  | {\n      mode: 'immediate';\n    }\n  | {\n      mode: 'deferred';\n      pollingInterval: HumanDuration;\n      stitchTimeout: HumanDuration;\n    };\n\nexport function stitchingStrategyFromConfig(config: Config): StitchingStrategy {\n  const strategyMode = config.getOptionalString(\n    'catalog.stitchingStrategy.mode',\n  );\n\n  if (strategyMode === undefined || strategyMode === 'immediate') {\n    return {\n      mode: 'immediate',\n    };\n  } else if (strategyMode === 'deferred') {\n    const pollingIntervalKey = 'catalog.stitchingStrategy.pollingInterval';\n    const stitchTimeoutKey = 'catalog.stitchingStrategy.stitchTimeout';\n\n    const pollingInterval = config.has(pollingIntervalKey)\n      ? readDurationFromConfig(config, { key: pollingIntervalKey })\n      : { seconds: 1 };\n    const stitchTimeout = config.has(stitchTimeoutKey)\n      ? readDurationFromConfig(config, { key: stitchTimeoutKey })\n      : { seconds: 60 };\n\n    return {\n      mode: 'deferred',\n      pollingInterval: pollingInterval,\n      stitchTimeout: stitchTimeout,\n    };\n  }\n\n  throw new Error(\n    `Invalid stitching strategy mode '${strategyMode}', expected one of 'immediate' or 'deferred'`,\n  );\n}\n"],"names":["config","readDurationFromConfig"],"mappings":";;;;AA0DO,SAAS,4BAA4BA,QAAmC,EAAA;AAC7E,EAAA,MAAM,eAAeA,QAAO,CAAA,iBAAA;AAAA,IAC1B;AAAA,GACF;AAEA,EAAI,IAAA,YAAA,KAAiB,KAAa,CAAA,IAAA,YAAA,KAAiB,WAAa,EAAA;AAC9D,IAAO,OAAA;AAAA,MACL,IAAM,EAAA;AAAA,KACR;AAAA,GACF,MAAA,IAAW,iBAAiB,UAAY,EAAA;AACtC,IAAA,MAAM,kBAAqB,GAAA,2CAAA;AAC3B,IAAA,MAAM,gBAAmB,GAAA,yCAAA;AAEzB,IAAA,MAAM,eAAkB,GAAAA,QAAA,CAAO,GAAI,CAAA,kBAAkB,IACjDC,6BAAuB,CAAAD,QAAA,EAAQ,EAAE,GAAA,EAAK,kBAAmB,EAAC,CAC1D,GAAA,EAAE,SAAS,CAAE,EAAA;AACjB,IAAA,MAAM,aAAgB,GAAAA,QAAA,CAAO,GAAI,CAAA,gBAAgB,IAC7CC,6BAAuB,CAAAD,QAAA,EAAQ,EAAE,GAAA,EAAK,gBAAiB,EAAC,CACxD,GAAA,EAAE,SAAS,EAAG,EAAA;AAElB,IAAO,OAAA;AAAA,MACL,IAAM,EAAA,UAAA;AAAA,MACN,eAAA;AAAA,MACA;AAAA,KACF;AAAA;AAGF,EAAA,MAAM,IAAI,KAAA;AAAA,IACR,oCAAoC,YAAY,CAAA,4CAAA;AAAA,GAClD;AACF;;;;"}

package/migrations/20241024104700_add_entity_ref_to_final_entities.js

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.up = async function up(knex) {
+  await knex.schema.alterTable('final_entities', table => {
+    table
+      .string('entity_ref')
+      .nullable()
+      .comment('The entity reference of the entity');
+  });
+
+  await knex
+    .update({
+      entity_ref: knex
+        .select('r.entity_ref')
+        .from('refresh_state as r')
+        .where('r.entity_id', knex.raw('f.entity_id')),
+    })
+    .table('final_entities as f');
+
+  await knex.schema.alterTable('final_entities', table => {
+    table.string('entity_ref').notNullable().alter({ alterNullable: true });
+    table.unique(['entity_ref'], {
+      indexName: 'final_entities_entity_ref_uniq',
+    });
+  });
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.down = async function down(knex) {
+  await knex.schema.alterTable('final_entities', table => {
+    table.dropUnique([], 'final_entities_entity_ref_uniq');
+    table.dropColumn('entity_ref');
+  });
+};

package/migrations/20241111000000_drop_redundant_indices.js

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.up = async function up(knex) {
+  await knex.schema.alterTable('final_entities', table => {
+    table.dropIndex([], 'final_entities_entity_id_idx'); // overlaps with final_entities_pkey
+  });
+
+  await knex.schema.alterTable('refresh_state', table => {
+    table.dropIndex([], 'refresh_state_entity_id_idx'); // overlaps with refresh_state_pkey
+    table.dropIndex([], 'refresh_state_entity_ref_idx'); // overlaps with refresh_state_entity_ref_uniq
+  });
+
+  await knex.schema.alterTable('search', table => {
+    table.dropIndex([], 'search_key_idx'); // was replaced by search_key_value_idx in 20240130092632_search_index
+    table.dropIndex([], 'search_value_idx'); // was replaced by search_key_value_idx in 20240130092632_search_index
+  });
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.down = async function down(knex) {
+  await knex.schema.alterTable('final_entities', table => {
+    table.index('entity_id', 'final_entities_entity_id_idx');
+  });
+
+  await knex.schema.alterTable('refresh_state', table => {
+    table.index('entity_id', 'refresh_state_entity_id_idx');
+    table.index('entity_ref', 'refresh_state_entity_ref_idx');
+  });
+
+  await knex.schema.alterTable('search', table => {
+    table.index(['key'], 'search_key_idx');
+    table.index(['value'], 'search_value_idx');
+  });
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend",
-  "version": "1.28.0-next.2",
+  "version": "1.28.0",
   "description": "The Backstage backend plugin that provides the Backstage catalog",
   "backstage": {
     "role": "backend-plugin",
@@ -43,11 +43,20 @@
   },
   "main": "./dist/index.cjs.js",
   "types": "./dist/index.d.ts",
+  "typesVersions": {
+    "*": {
+      "index": [
+        "dist/index.d.ts"
+      ],
+      "alpha": [
+        "dist/alpha.d.ts"
+      ]
+    }
+  },
   "files": [
     "dist",
     "migrations/**/*.{js,d.ts}",
-    "config.d.ts",
-    "alpha"
+    "config.d.ts"
   ],
   "scripts": {
     "build": "backstage-cli package build",
@@ -63,20 +72,20 @@
   },
   "dependencies": {
     "@backstage/backend-common": "^0.25.0",
-    "@backstage/backend-openapi-utils": "0.3.0-next.2",
-    "@backstage/backend-plugin-api": "1.0.2-next.2",
-    "@backstage/catalog-client": "1.8.0-next.1",
-    "@backstage/catalog-model": "1.7.0",
-    "@backstage/config": "1.2.0",
-    "@backstage/errors": "1.2.4",
-    "@backstage/integration": "1.15.1",
-    "@backstage/plugin-catalog-common": "1.1.0",
-    "@backstage/plugin-catalog-node": "1.14.0-next.2",
-    "@backstage/plugin-events-node": "0.4.5-next.2",
-    "@backstage/plugin-permission-common": "0.8.1",
-    "@backstage/plugin-permission-node": "0.8.5-next.2",
-    "@backstage/plugin-search-backend-module-catalog": "0.2.5-next.2",
-    "@backstage/types": "1.1.1",
+    "@backstage/backend-openapi-utils": "^0.3.0",
+    "@backstage/backend-plugin-api": "^1.0.2",
+    "@backstage/catalog-client": "^1.8.0",
+    "@backstage/catalog-model": "^1.7.1",
+    "@backstage/config": "^1.3.0",
+    "@backstage/errors": "^1.2.5",
+    "@backstage/integration": "^1.15.2",
+    "@backstage/plugin-catalog-common": "^1.1.1",
+    "@backstage/plugin-catalog-node": "^1.14.0",
+    "@backstage/plugin-events-node": "^0.4.5",
+    "@backstage/plugin-permission-common": "^0.8.2",
+    "@backstage/plugin-permission-node": "^0.8.5",
+    "@backstage/plugin-search-backend-module-catalog": "^0.2.5",
+    "@backstage/types": "^1.2.0",
     "@opentelemetry/api": "^1.3.0",
     "@types/express": "^4.17.6",
     "codeowners-utils": "^1.0.2",
@@ -93,23 +102,22 @@
     "node-fetch": "^2.7.0",
     "p-limit": "^3.0.2",
     "prom-client": "^15.0.0",
-    "uuid": "^9.0.0",
+    "uuid": "^11.0.0",
     "yaml": "^2.0.0",
     "yn": "^4.0.0",
     "zod": "^3.22.4"
   },
   "devDependencies": {
-    "@backstage/backend-defaults": "0.5.3-next.2",
-    "@backstage/backend-test-utils": "1.1.0-next.2",
-    "@backstage/cli": "0.29.0-next.2",
-    "@backstage/plugin-permission-common": "0.8.1",
-    "@backstage/repo-tools": "0.11.0-next.2",
+    "@backstage/backend-defaults": "^0.5.3",
+    "@backstage/backend-test-utils": "^1.1.0",
+    "@backstage/cli": "^0.29.0",
+    "@backstage/plugin-permission-common": "^0.8.2",
+    "@backstage/repo-tools": "^0.11.0",
     "@types/core-js": "^2.5.4",
     "@types/git-url-parse": "^9.0.0",
     "@types/glob": "^8.0.0",
     "@types/lodash": "^4.14.151",
     "@types/supertest": "^2.0.8",
-    "@types/uuid": "^9.0.0",
     "better-sqlite3": "^11.0.0",
     "luxon": "^3.0.0",
     "msw": "^1.0.0",

package/alpha/package.json

@@ -1,6 +0,0 @@
-{
-  "name": "@backstage/plugin-catalog-backend__alpha",
-  "version": "1.28.0-next.2",
-  "main": "../dist/alpha.cjs.js",
-  "types": "../dist/alpha.d.ts"
-}