@backstage/plugin-catalog-backend 3.7.0 → 3.8.0-next.0

package/CHANGELOG.md

@@ -1,5 +1,35 @@
 # @backstage/plugin-catalog-backend
 
+## 3.8.0-next.0
+
+### Minor Changes
+
+- 8f20cc2: `/entities/by-query` now accepts a `totalItems` parameter (`'include'` or `'exclude'`, default `'include'`) that controls whether the response's `totalItems` count is computed. Pass `'exclude'` to skip the count entirely when the caller doesn't need it — useful for cursor-paginated user interfaces that only display the count cosmetically. The accepted values list is forward-compatible: future modes (e.g. approximate counts) can be added without breaking existing callers.
+
+  The internal `QueryEntitiesInitialRequest.skipTotalItems` option has been replaced by `totalItems: 'include' | 'exclude'`. Note that `skipTotalItems` was never exposed as a REST API parameter, so this is only a TypeScript-level change affecting direct callers of `EntitiesCatalog.queryEntities`.
+
+  Sort field keys are now lowercased before comparing against `search.key`, fixing silent mismatches for camelCase field names. The `NULLS LAST` ordering clause has been removed since NULL sort values are already excluded by the `WHERE` clause.
+
+- dc7678c: Removed the immediate mode stitching strategy. All stitching now uses the deferred mode, which processes entities asynchronously via a worker queue. If your configuration includes `catalog.stitchingStrategy.mode: 'immediate'`, it will be ignored with a deprecation warning. The `pollingInterval` and `stitchTimeout` settings continue to work as before.
+
+### Patch Changes
+
+- 39c5fbb: Added extended multi-column statistics on `(key, value)` in the `search` table (PostgreSQL only). This tells the query planner about the correlation between the `key` and `value` columns, fixing severe row count estimation errors on compound filter queries. Without this, the planner could choose to materialize and sort thousands of rows instead of using the LIMIT short-circuit index scan — causing 10-40x slower catalog list views when multiple filters are active.
+- 4829e89: Split the `queryEntities` list and count into separate queries instead of a multi-reference CTE. When the `filtered` CTE was referenced twice (once for the count, once for the data), PostgreSQL refused to inline it, forcing full materialization of the filtered set before applying `LIMIT`. By running the count as a standalone query, the list CTE is only referenced once, allowing the planner to short-circuit on `LIMIT` and return the first page in milliseconds instead of waiting for the full filtered set to materialize.
+
+  The standalone count query also fixes a pre-existing bug where `totalItems` was inflated for entities with multi-valued sort fields (e.g. tags). The old CTE-based count counted search rows, so an entity with 3 tags would be counted 3 times. The new count uses `EXISTS` to count distinct entities, aligning `totalItems` with the number of entities actually reachable through cursor pagination.
+
+- 774d698: Fixed a race condition in the stitch queue and entity processing claim logic where `SELECT FOR UPDATE SKIP LOCKED` row locks were released before the subsequent timestamp bump, allowing multiple workers to claim the same rows. Both the select and update now run inside a single transaction for MySQL and PostgreSQL.
+- 0b8b677: Improved stitch queue semantics to prevent overlapping stitches for the same entity. New stitch requests that arrive while a stitch is in progress now only update the ticket (not the timestamp), so the in-progress worker is not interrupted. When the worker completes and detects a pending re-stitch, the queue entry becomes immediately eligible for pickup instead of waiting for the timeout period.
+- Updated dependencies
+  - @backstage/catalog-client@1.16.0-next.0
+  - @backstage/integration@2.0.3-next.0
+  - @backstage/plugin-catalog-node@2.2.2-next.0
+  - @backstage/plugin-permission-node@0.11.1-next.0
+  - @backstage/backend-plugin-api@1.9.2-next.0
+  - @backstage/plugin-events-node@0.4.23-next.0
+  - @backstage/backend-openapi-utils@0.6.10-next.0
+
 ## 3.7.0
 
 ### Minor Changes

package/config.d.ts

@@ -175,25 +175,16 @@ export interface Config {
     orphanProviderStrategy?: 'keep' | 'delete';
 
     /**
-     * The strategy to use when stitching together the final entities. The default mode is "deferred".
+     * The strategy to use when stitching together the final entities.
      */
-    stitchingStrategy?:
-      | {
-          /**
-           * Perform stitching in-band immediately when needed.
-           *
-           * @deprecated Immediate mode stitching has been deprecated and will be removed in a future release. Migrate to deferred mode (the default).
-           */
-          mode: 'immediate';
-        }
-      | {
-          /** Defer stitching to be performed asynchronously */
-          mode: 'deferred';
-          /** Polling interval for tasks in seconds */
-          pollingInterval?: HumanDuration | string;
-          /** How long to wait for a stitch to complete before giving up in seconds */
-          stitchTimeout?: HumanDuration | string;
-        };
+    stitchingStrategy?: {
+      /** @deprecated Immediate mode has been removed. This field is ignored. */
+      mode?: string;
+      /** Polling interval for tasks in seconds */
+      pollingInterval?: HumanDuration | string;
+      /** How long to wait for a stitch to complete before giving up in seconds */
+      stitchTimeout?: HumanDuration | string;
+    };
 
     /**
      * The strategy to use when there is a conflict with a location being registered.

package/dist/database/DefaultProcessingDatabase.cjs.js

@@ -107,35 +107,44 @@ class DefaultProcessingDatabase {
   }
   async getProcessableEntities(maybeTx, request) {
     const knex = maybeTx;
-    let itemsQuery = knex("refresh_state").select([
-      "entity_id",
-      "entity_ref",
-      "unprocessed_entity",
-      "result_hash",
-      "cache",
-      "errors",
-      "location_key",
-      "next_update_at"
-    ]);
-    if (["mysql", "mysql2", "pg"].includes(knex.client.config.client)) {
-      itemsQuery = itemsQuery.forUpdate().skipLocked();
-    }
-    const items = await itemsQuery.where("next_update_at", "<=", knex.fn.now()).limit(request.processBatchSize).orderBy("next_update_at", "asc");
+    const useLocking = ["mysql", "mysql2", "pg"].includes(
+      knex.client.config.client
+    );
     const interval = this.options.refreshInterval();
-    const nextUpdateAt = (refreshInterval) => {
-      if (knex.client.config.client.includes("sqlite3")) {
-        return knex.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);
-      } else if (knex.client.config.client.includes("mysql")) {
-        return knex.raw(`now() + interval ${refreshInterval} second`);
+    const nextUpdateAt = (tx, refreshInterval) => {
+      if (tx.client.config.client.includes("sqlite3")) {
+        return tx.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);
+      } else if (tx.client.config.client.includes("mysql")) {
+        return tx.raw(`now() + interval ${refreshInterval} second`);
       }
-      return knex.raw(`now() + interval '${refreshInterval} seconds'`);
+      return tx.raw(`now() + interval '${refreshInterval} seconds'`);
     };
-    await knex("refresh_state").whereIn(
-      "entity_ref",
-      items.map((i) => i.entity_ref)
-    ).update({
-      next_update_at: nextUpdateAt(interval)
-    });
+    const run = async (tx) => {
+      const items2 = await tx("refresh_state").select([
+        "entity_id",
+        "entity_ref",
+        "unprocessed_entity",
+        "result_hash",
+        "cache",
+        "errors",
+        "location_key",
+        "next_update_at"
+      ]).where("next_update_at", "<=", tx.fn.now()).limit(request.processBatchSize).orderBy("next_update_at", "asc").modify((qb) => {
+        if (useLocking) {
+          qb.forUpdate().skipLocked();
+        }
+      });
+      if (items2.length > 0) {
+        await tx("refresh_state").whereIn(
+          "entity_ref",
+          items2.map((i) => i.entity_ref)
+        ).update({
+          next_update_at: nextUpdateAt(tx, interval)
+        });
+      }
+      return items2;
+    };
+    const items = knex.isTransaction || !useLocking ? await run(knex) : await knex.transaction(run);
     return {
       items: items.map(
         (i) => ({

package/dist/database/DefaultProcessingDatabase.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultProcessingDatabase.cjs.js","sources":["../../src/database/DefaultProcessingDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError } from '@backstage/errors';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { ProcessingIntervalFunction } from '../processing/refresh';\nimport { rethrowError, timestampToDateTime } from './conversion';\nimport { initDatabaseMetrics } from './metrics';\nimport {\n  DbRefreshKeysRow,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n} from './tables';\nimport {\n  GetProcessableEntitiesResult,\n  ListParentsOptions,\n  ListParentsResult,\n  ProcessingDatabase,\n  RefreshStateItem,\n  Transaction,\n  UpdateEntityCacheOptions,\n  UpdateProcessedEntityOptions,\n} from './types';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { generateStableHash, generateTargetKey } from './util';\nimport { EventParams, EventsService } from '@backstage/plugin-events-node';\nimport { DateTime } from 'luxon';\nimport { CATALOG_CONFLICTS_TOPIC } from '../constants';\nimport { CatalogConflictEventPayload } from '../catalog/types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\nimport { MetricsService } from '@backstage/backend-plugin-api/alpha';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProcessingDatabase implements ProcessingDatabase {\n  private readonly options: {\n    database: Knex;\n    logger: LoggerService;\n    refreshInterval: ProcessingIntervalFunction;\n    events: EventsService;\n    metrics: MetricsService;\n  };\n\n  constructor(options: {\n    database: Knex;\n    logger: LoggerService;\n    refreshInterval: ProcessingIntervalFunction;\n    events: EventsService;\n    metrics: MetricsService;\n  }) {\n    this.options = options;\n    initDatabaseMetrics(options.database, options.metrics);\n  }\n\n  async updateProcessedEntity(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<{ previous: { relations: DbRelationsRow[] } }> {\n    const tx = txOpaque as Knex.Transaction;\n    const {\n      id,\n      processedEntity,\n      resultHash,\n      errors,\n      relations,\n      deferredEntities,\n      refreshKeys,\n      locationKey,\n    } = options;\n    const configClient = tx.client.config.client;\n    const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        processed_entity: JSON.stringify(processedEntity),\n        result_hash: resultHash,\n        errors,\n        location_key: locationKey,\n      })\n      .where('entity_id', id)\n      .andWhere(inner => {\n        if (!locationKey) {\n          return inner.whereNull('location_key');\n        }\n        return inner\n          .where('location_key', locationKey)\n          .orWhereNull('location_key');\n      });\n    if (refreshResult === 0) {\n      throw new ConflictError(\n        `Conflicting write of processing result for ${id} with location key '${locationKey}'`,\n      );\n    }\n    const sourceEntityRef = stringifyEntityRef(processedEntity);\n\n    // Schedule all deferred entities for future processing.\n    await this.addUnprocessedEntities(tx, {\n      entities: deferredEntities,\n      sourceEntityRef,\n    });\n\n    // Delete old relations\n    // NOTE(freben): knex implemented support for returning() on update queries for sqlite, but at the current time of writing (Sep 2022) not for delete() queries.\n    let previousRelationRows: DbRelationsRow[];\n    if (configClient.includes('sqlite3') || configClient.includes('mysql')) {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .select('*')\n        .where({ originating_entity_id: id });\n      await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete();\n    } else {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete()\n        .returning('*');\n    }\n\n    // Batch insert new relations\n    const relationRows: DbRelationsRow[] = relations.map(\n      ({ source, target, type }) => ({\n        originating_entity_id: id,\n        source_entity_ref: stringifyEntityRef(source),\n        target_entity_ref: stringifyEntityRef(target),\n        type,\n      }),\n    );\n\n    await tx.batchInsert(\n      'relations',\n      this.deduplicateRelations(relationRows),\n      BATCH_SIZE,\n    );\n\n    // Delete old refresh keys\n    await tx<DbRefreshKeysRow>('refresh_keys')\n      .where({ entity_id: id })\n      .delete();\n\n    // Insert the refresh keys for the processed entity\n    await tx.batchInsert(\n      'refresh_keys',\n      refreshKeys.map(k => ({\n        entity_id: id,\n        key: generateTargetKey(k.key),\n      })),\n      BATCH_SIZE,\n    );\n\n    return {\n      previous: {\n        relations: previousRelationRows,\n      },\n    };\n  }\n\n  async updateProcessedEntityErrors(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, errors, resultHash } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        errors,\n        result_hash: resultHash,\n      })\n      .where('entity_id', id);\n  }\n\n  async updateEntityCache(\n    txOpaque: Transaction,\n    options: UpdateEntityCacheOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, state } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({ cache: JSON.stringify(state ?? {}) })\n      .where('entity_id', id);\n  }\n\n  async getProcessableEntities(\n    maybeTx: Transaction | Knex,\n    request: { processBatchSize: number },\n  ): Promise<GetProcessableEntitiesResult> {\n    const knex = maybeTx as Knex.Transaction | Knex;\n\n    let itemsQuery = knex<DbRefreshStateRow>('refresh_state').select([\n      'entity_id',\n      'entity_ref',\n      'unprocessed_entity',\n      'result_hash',\n      'cache',\n      'errors',\n      'location_key',\n      'next_update_at',\n    ]);\n\n    // This avoids duplication of work because of race conditions and is\n    // also fast because locked rows are ignored rather than blocking.\n    // It's only available in MySQL and PostgreSQL\n    if (['mysql', 'mysql2', 'pg'].includes(knex.client.config.client)) {\n      itemsQuery = itemsQuery.forUpdate().skipLocked();\n    }\n\n    const items = await itemsQuery\n      .where('next_update_at', '<=', knex.fn.now())\n      .limit(request.processBatchSize)\n      .orderBy('next_update_at', 'asc');\n\n    const interval = this.options.refreshInterval();\n\n    const nextUpdateAt = (refreshInterval: number) => {\n      if (knex.client.config.client.includes('sqlite3')) {\n        return knex.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);\n      } else if (knex.client.config.client.includes('mysql')) {\n        return knex.raw(`now() + interval ${refreshInterval} second`);\n      }\n      return knex.raw(`now() + interval '${refreshInterval} seconds'`);\n    };\n\n    await knex<DbRefreshStateRow>('refresh_state')\n      .whereIn(\n        'entity_ref',\n        items.map(i => i.entity_ref),\n      )\n      .update({\n        next_update_at: nextUpdateAt(interval),\n      });\n\n    return {\n      items: items.map(\n        i =>\n          ({\n            id: i.entity_id,\n            entityRef: i.entity_ref,\n            unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity,\n            resultHash: i.result_hash || '',\n            nextUpdateAt: timestampToDateTime(i.next_update_at),\n            state: i.cache ? JSON.parse(i.cache) : undefined,\n            errors: i.errors,\n            locationKey: i.location_key,\n          } satisfies RefreshStateItem),\n      ),\n    };\n  }\n\n  async listParents(\n    txOpaque: Transaction,\n    options: ListParentsOptions,\n  ): Promise<ListParentsResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .whereIn('target_entity_ref', options.entityRefs)\n      .select();\n\n    const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);\n\n    return { entityRefs };\n  }\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] {\n    return lodash.uniqBy(\n      rows,\n      r => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`,\n    );\n  }\n\n  /**\n   * Add a set of deferred entities for processing.\n   * The entities will be added at the front of the processing queue.\n   */\n  private async addUnprocessedEntities(\n    txOpaque: Transaction,\n    options: {\n      sourceEntityRef: string;\n      entities: DeferredEntity[];\n    },\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n\n    // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards\n    const stateReferences = new Array<string>();\n\n    // Upsert all of the unprocessed entities into the refresh_state table, by\n    // their entity ref.\n    for (const { entity, locationKey } of options.entities) {\n      const entityRef = stringifyEntityRef(entity);\n      const hash = generateStableHash(entity);\n\n      const updated = await updateUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n      });\n      if (updated) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      const inserted = await insertUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n        logger: this.options.logger,\n      });\n      if (inserted) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      // If the row can't be inserted, we have a conflict, but it could be either\n      // because of a conflicting locationKey or a race with another instance, so check\n      // whether the conflicting entity has the same entityRef but a different locationKey\n      const conflictingKey = await checkLocationKeyConflict({\n        tx,\n        entityRef,\n        locationKey,\n      });\n      if (conflictingKey) {\n        this.options.logger.warn(\n          `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n        );\n        if (locationKey) {\n          const eventParams: EventParams<CatalogConflictEventPayload> = {\n            topic: CATALOG_CONFLICTS_TOPIC,\n            eventPayload: {\n              unprocessedEntity: entity,\n              entityRef,\n              newLocationKey: locationKey,\n              existingLocationKey: conflictingKey,\n              lastConflictAt: DateTime.now().toISO()!,\n            },\n          };\n          await this.options.events.publish(eventParams);\n        }\n      }\n    }\n\n    // Lastly, replace refresh state references for the originating entity and any successfully added entities\n    await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n      // Remove all existing references from the originating entity\n      .where({ source_entity_ref: options.sourceEntityRef })\n      // And remove any existing references to entities that we're inserting new references for\n      .orWhereIn('target_entity_ref', stateReferences)\n      .delete();\n    await tx.batchInsert(\n      'refresh_state_references',\n      stateReferences.map(entityRef => ({\n        source_entity_ref: options.sourceEntityRef,\n        target_entity_ref: entityRef,\n      })),\n      BATCH_SIZE,\n    );\n  }\n}\n"],"names":["initDatabaseMetrics","errors","ConflictError","stringifyEntityRef","generateTargetKey","timestampToDateTime","rethrowError","lodash","generateStableHash","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","CATALOG_CONFLICTS_TOPIC","DateTime"],"mappings":";;;;;;;;;;;;;;;;;;AAuDA,MAAM,UAAA,GAAa,EAAA;AAEZ,MAAM,yBAAA,CAAwD;AAAA,EAClD,OAAA;AAAA,EAQjB,YAAY,OAAA,EAMT;AACD,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA;AACf,IAAAA,2BAAA,CAAoB,OAAA,CAAQ,QAAA,EAAU,OAAA,CAAQ,OAAO,CAAA;AAAA,EACvD;AAAA,EAEA,MAAM,qBAAA,CACJ,QAAA,EACA,OAAA,EACwD;AACxD,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM;AAAA,MACJ,EAAA;AAAA,MACA,eAAA;AAAA,MACA,UAAA;AAAA,cACAC,QAAA;AAAA,MACA,SAAA;AAAA,MACA,gBAAA;AAAA,MACA,WAAA;AAAA,MACA;AAAA,KACF,GAAI,OAAA;AACJ,IAAA,MAAM,YAAA,GAAe,EAAA,CAAG,MAAA,CAAO,MAAA,CAAO,MAAA;AACtC,IAAA,MAAM,aAAA,GAAgB,MAAM,EAAA,CAAsB,eAAe,EAC9D,MAAA,CAAO;AAAA,MACN,gBAAA,EAAkB,IAAA,CAAK,SAAA,CAAU,eAAe,CAAA;AAAA,MAChD,WAAA,EAAa,UAAA;AAAA,cACbA,QAAA;AAAA,MACA,YAAA,EAAc;AAAA,KACf,CAAA,CACA,KAAA,CAAM,aAAa,EAAE,CAAA,CACrB,SAAS,CAAA,KAAA,KAAS;AACjB,MAAA,IAAI,CAAC,WAAA,EAAa;AAChB,QAAA,OAAO,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA,MACvC;AACA,MAAA,OAAO,MACJ,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,IAC/B,CAAC,CAAA;AACH,IAAA,IAAI,kBAAkB,CAAA,EAAG;AACvB,MAAA,MAAM,IAAIC,oBAAA;AAAA,QACR,CAAA,2CAAA,EAA8C,EAAE,CAAA,oBAAA,EAAuB,WAAW,CAAA,CAAA;AAAA,OACpF;AAAA,IACF;AACA,IAAA,MAAM,eAAA,GAAkBC,gCAAmB,eAAe,CAAA;AAG1D,IAAA,MAAM,IAAA,CAAK,uBAAuB,EAAA,EAAI;AAAA,MACpC,QAAA,EAAU,gBAAA;AAAA,MACV;AAAA,KACD,CAAA;AAID,IAAA,IAAI,oBAAA;AACJ,IAAA,IAAI,aAAa,QAAA,CAAS,SAAS,KAAK,YAAA,CAAa,QAAA,CAAS,OAAO,CAAA,EAAG;AACtE,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,MAAA,CAAO,GAAG,CAAA,CACV,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA;AACtC,MAAA,MAAM,EAAA,CAAmB,WAAW,CAAA,CACjC,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO;AAAA,IACZ,CAAA,MAAO;AACL,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO,CACP,UAAU,GAAG,CAAA;AAAA,IAClB;AAGA,IAAA,MAAM,eAAiC,SAAA,CAAU,GAAA;AAAA,MAC/C,CAAC,EAAE,MAAA,EAAQ,MAAA,EAAQ,MAAK,MAAO;AAAA,QAC7B,qBAAA,EAAuB,EAAA;AAAA,QACvB,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C;AAAA,OACF;AAAA,KACF;AAEA,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,WAAA;AAAA,MACA,IAAA,CAAK,qBAAqB,YAAY,CAAA;AAAA,MACtC;AAAA,KACF;AAGA,IAAA,MAAM,EAAA,CAAqB,cAAc,CAAA,CACtC,KAAA,CAAM,EAAE,SAAA,EAAW,EAAA,EAAI,CAAA,CACvB,MAAA,EAAO;AAGV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,cAAA;AAAA,MACA,WAAA,CAAY,IAAI,CAAA,CAAA,MAAM;AAAA,QACpB,SAAA,EAAW,EAAA;AAAA,QACX,GAAA,EAAKC,sBAAA,CAAkB,CAAA,CAAE,GAAG;AAAA,OAC9B,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAEA,IAAA,OAAO;AAAA,MACL,QAAA,EAAU;AAAA,QACR,SAAA,EAAW;AAAA;AACb,KACF;AAAA,EACF;AAAA,EAEA,MAAM,2BAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,MAAA,EAAQ,UAAA,EAAW,GAAI,OAAA;AAEnC,IAAA,MAAM,EAAA,CAAsB,eAAe,CAAA,CACxC,MAAA,CAAO;AAAA,MACN,MAAA;AAAA,MACA,WAAA,EAAa;AAAA,KACd,CAAA,CACA,KAAA,CAAM,WAAA,EAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,iBAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,KAAA,EAAM,GAAI,OAAA;AAEtB,IAAA,MAAM,GAAsB,eAAe,CAAA,CACxC,MAAA,CAAO,EAAE,OAAO,IAAA,CAAK,SAAA,CAAU,KAAA,IAAS,EAAE,CAAA,EAAG,CAAA,CAC7C,KAAA,CAAM,aAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,sBAAA,CACJ,OAAA,EACA,OAAA,EACuC;AACvC,IAAA,MAAM,IAAA,GAAO,OAAA;AAEb,IAAA,IAAI,UAAA,GAAa,IAAA,CAAwB,eAAe,CAAA,CAAE,MAAA,CAAO;AAAA,MAC/D,WAAA;AAAA,MACA,YAAA;AAAA,MACA,oBAAA;AAAA,MACA,aAAA;AAAA,MACA,OAAA;AAAA,MACA,QAAA;AAAA,MACA,cAAA;AAAA,MACA;AAAA,KACD,CAAA;AAKD,IAAA,IAAI,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,SAAS,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,MAAM,CAAA,EAAG;AACjE,MAAA,UAAA,GAAa,UAAA,CAAW,SAAA,EAAU,CAAE,UAAA,EAAW;AAAA,IACjD;AAEA,IAAA,MAAM,QAAQ,MAAM,UAAA,CACjB,KAAA,CAAM,gBAAA,EAAkB,MAAM,IAAA,CAAK,EAAA,CAAG,GAAA,EAAK,EAC3C,KAAA,CAAM,OAAA,CAAQ,gBAAgB,CAAA,CAC9B,OAAA,CAAQ,kBAAkB,KAAK,CAAA;AAElC,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,OAAA,CAAQ,eAAA,EAAgB;AAE9C,IAAA,MAAM,YAAA,GAAe,CAAC,eAAA,KAA4B;AAChD,MAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AACjD,QAAA,OAAO,KAAK,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,eAAe,UAAU,CAAC,CAAA;AAAA,MACtE,WAAW,IAAA,CAAK,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACtD,QAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,iBAAA,EAAoB,eAAe,CAAA,OAAA,CAAS,CAAA;AAAA,MAC9D;AACA,MAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,kBAAA,EAAqB,eAAe,CAAA,SAAA,CAAW,CAAA;AAAA,IACjE,CAAA;AAEA,IAAA,MAAM,IAAA,CAAwB,eAAe,CAAA,CAC1C,OAAA;AAAA,MACC,YAAA;AAAA,MACA,KAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,MAE5B,MAAA,CAAO;AAAA,MACN,cAAA,EAAgB,aAAa,QAAQ;AAAA,KACtC,CAAA;AAEH,IAAA,OAAO;AAAA,MACL,OAAO,KAAA,CAAM,GAAA;AAAA,QACX,CAAA,CAAA,MACG;AAAA,UACC,IAAI,CAAA,CAAE,SAAA;AAAA,UACN,WAAW,CAAA,CAAE,UAAA;AAAA,UACb,iBAAA,EAAmB,IAAA,CAAK,KAAA,CAAM,CAAA,CAAE,kBAAkB,CAAA;AAAA,UAClD,UAAA,EAAY,EAAE,WAAA,IAAe,EAAA;AAAA,UAC7B,YAAA,EAAcC,8BAAA,CAAoB,CAAA,CAAE,cAAc,CAAA;AAAA,UAClD,OAAO,CAAA,CAAE,KAAA,GAAQ,KAAK,KAAA,CAAM,CAAA,CAAE,KAAK,CAAA,GAAI,MAAA;AAAA,UACvC,QAAQ,CAAA,CAAE,MAAA;AAAA,UACV,aAAa,CAAA,CAAE;AAAA,SACjB;AAAA;AACJ,KACF;AAAA,EACF;AAAA,EAEA,MAAM,WAAA,CACJ,QAAA,EACA,OAAA,EAC4B;AAC5B,IAAA,MAAM,EAAA,GAAK,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,MAEC,OAAA,CAAQ,mBAAA,EAAqB,OAAA,CAAQ,UAAU,EAC/C,MAAA,EAAO;AAEV,IAAA,MAAM,UAAA,GAAa,KAAK,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,iBAAkB,CAAA,CAAE,OAAO,OAAO,CAAA;AAErE,IAAA,OAAO,EAAE,UAAA,EAAW;AAAA,EACtB;AAAA,EAEA,MAAM,YAAe,EAAA,EAAiD;AACpE,IAAA,IAAI;AACF,MAAA,IAAI,MAAA,GAAwB,KAAA,CAAA;AAE5B,MAAA,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,WAAA;AAAA,QAC1B,OAAM,EAAA,KAAM;AAGV,UAAA,MAAA,GAAS,MAAM,GAAG,EAAE,CAAA;AAAA,QACtB,CAAA;AAAA,QACA;AAAA;AAAA,UAEE,qBAAA,EAAuB;AAAA;AACzB,OACF;AAEA,MAAA,OAAO,MAAA;AAAA,IACT,SAAS,CAAA,EAAG;AACV,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,0BAAA,EAA6B,CAAC,CAAA,CAAE,CAAA;AAC1D,MAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA,IACtB;AAAA,EACF;AAAA,EAEQ,qBAAqB,IAAA,EAA0C;AACrE,IAAA,OAAOC,uBAAA,CAAO,MAAA;AAAA,MACZ,IAAA;AAAA,MACA,CAAA,CAAA,KAAK,GAAG,CAAA,CAAE,iBAAiB,IAAI,CAAA,CAAE,iBAAiB,CAAA,CAAA,EAAI,CAAA,CAAE,IAAI,CAAA;AAAA,KAC9D;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,sBAAA,CACZ,QAAA,EACA,OAAA,EAIe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AAGX,IAAA,MAAM,eAAA,GAAkB,IAAI,KAAA,EAAc;AAI1C,IAAA,KAAA,MAAW,EAAE,MAAA,EAAQ,WAAA,EAAY,IAAK,QAAQ,QAAA,EAAU;AACtD,MAAA,MAAM,SAAA,GAAYJ,gCAAmB,MAAM,CAAA;AAC3C,MAAA,MAAM,IAAA,GAAOK,wBAAmB,MAAM,CAAA;AAEtC,MAAA,MAAM,OAAA,GAAU,MAAMC,+CAAA,CAAwB;AAAA,QAC5C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,OAAA,EAAS;AACX,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,QAAA,GAAW,MAAMC,+CAAA,CAAwB;AAAA,QAC7C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA,WAAA;AAAA,QACA,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAKA,MAAA,MAAM,cAAA,GAAiB,MAAMC,iDAAA,CAAyB;AAAA,QACpD,EAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,cAAA,EAAgB;AAClB,QAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,UAClB,CAAA,+BAAA,EAAkC,SAAS,CAAA,uBAAA,EAA0B,cAAc,iBAAiB,WAAW,CAAA;AAAA,SACjH;AACA,QAAA,IAAI,WAAA,EAAa;AACf,UAAA,MAAM,WAAA,GAAwD;AAAA,YAC5D,KAAA,EAAOC,iCAAA;AAAA,YACP,YAAA,EAAc;AAAA,cACZ,iBAAA,EAAmB,MAAA;AAAA,cACnB,SAAA;AAAA,cACA,cAAA,EAAgB,WAAA;AAAA,cAChB,mBAAA,EAAqB,cAAA;AAAA,cACrB,cAAA,EAAgBC,cAAA,CAAS,GAAA,EAAI,CAAE,KAAA;AAAM;AACvC,WACF;AACA,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,OAAA,CAAQ,WAAW,CAAA;AAAA,QAC/C;AAAA,MACF;AAAA,IACF;AAGA,IAAA,MAAM,EAAA,CAAgC,0BAA0B,CAAA,CAE7D,KAAA,CAAM,EAAE,iBAAA,EAAmB,OAAA,CAAQ,eAAA,EAAiB,CAAA,CAEpD,SAAA,CAAU,mBAAA,EAAqB,eAAe,EAC9C,MAAA,EAAO;AACV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,0BAAA;AAAA,MACA,eAAA,CAAgB,IAAI,CAAA,SAAA,MAAc;AAAA,QAChC,mBAAmB,OAAA,CAAQ,eAAA;AAAA,QAC3B,iBAAA,EAAmB;AAAA,OACrB,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAAA,EACF;AACF;;;;"}
+{"version":3,"file":"DefaultProcessingDatabase.cjs.js","sources":["../../src/database/DefaultProcessingDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError } from '@backstage/errors';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { ProcessingIntervalFunction } from '../processing/refresh';\nimport { rethrowError, timestampToDateTime } from './conversion';\nimport { initDatabaseMetrics } from './metrics';\nimport {\n  DbRefreshKeysRow,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n} from './tables';\nimport {\n  GetProcessableEntitiesResult,\n  ListParentsOptions,\n  ListParentsResult,\n  ProcessingDatabase,\n  RefreshStateItem,\n  Transaction,\n  UpdateEntityCacheOptions,\n  UpdateProcessedEntityOptions,\n} from './types';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { generateStableHash, generateTargetKey } from './util';\nimport { EventParams, EventsService } from '@backstage/plugin-events-node';\nimport { DateTime } from 'luxon';\nimport { CATALOG_CONFLICTS_TOPIC } from '../constants';\nimport { CatalogConflictEventPayload } from '../catalog/types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\nimport { MetricsService } from '@backstage/backend-plugin-api/alpha';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProcessingDatabase implements ProcessingDatabase {\n  private readonly options: {\n    database: Knex;\n    logger: LoggerService;\n    refreshInterval: ProcessingIntervalFunction;\n    events: EventsService;\n    metrics: MetricsService;\n  };\n\n  constructor(options: {\n    database: Knex;\n    logger: LoggerService;\n    refreshInterval: ProcessingIntervalFunction;\n    events: EventsService;\n    metrics: MetricsService;\n  }) {\n    this.options = options;\n    initDatabaseMetrics(options.database, options.metrics);\n  }\n\n  async updateProcessedEntity(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<{ previous: { relations: DbRelationsRow[] } }> {\n    const tx = txOpaque as Knex.Transaction;\n    const {\n      id,\n      processedEntity,\n      resultHash,\n      errors,\n      relations,\n      deferredEntities,\n      refreshKeys,\n      locationKey,\n    } = options;\n    const configClient = tx.client.config.client;\n    const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        processed_entity: JSON.stringify(processedEntity),\n        result_hash: resultHash,\n        errors,\n        location_key: locationKey,\n      })\n      .where('entity_id', id)\n      .andWhere(inner => {\n        if (!locationKey) {\n          return inner.whereNull('location_key');\n        }\n        return inner\n          .where('location_key', locationKey)\n          .orWhereNull('location_key');\n      });\n    if (refreshResult === 0) {\n      throw new ConflictError(\n        `Conflicting write of processing result for ${id} with location key '${locationKey}'`,\n      );\n    }\n    const sourceEntityRef = stringifyEntityRef(processedEntity);\n\n    // Schedule all deferred entities for future processing.\n    await this.addUnprocessedEntities(tx, {\n      entities: deferredEntities,\n      sourceEntityRef,\n    });\n\n    // Delete old relations\n    // NOTE(freben): knex implemented support for returning() on update queries for sqlite, but at the current time of writing (Sep 2022) not for delete() queries.\n    let previousRelationRows: DbRelationsRow[];\n    if (configClient.includes('sqlite3') || configClient.includes('mysql')) {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .select('*')\n        .where({ originating_entity_id: id });\n      await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete();\n    } else {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete()\n        .returning('*');\n    }\n\n    // Batch insert new relations\n    const relationRows: DbRelationsRow[] = relations.map(\n      ({ source, target, type }) => ({\n        originating_entity_id: id,\n        source_entity_ref: stringifyEntityRef(source),\n        target_entity_ref: stringifyEntityRef(target),\n        type,\n      }),\n    );\n\n    await tx.batchInsert(\n      'relations',\n      this.deduplicateRelations(relationRows),\n      BATCH_SIZE,\n    );\n\n    // Delete old refresh keys\n    await tx<DbRefreshKeysRow>('refresh_keys')\n      .where({ entity_id: id })\n      .delete();\n\n    // Insert the refresh keys for the processed entity\n    await tx.batchInsert(\n      'refresh_keys',\n      refreshKeys.map(k => ({\n        entity_id: id,\n        key: generateTargetKey(k.key),\n      })),\n      BATCH_SIZE,\n    );\n\n    return {\n      previous: {\n        relations: previousRelationRows,\n      },\n    };\n  }\n\n  async updateProcessedEntityErrors(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, errors, resultHash } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        errors,\n        result_hash: resultHash,\n      })\n      .where('entity_id', id);\n  }\n\n  async updateEntityCache(\n    txOpaque: Transaction,\n    options: UpdateEntityCacheOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, state } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({ cache: JSON.stringify(state ?? {}) })\n      .where('entity_id', id);\n  }\n\n  async getProcessableEntities(\n    maybeTx: Transaction | Knex,\n    request: { processBatchSize: number },\n  ): Promise<GetProcessableEntitiesResult> {\n    const knex = maybeTx as Knex.Transaction | Knex;\n    const useLocking = ['mysql', 'mysql2', 'pg'].includes(\n      knex.client.config.client,\n    );\n\n    const interval = this.options.refreshInterval();\n\n    const nextUpdateAt = (\n      tx: Knex | Knex.Transaction,\n      refreshInterval: number,\n    ) => {\n      if (tx.client.config.client.includes('sqlite3')) {\n        return tx.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);\n      } else if (tx.client.config.client.includes('mysql')) {\n        return tx.raw(`now() + interval ${refreshInterval} second`);\n      }\n      return tx.raw(`now() + interval '${refreshInterval} seconds'`);\n    };\n\n    // The SELECT FOR UPDATE SKIP LOCKED + UPDATE must run inside a\n    // single transaction so that the row locks persist until\n    // next_update_at has been bumped.\n    const run = async (tx: Knex | Knex.Transaction) => {\n      const items: DbRefreshStateRow[] = await tx('refresh_state')\n        .select([\n          'entity_id',\n          'entity_ref',\n          'unprocessed_entity',\n          'result_hash',\n          'cache',\n          'errors',\n          'location_key',\n          'next_update_at',\n        ])\n        .where('next_update_at', '<=', tx.fn.now())\n        .limit(request.processBatchSize)\n        .orderBy('next_update_at', 'asc')\n        .modify(qb => {\n          if (useLocking) {\n            qb.forUpdate().skipLocked();\n          }\n        });\n\n      if (items.length > 0) {\n        await tx('refresh_state')\n          .whereIn(\n            'entity_ref',\n            items.map(i => i.entity_ref),\n          )\n          .update({\n            next_update_at: nextUpdateAt(tx, interval),\n          });\n      }\n\n      return items;\n    };\n\n    const items =\n      knex.isTransaction || !useLocking\n        ? await run(knex)\n        : await knex.transaction(run);\n\n    return {\n      items: items.map(\n        i =>\n          ({\n            id: i.entity_id,\n            entityRef: i.entity_ref,\n            unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity,\n            resultHash: i.result_hash || '',\n            nextUpdateAt: timestampToDateTime(i.next_update_at),\n            state: i.cache ? JSON.parse(i.cache) : undefined,\n            errors: i.errors,\n            locationKey: i.location_key,\n          } satisfies RefreshStateItem),\n      ),\n    };\n  }\n\n  async listParents(\n    txOpaque: Transaction,\n    options: ListParentsOptions,\n  ): Promise<ListParentsResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .whereIn('target_entity_ref', options.entityRefs)\n      .select();\n\n    const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);\n\n    return { entityRefs };\n  }\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] {\n    return lodash.uniqBy(\n      rows,\n      r => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`,\n    );\n  }\n\n  /**\n   * Add a set of deferred entities for processing.\n   * The entities will be added at the front of the processing queue.\n   */\n  private async addUnprocessedEntities(\n    txOpaque: Transaction,\n    options: {\n      sourceEntityRef: string;\n      entities: DeferredEntity[];\n    },\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n\n    // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards\n    const stateReferences = new Array<string>();\n\n    // Upsert all of the unprocessed entities into the refresh_state table, by\n    // their entity ref.\n    for (const { entity, locationKey } of options.entities) {\n      const entityRef = stringifyEntityRef(entity);\n      const hash = generateStableHash(entity);\n\n      const updated = await updateUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n      });\n      if (updated) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      const inserted = await insertUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n        logger: this.options.logger,\n      });\n      if (inserted) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      // If the row can't be inserted, we have a conflict, but it could be either\n      // because of a conflicting locationKey or a race with another instance, so check\n      // whether the conflicting entity has the same entityRef but a different locationKey\n      const conflictingKey = await checkLocationKeyConflict({\n        tx,\n        entityRef,\n        locationKey,\n      });\n      if (conflictingKey) {\n        this.options.logger.warn(\n          `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n        );\n        if (locationKey) {\n          const eventParams: EventParams<CatalogConflictEventPayload> = {\n            topic: CATALOG_CONFLICTS_TOPIC,\n            eventPayload: {\n              unprocessedEntity: entity,\n              entityRef,\n              newLocationKey: locationKey,\n              existingLocationKey: conflictingKey,\n              lastConflictAt: DateTime.now().toISO()!,\n            },\n          };\n          await this.options.events.publish(eventParams);\n        }\n      }\n    }\n\n    // Lastly, replace refresh state references for the originating entity and any successfully added entities\n    await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n      // Remove all existing references from the originating entity\n      .where({ source_entity_ref: options.sourceEntityRef })\n      // And remove any existing references to entities that we're inserting new references for\n      .orWhereIn('target_entity_ref', stateReferences)\n      .delete();\n    await tx.batchInsert(\n      'refresh_state_references',\n      stateReferences.map(entityRef => ({\n        source_entity_ref: options.sourceEntityRef,\n        target_entity_ref: entityRef,\n      })),\n      BATCH_SIZE,\n    );\n  }\n}\n"],"names":["initDatabaseMetrics","errors","ConflictError","stringifyEntityRef","generateTargetKey","items","timestampToDateTime","rethrowError","lodash","generateStableHash","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","CATALOG_CONFLICTS_TOPIC","DateTime"],"mappings":";;;;;;;;;;;;;;;;;;AAuDA,MAAM,UAAA,GAAa,EAAA;AAEZ,MAAM,yBAAA,CAAwD;AAAA,EAClD,OAAA;AAAA,EAQjB,YAAY,OAAA,EAMT;AACD,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA;AACf,IAAAA,2BAAA,CAAoB,OAAA,CAAQ,QAAA,EAAU,OAAA,CAAQ,OAAO,CAAA;AAAA,EACvD;AAAA,EAEA,MAAM,qBAAA,CACJ,QAAA,EACA,OAAA,EACwD;AACxD,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM;AAAA,MACJ,EAAA;AAAA,MACA,eAAA;AAAA,MACA,UAAA;AAAA,cACAC,QAAA;AAAA,MACA,SAAA;AAAA,MACA,gBAAA;AAAA,MACA,WAAA;AAAA,MACA;AAAA,KACF,GAAI,OAAA;AACJ,IAAA,MAAM,YAAA,GAAe,EAAA,CAAG,MAAA,CAAO,MAAA,CAAO,MAAA;AACtC,IAAA,MAAM,aAAA,GAAgB,MAAM,EAAA,CAAsB,eAAe,EAC9D,MAAA,CAAO;AAAA,MACN,gBAAA,EAAkB,IAAA,CAAK,SAAA,CAAU,eAAe,CAAA;AAAA,MAChD,WAAA,EAAa,UAAA;AAAA,cACbA,QAAA;AAAA,MACA,YAAA,EAAc;AAAA,KACf,CAAA,CACA,KAAA,CAAM,aAAa,EAAE,CAAA,CACrB,SAAS,CAAA,KAAA,KAAS;AACjB,MAAA,IAAI,CAAC,WAAA,EAAa;AAChB,QAAA,OAAO,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA,MACvC;AACA,MAAA,OAAO,MACJ,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,IAC/B,CAAC,CAAA;AACH,IAAA,IAAI,kBAAkB,CAAA,EAAG;AACvB,MAAA,MAAM,IAAIC,oBAAA;AAAA,QACR,CAAA,2CAAA,EAA8C,EAAE,CAAA,oBAAA,EAAuB,WAAW,CAAA,CAAA;AAAA,OACpF;AAAA,IACF;AACA,IAAA,MAAM,eAAA,GAAkBC,gCAAmB,eAAe,CAAA;AAG1D,IAAA,MAAM,IAAA,CAAK,uBAAuB,EAAA,EAAI;AAAA,MACpC,QAAA,EAAU,gBAAA;AAAA,MACV;AAAA,KACD,CAAA;AAID,IAAA,IAAI,oBAAA;AACJ,IAAA,IAAI,aAAa,QAAA,CAAS,SAAS,KAAK,YAAA,CAAa,QAAA,CAAS,OAAO,CAAA,EAAG;AACtE,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,MAAA,CAAO,GAAG,CAAA,CACV,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA;AACtC,MAAA,MAAM,EAAA,CAAmB,WAAW,CAAA,CACjC,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO;AAAA,IACZ,CAAA,MAAO;AACL,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO,CACP,UAAU,GAAG,CAAA;AAAA,IAClB;AAGA,IAAA,MAAM,eAAiC,SAAA,CAAU,GAAA;AAAA,MAC/C,CAAC,EAAE,MAAA,EAAQ,MAAA,EAAQ,MAAK,MAAO;AAAA,QAC7B,qBAAA,EAAuB,EAAA;AAAA,QACvB,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C;AAAA,OACF;AAAA,KACF;AAEA,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,WAAA;AAAA,MACA,IAAA,CAAK,qBAAqB,YAAY,CAAA;AAAA,MACtC;AAAA,KACF;AAGA,IAAA,MAAM,EAAA,CAAqB,cAAc,CAAA,CACtC,KAAA,CAAM,EAAE,SAAA,EAAW,EAAA,EAAI,CAAA,CACvB,MAAA,EAAO;AAGV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,cAAA;AAAA,MACA,WAAA,CAAY,IAAI,CAAA,CAAA,MAAM;AAAA,QACpB,SAAA,EAAW,EAAA;AAAA,QACX,GAAA,EAAKC,sBAAA,CAAkB,CAAA,CAAE,GAAG;AAAA,OAC9B,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAEA,IAAA,OAAO;AAAA,MACL,QAAA,EAAU;AAAA,QACR,SAAA,EAAW;AAAA;AACb,KACF;AAAA,EACF;AAAA,EAEA,MAAM,2BAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,MAAA,EAAQ,UAAA,EAAW,GAAI,OAAA;AAEnC,IAAA,MAAM,EAAA,CAAsB,eAAe,CAAA,CACxC,MAAA,CAAO;AAAA,MACN,MAAA;AAAA,MACA,WAAA,EAAa;AAAA,KACd,CAAA,CACA,KAAA,CAAM,WAAA,EAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,iBAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,KAAA,EAAM,GAAI,OAAA;AAEtB,IAAA,MAAM,GAAsB,eAAe,CAAA,CACxC,MAAA,CAAO,EAAE,OAAO,IAAA,CAAK,SAAA,CAAU,KAAA,IAAS,EAAE,CAAA,EAAG,CAAA,CAC7C,KAAA,CAAM,aAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,sBAAA,CACJ,OAAA,EACA,OAAA,EACuC;AACvC,IAAA,MAAM,IAAA,GAAO,OAAA;AACb,IAAA,MAAM,UAAA,GAAa,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,QAAA;AAAA,MAC3C,IAAA,CAAK,OAAO,MAAA,CAAO;AAAA,KACrB;AAEA,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,OAAA,CAAQ,eAAA,EAAgB;AAE9C,IAAA,MAAM,YAAA,GAAe,CACnB,EAAA,EACA,eAAA,KACG;AACH,MAAA,IAAI,GAAG,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AAC/C,QAAA,OAAO,GAAG,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,eAAe,UAAU,CAAC,CAAA;AAAA,MACpE,WAAW,EAAA,CAAG,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACpD,QAAA,OAAO,EAAA,CAAG,GAAA,CAAI,CAAA,iBAAA,EAAoB,eAAe,CAAA,OAAA,CAAS,CAAA;AAAA,MAC5D;AACA,MAAA,OAAO,EAAA,CAAG,GAAA,CAAI,CAAA,kBAAA,EAAqB,eAAe,CAAA,SAAA,CAAW,CAAA;AAAA,IAC/D,CAAA;AAKA,IAAA,MAAM,GAAA,GAAM,OAAO,EAAA,KAAgC;AACjD,MAAA,MAAMC,MAAAA,GAA6B,MAAM,EAAA,CAAG,eAAe,EACxD,MAAA,CAAO;AAAA,QACN,WAAA;AAAA,QACA,YAAA;AAAA,QACA,oBAAA;AAAA,QACA,aAAA;AAAA,QACA,OAAA;AAAA,QACA,QAAA;AAAA,QACA,cAAA;AAAA,QACA;AAAA,OACD,CAAA,CACA,KAAA,CAAM,kBAAkB,IAAA,EAAM,EAAA,CAAG,GAAG,GAAA,EAAK,EACzC,KAAA,CAAM,OAAA,CAAQ,gBAAgB,CAAA,CAC9B,OAAA,CAAQ,kBAAkB,KAAK,CAAA,CAC/B,OAAO,CAAA,EAAA,KAAM;AACZ,QAAA,IAAI,UAAA,EAAY;AACd,UAAA,EAAA,CAAG,SAAA,GAAY,UAAA,EAAW;AAAA,QAC5B;AAAA,MACF,CAAC,CAAA;AAEH,MAAA,IAAIA,MAAAA,CAAM,SAAS,CAAA,EAAG;AACpB,QAAA,MAAM,EAAA,CAAG,eAAe,CAAA,CACrB,OAAA;AAAA,UACC,YAAA;AAAA,UACAA,MAAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,UAE5B,MAAA,CAAO;AAAA,UACN,cAAA,EAAgB,YAAA,CAAa,EAAA,EAAI,QAAQ;AAAA,SAC1C,CAAA;AAAA,MACL;AAEA,MAAA,OAAOA,MAAAA;AAAA,IACT,CAAA;AAEA,IAAA,MAAM,KAAA,GACJ,IAAA,CAAK,aAAA,IAAiB,CAAC,UAAA,GACnB,MAAM,GAAA,CAAI,IAAI,CAAA,GACd,MAAM,IAAA,CAAK,WAAA,CAAY,GAAG,CAAA;AAEhC,IAAA,OAAO;AAAA,MACL,OAAO,KAAA,CAAM,GAAA;AAAA,QACX,CAAA,CAAA,MACG;AAAA,UACC,IAAI,CAAA,CAAE,SAAA;AAAA,UACN,WAAW,CAAA,CAAE,UAAA;AAAA,UACb,iBAAA,EAAmB,IAAA,CAAK,KAAA,CAAM,CAAA,CAAE,kBAAkB,CAAA;AAAA,UAClD,UAAA,EAAY,EAAE,WAAA,IAAe,EAAA;AAAA,UAC7B,YAAA,EAAcC,8BAAA,CAAoB,CAAA,CAAE,cAAc,CAAA;AAAA,UAClD,OAAO,CAAA,CAAE,KAAA,GAAQ,KAAK,KAAA,CAAM,CAAA,CAAE,KAAK,CAAA,GAAI,MAAA;AAAA,UACvC,QAAQ,CAAA,CAAE,MAAA;AAAA,UACV,aAAa,CAAA,CAAE;AAAA,SACjB;AAAA;AACJ,KACF;AAAA,EACF;AAAA,EAEA,MAAM,WAAA,CACJ,QAAA,EACA,OAAA,EAC4B;AAC5B,IAAA,MAAM,EAAA,GAAK,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,MAEC,OAAA,CAAQ,mBAAA,EAAqB,OAAA,CAAQ,UAAU,EAC/C,MAAA,EAAO;AAEV,IAAA,MAAM,UAAA,GAAa,KAAK,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,iBAAkB,CAAA,CAAE,OAAO,OAAO,CAAA;AAErE,IAAA,OAAO,EAAE,UAAA,EAAW;AAAA,EACtB;AAAA,EAEA,MAAM,YAAe,EAAA,EAAiD;AACpE,IAAA,IAAI;AACF,MAAA,IAAI,MAAA,GAAwB,KAAA,CAAA;AAE5B,MAAA,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,WAAA;AAAA,QAC1B,OAAM,EAAA,KAAM;AAGV,UAAA,MAAA,GAAS,MAAM,GAAG,EAAE,CAAA;AAAA,QACtB,CAAA;AAAA,QACA;AAAA;AAAA,UAEE,qBAAA,EAAuB;AAAA;AACzB,OACF;AAEA,MAAA,OAAO,MAAA;AAAA,IACT,SAAS,CAAA,EAAG;AACV,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,0BAAA,EAA6B,CAAC,CAAA,CAAE,CAAA;AAC1D,MAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA,IACtB;AAAA,EACF;AAAA,EAEQ,qBAAqB,IAAA,EAA0C;AACrE,IAAA,OAAOC,uBAAA,CAAO,MAAA;AAAA,MACZ,IAAA;AAAA,MACA,CAAA,CAAA,KAAK,GAAG,CAAA,CAAE,iBAAiB,IAAI,CAAA,CAAE,iBAAiB,CAAA,CAAA,EAAI,CAAA,CAAE,IAAI,CAAA;AAAA,KAC9D;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,sBAAA,CACZ,QAAA,EACA,OAAA,EAIe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AAGX,IAAA,MAAM,eAAA,GAAkB,IAAI,KAAA,EAAc;AAI1C,IAAA,KAAA,MAAW,EAAE,MAAA,EAAQ,WAAA,EAAY,IAAK,QAAQ,QAAA,EAAU;AACtD,MAAA,MAAM,SAAA,GAAYL,gCAAmB,MAAM,CAAA;AAC3C,MAAA,MAAM,IAAA,GAAOM,wBAAmB,MAAM,CAAA;AAEtC,MAAA,MAAM,OAAA,GAAU,MAAMC,+CAAA,CAAwB;AAAA,QAC5C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,OAAA,EAAS;AACX,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,QAAA,GAAW,MAAMC,+CAAA,CAAwB;AAAA,QAC7C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA,WAAA;AAAA,QACA,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAKA,MAAA,MAAM,cAAA,GAAiB,MAAMC,iDAAA,CAAyB;AAAA,QACpD,EAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,cAAA,EAAgB;AAClB,QAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,UAClB,CAAA,+BAAA,EAAkC,SAAS,CAAA,uBAAA,EAA0B,cAAc,iBAAiB,WAAW,CAAA;AAAA,SACjH;AACA,QAAA,IAAI,WAAA,EAAa;AACf,UAAA,MAAM,WAAA,GAAwD;AAAA,YAC5D,KAAA,EAAOC,iCAAA;AAAA,YACP,YAAA,EAAc;AAAA,cACZ,iBAAA,EAAmB,MAAA;AAAA,cACnB,SAAA;AAAA,cACA,cAAA,EAAgB,WAAA;AAAA,cAChB,mBAAA,EAAqB,cAAA;AAAA,cACrB,cAAA,EAAgBC,cAAA,CAAS,GAAA,EAAI,CAAE,KAAA;AAAM;AACvC,WACF;AACA,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,OAAA,CAAQ,WAAW,CAAA;AAAA,QAC/C;AAAA,MACF;AAAA,IACF;AAGA,IAAA,MAAM,EAAA,CAAgC,0BAA0B,CAAA,CAE7D,KAAA,CAAM,EAAE,iBAAA,EAAmB,OAAA,CAAQ,eAAA,EAAiB,CAAA,CAEpD,SAAA,CAAU,mBAAA,EAAqB,eAAe,EAC9C,MAAA,EAAO;AACV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,0BAAA;AAAA,MACA,eAAA,CAAgB,IAAI,CAAA,SAAA,MAAc;AAAA,QAChC,mBAAmB,OAAA,CAAQ,eAAA;AAAA,QAC3B,iBAAA,EAAmB;AAAA,OACrB,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAAA,EACF;AACF;;;;"}

package/dist/database/operations/provider/deleteWithEagerPruningOfChildren.cjs.js

@@ -1,6 +1,7 @@
 'use strict';
 
 var lodash = require('lodash');
+var markForStitching = require('../stitcher/markForStitching.cjs.js');
 
 function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e : { default: e }; }
 
@@ -21,6 +22,7 @@ async function deleteWithEagerPruningOfChildren(options) {
         entityRefs: refsToDelete
       });
       await knex.delete().from("refresh_state").whereIn("entity_ref", refsToDelete);
+      await knex.delete().from("stitch_queue").whereIn("entity_ref", refsToDelete);
     }
     await knex("refresh_state_references").where("source_key", "=", sourceKey).whereIn("target_entity_ref", refs).delete();
     removedCount += orphanEntityRefs.length;
@@ -103,20 +105,15 @@ async function findDescendantsThatWouldHaveBeenOrphanedByDeletion(options) {
 }
 async function markEntitiesAffectedByDeletionForStitching(options) {
   const { knex, entityRefs } = options;
-  const affectedIds = await knex.select("refresh_state.entity_id AS entity_id").from("relations").join(
+  const affectedIds = await knex.distinct("refresh_state.entity_id AS entity_id").from("relations").join(
     "refresh_state",
     "relations.source_entity_ref",
     "refresh_state.entity_ref"
   ).whereIn("relations.target_entity_ref", entityRefs).then((rows) => rows.map((row) => row.entity_id));
-  for (const ids of lodash__default.default.chunk(affectedIds, 1e3)) {
-    await knex.table("final_entities").update({
-      hash: "force-stitching"
-    }).whereIn("entity_id", ids);
-    await knex.table("refresh_state").update({
-      result_hash: "force-stitching",
-      next_update_at: knex.fn.now()
-    }).whereIn("entity_id", ids);
-  }
+  await markForStitching.markForStitching({
+    knex,
+    entityIds: affectedIds
+  });
 }
 
 exports.deleteWithEagerPruningOfChildren = deleteWithEagerPruningOfChildren;

package/dist/database/operations/provider/deleteWithEagerPruningOfChildren.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"deleteWithEagerPruningOfChildren.cjs.js","sources":["../../../../src/database/operations/provider/deleteWithEagerPruningOfChildren.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport {\n  DbFinalEntitiesRow,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n} from '../../tables';\n\n/**\n * Given a number of entity refs originally created by a given entity provider\n * (source key), remove those entities from the refresh state, and at the same\n * time recursively remove every child that is a direct or indirect result of\n * processing those entities, if they would have otherwise become orphaned by\n * the removal of their parents.\n */\nexport async function deleteWithEagerPruningOfChildren(options: {\n  knex: Knex | Knex.Transaction;\n  entityRefs: string[];\n  sourceKey: string;\n}): Promise<number> {\n  const { knex, entityRefs, sourceKey } = options;\n\n  // Split up the operation by (large) chunks, so that we do not hit database\n  // limits for the number of permitted bindings on a precompiled statement\n  let removedCount = 0;\n  for (const refs of lodash.chunk(entityRefs, 1000)) {\n    const { orphanEntityRefs } =\n      await findDescendantsThatWouldHaveBeenOrphanedByDeletion({\n        knex: options.knex,\n        refs,\n        sourceKey,\n      });\n\n    // Chunk again - these can be many more than the outer chunk size\n    for (const refsToDelete of lodash.chunk(orphanEntityRefs, 1000)) {\n      await markEntitiesAffectedByDeletionForStitching({\n        knex: options.knex,\n        entityRefs: refsToDelete,\n      });\n      await knex\n        .delete()\n        .from('refresh_state')\n        .whereIn('entity_ref', refsToDelete);\n    }\n\n    // Delete the references that originate only from this entity provider. Note\n    // that there may be more than one entity provider making a \"claim\" for a\n    // given root entity, if they emit with the same location key.\n    await knex<DbRefreshStateReferencesRow>('refresh_state_references')\n      .where('source_key', '=', sourceKey)\n      .whereIn('target_entity_ref', refs)\n      .delete();\n\n    removedCount += orphanEntityRefs.length;\n  }\n\n  return removedCount;\n}\n\nasync function findDescendantsThatWouldHaveBeenOrphanedByDeletion(options: {\n  knex: Knex | Knex.Transaction;\n  refs: string[];\n  sourceKey: string;\n}): Promise<{ orphanEntityRefs: string[] }> {\n  const { knex, refs, sourceKey } = options;\n\n  const orphans: string[] =\n    // First find all nodes that can be reached downwards from the roots\n    // (deletion targets), including the roots themselves, by traversing\n    // down the refresh_state_references table. Note that this query\n    // starts with a condition that source_key = our source key, and\n    // target_entity_ref is one of the deletion targets. This has two\n    // effects: it won't match attempts at deleting something that didn't\n    // originate from us in the first place, and also won't match non-root\n    // entities (source_key would be null for those).\n    //\n    //   KeyA - R1 - R2        Legend:\n    //                 \\       -----------------------------------------\n    //                  R3     Key*    Source key\n    //                 /       R*      Entity ref\n    //   KeyA - R4 - R5        lines   Individual references; sources to\n    //              /                  the left and targets to the right\n    //   KeyB --- R6\n    //\n    // The scenario is that KeyA wants to delete R1.\n    //\n    // The query starts with the KeyA-R1 reference, and then traverses\n    // down to also find R2 and R3. It uses union instead of union all,\n    // because it wants to find the set of unique descendants even if\n    // the tree has unexpected loops etc.\n    await knex\n      .withRecursive('descendants', ['entity_ref'], initial =>\n        initial\n          .select('target_entity_ref')\n          .from('refresh_state_references')\n          .where('source_key', '=', sourceKey)\n          .whereIn('target_entity_ref', refs)\n          .union(recursive =>\n            recursive\n              .select('refresh_state_references.target_entity_ref')\n              .from('descendants')\n              .join(\n                'refresh_state_references',\n                'descendants.entity_ref',\n                'refresh_state_references.source_entity_ref',\n              ),\n          ),\n      )\n      // Then for each descendant, traverse all the way back upwards through\n      // the refresh_state_references table to get an exhaustive list of all\n      // references that are part of keeping that particular descendant\n      // alive.\n      //\n      // Continuing the scenario from above, starting from R3, it goes\n      // upwards to find every pair along every relation line.\n      //\n      //   Top branch:     R2-R3, R1-R2, KeyA-R1\n      //   Middle branch:  R5-R3, R4-R5, KeyA-R4\n      //   Bottom branch:  R6-R5, KeyB-R6\n      //\n      // Note that this all applied to the subject R3. The exact same thing\n      // will be done starting from each other descendant (R2 and R1). They\n      // only have one and two references to find, respectively.\n      //\n      // This query also uses union instead of union all, to get the set of\n      // distinct relations even if the tree has unexpected loops etc.\n      .withRecursive(\n        'ancestors',\n        ['source_key', 'source_entity_ref', 'target_entity_ref', 'subject'],\n        initial =>\n          initial\n            .select(\n              'refresh_state_references.source_key',\n              'refresh_state_references.source_entity_ref',\n              'refresh_state_references.target_entity_ref',\n              'descendants.entity_ref',\n            )\n            .from('descendants')\n            .join(\n              'refresh_state_references',\n              'refresh_state_references.target_entity_ref',\n              'descendants.entity_ref',\n            )\n            .union(recursive =>\n              recursive\n                .select(\n                  'refresh_state_references.source_key',\n                  'refresh_state_references.source_entity_ref',\n                  'refresh_state_references.target_entity_ref',\n                  'ancestors.subject',\n                )\n                .from('ancestors')\n                .join(\n                  'refresh_state_references',\n                  'refresh_state_references.target_entity_ref',\n                  'ancestors.source_entity_ref',\n                ),\n            ),\n      )\n      // Finally, from that list of ancestor relations per descendant, pick\n      // out the ones that are roots (have a source_key). Specifically, find\n      // ones that seem to be be either (1) from another source, or (2)\n      // aren't part of the deletion targets. Those are markers that tell us\n      // that the corresponding descendant should be kept alive and NOT\n      // subject to eager deletion, because there's \"something else\" (not\n      // targeted for deletion) that has references down through the tree to\n      // it.\n      //\n      // Continuing the scenario from above, for R3 we have\n      //\n      //   KeyA-R1, KeyA-R4, KeyB-R6\n      //\n      // This tells us that R3 should be kept alive for two reasons: it's\n      // referenced by a node that isn't being deleted (R4), and also by\n      // another source (KeyB). What about R1 and R2? They both have\n      //\n      //   KeyA-R1\n      //\n      // So those should be deleted, since they are definitely only being\n      // kept alive by something that's about to be deleted.\n      //\n      // Final shape of the tree:\n      //\n      //                  R3\n      //                 /\n      //   KeyA - R4 - R5\n      //              /\n      //   KeyB --- R6\n      .with('retained', ['entity_ref'], notPartOfDeletion =>\n        notPartOfDeletion\n          .select('subject')\n          .from('ancestors')\n          .whereNotNull('ancestors.source_key')\n          .where(foreignKeyOrRef =>\n            foreignKeyOrRef\n              .where('ancestors.source_key', '!=', sourceKey)\n              .orWhereNotIn('ancestors.target_entity_ref', refs),\n          ),\n      )\n      // Return all descendants minus the retained ones\n      .select('descendants.entity_ref AS entity_ref')\n      .from('descendants')\n      .leftOuterJoin(\n        'retained',\n        'retained.entity_ref',\n        'descendants.entity_ref',\n      )\n      .whereNull('retained.entity_ref')\n      .then(rows => rows.map(row => row.entity_ref));\n\n  return { orphanEntityRefs: orphans };\n}\n\nasync function markEntitiesAffectedByDeletionForStitching(options: {\n  knex: Knex | Knex.Transaction;\n  entityRefs: string[];\n}) {\n  const { knex, entityRefs } = options;\n\n  // We want to re-stitch anything that has a relation pointing to the\n  // soon-to-be-deleted entity. In many circumstances we also re-stitch children\n  // in the refresh_state_references graph because their orphan state might\n  // change, but not here - this code by its very definition is meant to not\n  // leave any orphans behind, so we can simplify away that.\n  const affectedIds = await knex\n    .select('refresh_state.entity_id AS entity_id')\n    .from('relations')\n    .join(\n      'refresh_state',\n      'relations.source_entity_ref',\n      'refresh_state.entity_ref',\n    )\n    .whereIn('relations.target_entity_ref', entityRefs)\n    .then(rows => rows.map(row => row.entity_id));\n\n  for (const ids of lodash.chunk(affectedIds, 1000)) {\n    await knex\n      .table<DbFinalEntitiesRow>('final_entities')\n      .update({\n        hash: 'force-stitching',\n      })\n      .whereIn('entity_id', ids);\n    await knex\n      .table<DbRefreshStateRow>('refresh_state')\n      .update({\n        result_hash: 'force-stitching',\n        next_update_at: knex.fn.now(),\n      })\n      .whereIn('entity_id', ids);\n  }\n}\n"],"names":["lodash"],"mappings":";;;;;;;;AA+BA,eAAsB,iCAAiC,OAAA,EAInC;AAClB,EAAA,MAAM,EAAE,IAAA,EAAM,UAAA,EAAY,SAAA,EAAU,GAAI,OAAA;AAIxC,EAAA,IAAI,YAAA,GAAe,CAAA;AACnB,EAAA,KAAA,MAAW,IAAA,IAAQA,uBAAA,CAAO,KAAA,CAAM,UAAA,EAAY,GAAI,CAAA,EAAG;AACjD,IAAA,MAAM,EAAE,gBAAA,EAAiB,GACvB,MAAM,kDAAA,CAAmD;AAAA,MACvD,MAAM,OAAA,CAAQ,IAAA;AAAA,MACd,IAAA;AAAA,MACA;AAAA,KACD,CAAA;AAGH,IAAA,KAAA,MAAW,YAAA,IAAgBA,uBAAA,CAAO,KAAA,CAAM,gBAAA,EAAkB,GAAI,CAAA,EAAG;AAC/D,MAAA,MAAM,0CAAA,CAA2C;AAAA,QAC/C,MAAM,OAAA,CAAQ,IAAA;AAAA,QACd,UAAA,EAAY;AAAA,OACb,CAAA;AACD,MAAA,MAAM,IAAA,CACH,QAAO,CACP,IAAA,CAAK,eAAe,CAAA,CACpB,OAAA,CAAQ,cAAc,YAAY,CAAA;AAAA,IACvC;AAKA,IAAA,MAAM,IAAA,CAAkC,0BAA0B,CAAA,CAC/D,KAAA,CAAM,YAAA,EAAc,GAAA,EAAK,SAAS,CAAA,CAClC,OAAA,CAAQ,mBAAA,EAAqB,IAAI,CAAA,CACjC,MAAA,EAAO;AAEV,IAAA,YAAA,IAAgB,gBAAA,CAAiB,MAAA;AAAA,EACnC;AAEA,EAAA,OAAO,YAAA;AACT;AAEA,eAAe,mDAAmD,OAAA,EAItB;AAC1C,EAAA,MAAM,EAAE,IAAA,EAAM,IAAA,EAAM,SAAA,EAAU,GAAI,OAAA;AAElC,EAAA,MAAM,OAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAwBJ,MAAM,IAAA,CACH,aAAA;AAAA,MAAc,aAAA;AAAA,MAAe,CAAC,YAAY,CAAA;AAAA,MAAG,aAC5C,OAAA,CACG,MAAA,CAAO,mBAAmB,CAAA,CAC1B,KAAK,0BAA0B,CAAA,CAC/B,KAAA,CAAM,YAAA,EAAc,KAAK,SAAS,CAAA,CAClC,OAAA,CAAQ,mBAAA,EAAqB,IAAI,CAAA,CACjC,KAAA;AAAA,QAAM,eACL,SAAA,CACG,MAAA,CAAO,4CAA4C,CAAA,CACnD,IAAA,CAAK,aAAa,CAAA,CAClB,IAAA;AAAA,UACC,0BAAA;AAAA,UACA,wBAAA;AAAA,UACA;AAAA;AACF;AACJ,KACJ,CAmBC,aAAA;AAAA,MACC,WAAA;AAAA,MACA,CAAC,YAAA,EAAc,mBAAA,EAAqB,mBAAA,EAAqB,SAAS,CAAA;AAAA,MAClE,aACE,OAAA,CACG,MAAA;AAAA,QACC,qCAAA;AAAA,QACA,4CAAA;AAAA,QACA,4CAAA;AAAA,QACA;AAAA,OACF,CACC,IAAA,CAAK,aAAa,CAAA,CAClB,IAAA;AAAA,QACC,0BAAA;AAAA,QACA,4CAAA;AAAA,QACA;AAAA,OACF,CACC,KAAA;AAAA,QAAM,eACL,SAAA,CACG,MAAA;AAAA,UACC,qCAAA;AAAA,UACA,4CAAA;AAAA,UACA,4CAAA;AAAA,UACA;AAAA,SACF,CACC,IAAA,CAAK,WAAW,CAAA,CAChB,IAAA;AAAA,UACC,0BAAA;AAAA,UACA,4CAAA;AAAA,UACA;AAAA;AACF;AACJ,KACN,CA8BC,IAAA;AAAA,MAAK,UAAA;AAAA,MAAY,CAAC,YAAY,CAAA;AAAA,MAAG,CAAA,iBAAA,KAChC,iBAAA,CACG,MAAA,CAAO,SAAS,CAAA,CAChB,KAAK,WAAW,CAAA,CAChB,YAAA,CAAa,sBAAsB,CAAA,CACnC,KAAA;AAAA,QAAM,CAAA,eAAA,KACL,gBACG,KAAA,CAAM,sBAAA,EAAwB,MAAM,SAAS,CAAA,CAC7C,YAAA,CAAa,6BAAA,EAA+B,IAAI;AAAA;AACrD,MAGH,MAAA,CAAO,sCAAsC,CAAA,CAC7C,IAAA,CAAK,aAAa,CAAA,CAClB,aAAA;AAAA,MACC,UAAA;AAAA,MACA,qBAAA;AAAA,MACA;AAAA,KACF,CACC,SAAA,CAAU,qBAAqB,CAAA,CAC/B,IAAA,CAAK,CAAA,IAAA,KAAQ,IAAA,CAAK,GAAA,CAAI,CAAA,GAAA,KAAO,GAAA,CAAI,UAAU,CAAC;AAAA,GAAA;AAEjD,EAAA,OAAO,EAAE,kBAAkB,OAAA,EAAQ;AACrC;AAEA,eAAe,2CAA2C,OAAA,EAGvD;AACD,EAAA,MAAM,EAAE,IAAA,EAAM,UAAA,EAAW,GAAI,OAAA;AAO7B,EAAA,MAAM,WAAA,GAAc,MAAM,IAAA,CACvB,MAAA,CAAO,sCAAsC,CAAA,CAC7C,IAAA,CAAK,WAAW,CAAA,CAChB,IAAA;AAAA,IACC,eAAA;AAAA,IACA,6BAAA;AAAA,IACA;AAAA,GACF,CACC,OAAA,CAAQ,6BAAA,EAA+B,UAAU,CAAA,CACjD,IAAA,CAAK,CAAA,IAAA,KAAQ,IAAA,CAAK,GAAA,CAAI,CAAA,GAAA,KAAO,GAAA,CAAI,SAAS,CAAC,CAAA;AAE9C,EAAA,KAAA,MAAW,GAAA,IAAOA,uBAAA,CAAO,KAAA,CAAM,WAAA,EAAa,GAAI,CAAA,EAAG;AACjD,IAAA,MAAM,IAAA,CACH,KAAA,CAA0B,gBAAgB,CAAA,CAC1C,MAAA,CAAO;AAAA,MACN,IAAA,EAAM;AAAA,KACP,CAAA,CACA,OAAA,CAAQ,WAAA,EAAa,GAAG,CAAA;AAC3B,IAAA,MAAM,IAAA,CACH,KAAA,CAAyB,eAAe,CAAA,CACxC,MAAA,CAAO;AAAA,MACN,WAAA,EAAa,iBAAA;AAAA,MACb,cAAA,EAAgB,IAAA,CAAK,EAAA,CAAG,GAAA;AAAI,KAC7B,CAAA,CACA,OAAA,CAAQ,WAAA,EAAa,GAAG,CAAA;AAAA,EAC7B;AACF;;;;"}
+{"version":3,"file":"deleteWithEagerPruningOfChildren.cjs.js","sources":["../../../../src/database/operations/provider/deleteWithEagerPruningOfChildren.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { DbRefreshStateReferencesRow } from '../../tables';\nimport { markForStitching } from '../stitcher/markForStitching';\n\n/**\n * Given a number of entity refs originally created by a given entity provider\n * (source key), remove those entities from the refresh state, and at the same\n * time recursively remove every child that is a direct or indirect result of\n * processing those entities, if they would have otherwise become orphaned by\n * the removal of their parents.\n */\nexport async function deleteWithEagerPruningOfChildren(options: {\n  knex: Knex | Knex.Transaction;\n  entityRefs: string[];\n  sourceKey: string;\n}): Promise<number> {\n  const { knex, entityRefs, sourceKey } = options;\n\n  // Split up the operation by (large) chunks, so that we do not hit database\n  // limits for the number of permitted bindings on a precompiled statement\n  let removedCount = 0;\n  for (const refs of lodash.chunk(entityRefs, 1000)) {\n    const { orphanEntityRefs } =\n      await findDescendantsThatWouldHaveBeenOrphanedByDeletion({\n        knex: options.knex,\n        refs,\n        sourceKey,\n      });\n\n    // Chunk again - these can be many more than the outer chunk size\n    for (const refsToDelete of lodash.chunk(orphanEntityRefs, 1000)) {\n      await markEntitiesAffectedByDeletionForStitching({\n        knex: options.knex,\n        entityRefs: refsToDelete,\n      });\n      await knex\n        .delete()\n        .from('refresh_state')\n        .whereIn('entity_ref', refsToDelete);\n      await knex\n        .delete()\n        .from('stitch_queue')\n        .whereIn('entity_ref', refsToDelete);\n    }\n\n    // Delete the references that originate only from this entity provider. Note\n    // that there may be more than one entity provider making a \"claim\" for a\n    // given root entity, if they emit with the same location key.\n    await knex<DbRefreshStateReferencesRow>('refresh_state_references')\n      .where('source_key', '=', sourceKey)\n      .whereIn('target_entity_ref', refs)\n      .delete();\n\n    removedCount += orphanEntityRefs.length;\n  }\n\n  return removedCount;\n}\n\nasync function findDescendantsThatWouldHaveBeenOrphanedByDeletion(options: {\n  knex: Knex | Knex.Transaction;\n  refs: string[];\n  sourceKey: string;\n}): Promise<{ orphanEntityRefs: string[] }> {\n  const { knex, refs, sourceKey } = options;\n\n  const orphans: string[] =\n    // First find all nodes that can be reached downwards from the roots\n    // (deletion targets), including the roots themselves, by traversing\n    // down the refresh_state_references table. Note that this query\n    // starts with a condition that source_key = our source key, and\n    // target_entity_ref is one of the deletion targets. This has two\n    // effects: it won't match attempts at deleting something that didn't\n    // originate from us in the first place, and also won't match non-root\n    // entities (source_key would be null for those).\n    //\n    //   KeyA - R1 - R2        Legend:\n    //                 \\       -----------------------------------------\n    //                  R3     Key*    Source key\n    //                 /       R*      Entity ref\n    //   KeyA - R4 - R5        lines   Individual references; sources to\n    //              /                  the left and targets to the right\n    //   KeyB --- R6\n    //\n    // The scenario is that KeyA wants to delete R1.\n    //\n    // The query starts with the KeyA-R1 reference, and then traverses\n    // down to also find R2 and R3. It uses union instead of union all,\n    // because it wants to find the set of unique descendants even if\n    // the tree has unexpected loops etc.\n    await knex\n      .withRecursive('descendants', ['entity_ref'], initial =>\n        initial\n          .select('target_entity_ref')\n          .from('refresh_state_references')\n          .where('source_key', '=', sourceKey)\n          .whereIn('target_entity_ref', refs)\n          .union(recursive =>\n            recursive\n              .select('refresh_state_references.target_entity_ref')\n              .from('descendants')\n              .join(\n                'refresh_state_references',\n                'descendants.entity_ref',\n                'refresh_state_references.source_entity_ref',\n              ),\n          ),\n      )\n      // Then for each descendant, traverse all the way back upwards through\n      // the refresh_state_references table to get an exhaustive list of all\n      // references that are part of keeping that particular descendant\n      // alive.\n      //\n      // Continuing the scenario from above, starting from R3, it goes\n      // upwards to find every pair along every relation line.\n      //\n      //   Top branch:     R2-R3, R1-R2, KeyA-R1\n      //   Middle branch:  R5-R3, R4-R5, KeyA-R4\n      //   Bottom branch:  R6-R5, KeyB-R6\n      //\n      // Note that this all applied to the subject R3. The exact same thing\n      // will be done starting from each other descendant (R2 and R1). They\n      // only have one and two references to find, respectively.\n      //\n      // This query also uses union instead of union all, to get the set of\n      // distinct relations even if the tree has unexpected loops etc.\n      .withRecursive(\n        'ancestors',\n        ['source_key', 'source_entity_ref', 'target_entity_ref', 'subject'],\n        initial =>\n          initial\n            .select(\n              'refresh_state_references.source_key',\n              'refresh_state_references.source_entity_ref',\n              'refresh_state_references.target_entity_ref',\n              'descendants.entity_ref',\n            )\n            .from('descendants')\n            .join(\n              'refresh_state_references',\n              'refresh_state_references.target_entity_ref',\n              'descendants.entity_ref',\n            )\n            .union(recursive =>\n              recursive\n                .select(\n                  'refresh_state_references.source_key',\n                  'refresh_state_references.source_entity_ref',\n                  'refresh_state_references.target_entity_ref',\n                  'ancestors.subject',\n                )\n                .from('ancestors')\n                .join(\n                  'refresh_state_references',\n                  'refresh_state_references.target_entity_ref',\n                  'ancestors.source_entity_ref',\n                ),\n            ),\n      )\n      // Finally, from that list of ancestor relations per descendant, pick\n      // out the ones that are roots (have a source_key). Specifically, find\n      // ones that seem to be be either (1) from another source, or (2)\n      // aren't part of the deletion targets. Those are markers that tell us\n      // that the corresponding descendant should be kept alive and NOT\n      // subject to eager deletion, because there's \"something else\" (not\n      // targeted for deletion) that has references down through the tree to\n      // it.\n      //\n      // Continuing the scenario from above, for R3 we have\n      //\n      //   KeyA-R1, KeyA-R4, KeyB-R6\n      //\n      // This tells us that R3 should be kept alive for two reasons: it's\n      // referenced by a node that isn't being deleted (R4), and also by\n      // another source (KeyB). What about R1 and R2? They both have\n      //\n      //   KeyA-R1\n      //\n      // So those should be deleted, since they are definitely only being\n      // kept alive by something that's about to be deleted.\n      //\n      // Final shape of the tree:\n      //\n      //                  R3\n      //                 /\n      //   KeyA - R4 - R5\n      //              /\n      //   KeyB --- R6\n      .with('retained', ['entity_ref'], notPartOfDeletion =>\n        notPartOfDeletion\n          .select('subject')\n          .from('ancestors')\n          .whereNotNull('ancestors.source_key')\n          .where(foreignKeyOrRef =>\n            foreignKeyOrRef\n              .where('ancestors.source_key', '!=', sourceKey)\n              .orWhereNotIn('ancestors.target_entity_ref', refs),\n          ),\n      )\n      // Return all descendants minus the retained ones\n      .select('descendants.entity_ref AS entity_ref')\n      .from('descendants')\n      .leftOuterJoin(\n        'retained',\n        'retained.entity_ref',\n        'descendants.entity_ref',\n      )\n      .whereNull('retained.entity_ref')\n      .then(rows => rows.map(row => row.entity_ref));\n\n  return { orphanEntityRefs: orphans };\n}\n\nasync function markEntitiesAffectedByDeletionForStitching(options: {\n  knex: Knex | Knex.Transaction;\n  entityRefs: string[];\n}) {\n  const { knex, entityRefs } = options;\n\n  // We want to re-stitch anything that has a relation pointing to the\n  // soon-to-be-deleted entity. In many circumstances we also re-stitch children\n  // in the refresh_state_references graph because their orphan state might\n  // change, but not here - this code by its very definition is meant to not\n  // leave any orphans behind, so we can simplify away that.\n  const affectedIds = await knex\n    .distinct('refresh_state.entity_id AS entity_id')\n    .from('relations')\n    .join(\n      'refresh_state',\n      'relations.source_entity_ref',\n      'refresh_state.entity_ref',\n    )\n    .whereIn('relations.target_entity_ref', entityRefs)\n    .then(rows => rows.map(row => row.entity_id));\n\n  await markForStitching({\n    knex,\n    entityIds: affectedIds,\n  });\n}\n"],"names":["lodash","markForStitching"],"mappings":";;;;;;;;;AA4BA,eAAsB,iCAAiC,OAAA,EAInC;AAClB,EAAA,MAAM,EAAE,IAAA,EAAM,UAAA,EAAY,SAAA,EAAU,GAAI,OAAA;AAIxC,EAAA,IAAI,YAAA,GAAe,CAAA;AACnB,EAAA,KAAA,MAAW,IAAA,IAAQA,uBAAA,CAAO,KAAA,CAAM,UAAA,EAAY,GAAI,CAAA,EAAG;AACjD,IAAA,MAAM,EAAE,gBAAA,EAAiB,GACvB,MAAM,kDAAA,CAAmD;AAAA,MACvD,MAAM,OAAA,CAAQ,IAAA;AAAA,MACd,IAAA;AAAA,MACA;AAAA,KACD,CAAA;AAGH,IAAA,KAAA,MAAW,YAAA,IAAgBA,uBAAA,CAAO,KAAA,CAAM,gBAAA,EAAkB,GAAI,CAAA,EAAG;AAC/D,MAAA,MAAM,0CAAA,CAA2C;AAAA,QAC/C,MAAM,OAAA,CAAQ,IAAA;AAAA,QACd,UAAA,EAAY;AAAA,OACb,CAAA;AACD,MAAA,MAAM,IAAA,CACH,QAAO,CACP,IAAA,CAAK,eAAe,CAAA,CACpB,OAAA,CAAQ,cAAc,YAAY,CAAA;AACrC,MAAA,MAAM,IAAA,CACH,QAAO,CACP,IAAA,CAAK,cAAc,CAAA,CACnB,OAAA,CAAQ,cAAc,YAAY,CAAA;AAAA,IACvC;AAKA,IAAA,MAAM,IAAA,CAAkC,0BAA0B,CAAA,CAC/D,KAAA,CAAM,YAAA,EAAc,GAAA,EAAK,SAAS,CAAA,CAClC,OAAA,CAAQ,mBAAA,EAAqB,IAAI,CAAA,CACjC,MAAA,EAAO;AAEV,IAAA,YAAA,IAAgB,gBAAA,CAAiB,MAAA;AAAA,EACnC;AAEA,EAAA,OAAO,YAAA;AACT;AAEA,eAAe,mDAAmD,OAAA,EAItB;AAC1C,EAAA,MAAM,EAAE,IAAA,EAAM,IAAA,EAAM,SAAA,EAAU,GAAI,OAAA;AAElC,EAAA,MAAM,OAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAwBJ,MAAM,IAAA,CACH,aAAA;AAAA,MAAc,aAAA;AAAA,MAAe,CAAC,YAAY,CAAA;AAAA,MAAG,aAC5C,OAAA,CACG,MAAA,CAAO,mBAAmB,CAAA,CAC1B,KAAK,0BAA0B,CAAA,CAC/B,KAAA,CAAM,YAAA,EAAc,KAAK,SAAS,CAAA,CAClC,OAAA,CAAQ,mBAAA,EAAqB,IAAI,CAAA,CACjC,KAAA;AAAA,QAAM,eACL,SAAA,CACG,MAAA,CAAO,4CAA4C,CAAA,CACnD,IAAA,CAAK,aAAa,CAAA,CAClB,IAAA;AAAA,UACC,0BAAA;AAAA,UACA,wBAAA;AAAA,UACA;AAAA;AACF;AACJ,KACJ,CAmBC,aAAA;AAAA,MACC,WAAA;AAAA,MACA,CAAC,YAAA,EAAc,mBAAA,EAAqB,mBAAA,EAAqB,SAAS,CAAA;AAAA,MAClE,aACE,OAAA,CACG,MAAA;AAAA,QACC,qCAAA;AAAA,QACA,4CAAA;AAAA,QACA,4CAAA;AAAA,QACA;AAAA,OACF,CACC,IAAA,CAAK,aAAa,CAAA,CAClB,IAAA;AAAA,QACC,0BAAA;AAAA,QACA,4CAAA;AAAA,QACA;AAAA,OACF,CACC,KAAA;AAAA,QAAM,eACL,SAAA,CACG,MAAA;AAAA,UACC,qCAAA;AAAA,UACA,4CAAA;AAAA,UACA,4CAAA;AAAA,UACA;AAAA,SACF,CACC,IAAA,CAAK,WAAW,CAAA,CAChB,IAAA;AAAA,UACC,0BAAA;AAAA,UACA,4CAAA;AAAA,UACA;AAAA;AACF;AACJ,KACN,CA8BC,IAAA;AAAA,MAAK,UAAA;AAAA,MAAY,CAAC,YAAY,CAAA;AAAA,MAAG,CAAA,iBAAA,KAChC,iBAAA,CACG,MAAA,CAAO,SAAS,CAAA,CAChB,KAAK,WAAW,CAAA,CAChB,YAAA,CAAa,sBAAsB,CAAA,CACnC,KAAA;AAAA,QAAM,CAAA,eAAA,KACL,gBACG,KAAA,CAAM,sBAAA,EAAwB,MAAM,SAAS,CAAA,CAC7C,YAAA,CAAa,6BAAA,EAA+B,IAAI;AAAA;AACrD,MAGH,MAAA,CAAO,sCAAsC,CAAA,CAC7C,IAAA,CAAK,aAAa,CAAA,CAClB,aAAA;AAAA,MACC,UAAA;AAAA,MACA,qBAAA;AAAA,MACA;AAAA,KACF,CACC,SAAA,CAAU,qBAAqB,CAAA,CAC/B,IAAA,CAAK,CAAA,IAAA,KAAQ,IAAA,CAAK,GAAA,CAAI,CAAA,GAAA,KAAO,GAAA,CAAI,UAAU,CAAC;AAAA,GAAA;AAEjD,EAAA,OAAO,EAAE,kBAAkB,OAAA,EAAQ;AACrC;AAEA,eAAe,2CAA2C,OAAA,EAGvD;AACD,EAAA,MAAM,EAAE,IAAA,EAAM,UAAA,EAAW,GAAI,OAAA;AAO7B,EAAA,MAAM,WAAA,GAAc,MAAM,IAAA,CACvB,QAAA,CAAS,sCAAsC,CAAA,CAC/C,IAAA,CAAK,WAAW,CAAA,CAChB,IAAA;AAAA,IACC,eAAA;AAAA,IACA,6BAAA;AAAA,IACA;AAAA,GACF,CACC,OAAA,CAAQ,6BAAA,EAA+B,UAAU,CAAA,CACjD,IAAA,CAAK,CAAA,IAAA,KAAQ,IAAA,CAAK,GAAA,CAAI,CAAA,GAAA,KAAO,GAAA,CAAI,SAAS,CAAC,CAAA;AAE9C,EAAA,MAAMC,iCAAA,CAAiB;AAAA,IACrB,IAAA;AAAA,IACA,SAAA,EAAW;AAAA,GACZ,CAAA;AACH;;;;"}

package/dist/database/operations/stitcher/getDeferredStitchableEntities.cjs.js

@@ -5,29 +5,31 @@ var conversion = require('../../conversion.cjs.js');
 
 async function getDeferredStitchableEntities(options) {
   const { knex, batchSize, stitchTimeout } = options;
-  let itemsQuery = knex("stitch_queue").select(
-    "entity_ref",
-    "next_stitch_at",
-    "stitch_ticket"
+  const useLocking = ["mysql", "mysql2", "pg"].includes(
+    knex.client.config.client
   );
-  if (["mysql", "mysql2", "pg"].includes(knex.client.config.client)) {
-    itemsQuery = itemsQuery.forUpdate().skipLocked();
-  }
-  const items = await itemsQuery.where("next_stitch_at", "<=", knex.fn.now()).orderBy("next_stitch_at", "asc").limit(batchSize);
-  if (!items.length) {
-    return [];
-  }
-  await knex("stitch_queue").whereIn(
-    "entity_ref",
-    items.map((i) => i.entity_ref)
-  ).update({
-    next_stitch_at: nowPlus(knex, stitchTimeout)
-  });
-  return items.map((i) => ({
-    entityRef: i.entity_ref,
-    stitchTicket: i.stitch_ticket,
-    stitchRequestedAt: conversion.timestampToDateTime(i.next_stitch_at)
-  }));
+  const run = async (tx) => {
+    const items = await tx("stitch_queue").select("entity_ref", "next_stitch_at", "stitch_ticket").where("next_stitch_at", "<=", tx.fn.now()).orderBy("next_stitch_at", "asc").limit(batchSize).modify((qb) => {
+      if (useLocking) {
+        qb.forUpdate().skipLocked();
+      }
+    });
+    if (!items.length) {
+      return [];
+    }
+    await tx("stitch_queue").whereIn(
+      "entity_ref",
+      items.map((i) => i.entity_ref)
+    ).update({
+      next_stitch_at: nowPlus(tx, stitchTimeout)
+    });
+    return items.map((i) => ({
+      entityRef: i.entity_ref,
+      stitchTicket: i.stitch_ticket,
+      stitchRequestedAt: conversion.timestampToDateTime(i.next_stitch_at)
+    }));
+  };
+  return knex.isTransaction || !useLocking ? await run(knex) : await knex.transaction(run);
 }
 function nowPlus(knex, duration) {
   const seconds = types.durationToMilliseconds(duration) / 1e3;

package/dist/database/operations/stitcher/getDeferredStitchableEntities.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"getDeferredStitchableEntities.cjs.js","sources":["../../../../src/database/operations/stitcher/getDeferredStitchableEntities.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { durationToMilliseconds, HumanDuration } from '@backstage/types';\nimport { Knex } from 'knex';\nimport { DateTime } from 'luxon';\nimport { timestampToDateTime } from '../../conversion';\nimport { DbStitchQueueRow } from '../../tables';\n\n// TODO(freben): There is no retry counter or similar. If items start\n// perpetually crashing during stitching, they'll just get silently retried over\n// and over again, for better or worse. This will be visible in metrics though.\n\n/**\n * Finds entities that are marked for deferred stitching.\n *\n * @remarks\n *\n * This assumes that the stitching strategy is set to deferred.\n *\n * They are expected to already have the stitch_ticket set (by\n * markForStitching) so that their tickets can be returned with each item.\n *\n * All returned items have their next_stitch_at updated to be moved forward by\n * the given timeout duration. This has the effect that they will be picked up\n * for stitching again in the future, if it hasn't completed by that point for\n * some reason (restarts, crashes, etc).\n */\nexport async function getDeferredStitchableEntities(options: {\n  knex: Knex | Knex.Transaction;\n  batchSize: number;\n  stitchTimeout: HumanDuration;\n}): Promise<\n  Array<{\n    entityRef: string;\n    stitchTicket: string;\n    stitchRequestedAt: DateTime; // the time BEFORE moving it forward by the timeout\n  }>\n> {\n  const { knex, batchSize, stitchTimeout } = options;\n\n  let itemsQuery = knex<DbStitchQueueRow>('stitch_queue').select(\n    'entity_ref',\n    'next_stitch_at',\n    'stitch_ticket',\n  );\n\n  // This avoids duplication of work because of race conditions and is\n  // also fast because locked rows are ignored rather than blocking.\n  // It's only available in MySQL and PostgreSQL\n  if (['mysql', 'mysql2', 'pg'].includes(knex.client.config.client)) {\n    itemsQuery = itemsQuery.forUpdate().skipLocked();\n  }\n\n  const items = await itemsQuery\n    .where('next_stitch_at', '<=', knex.fn.now())\n    .orderBy('next_stitch_at', 'asc')\n    .limit(batchSize);\n\n  if (!items.length) {\n    return [];\n  }\n\n  await knex<DbStitchQueueRow>('stitch_queue')\n    .whereIn(\n      'entity_ref',\n      items.map(i => i.entity_ref),\n    )\n    .update({\n      next_stitch_at: nowPlus(knex, stitchTimeout),\n    });\n\n  return items.map(i => ({\n    entityRef: i.entity_ref,\n    stitchTicket: i.stitch_ticket,\n    stitchRequestedAt: timestampToDateTime(i.next_stitch_at),\n  }));\n}\n\nfunction nowPlus(knex: Knex, duration: HumanDuration): Knex.Raw {\n  const seconds = durationToMilliseconds(duration) / 1000;\n  if (knex.client.config.client.includes('sqlite3')) {\n    return knex.raw(`datetime('now', ?)`, [`${seconds} seconds`]);\n  } else if (knex.client.config.client.includes('mysql')) {\n    return knex.raw(`now() + interval ${seconds} second`);\n  }\n  return knex.raw(`now() + interval '${seconds} seconds'`);\n}\n"],"names":["timestampToDateTime","durationToMilliseconds"],"mappings":";;;;;AAyCA,eAAsB,8BAA8B,OAAA,EAUlD;AACA,EAAA,MAAM,EAAE,IAAA,EAAM,SAAA,EAAW,aAAA,EAAc,GAAI,OAAA;AAE3C,EAAA,IAAI,UAAA,GAAa,IAAA,CAAuB,cAAc,CAAA,CAAE,MAAA;AAAA,IACtD,YAAA;AAAA,IACA,gBAAA;AAAA,IACA;AAAA,GACF;AAKA,EAAA,IAAI,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,SAAS,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,MAAM,CAAA,EAAG;AACjE,IAAA,UAAA,GAAa,UAAA,CAAW,SAAA,EAAU,CAAE,UAAA,EAAW;AAAA,EACjD;AAEA,EAAA,MAAM,QAAQ,MAAM,UAAA,CACjB,KAAA,CAAM,gBAAA,EAAkB,MAAM,IAAA,CAAK,EAAA,CAAG,GAAA,EAAK,EAC3C,OAAA,CAAQ,gBAAA,EAAkB,KAAK,CAAA,CAC/B,MAAM,SAAS,CAAA;AAElB,EAAA,IAAI,CAAC,MAAM,MAAA,EAAQ;AACjB,IAAA,OAAO,EAAC;AAAA,EACV;AAEA,EAAA,MAAM,IAAA,CAAuB,cAAc,CAAA,CACxC,OAAA;AAAA,IACC,YAAA;AAAA,IACA,KAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,IAE5B,MAAA,CAAO;AAAA,IACN,cAAA,EAAgB,OAAA,CAAQ,IAAA,EAAM,aAAa;AAAA,GAC5C,CAAA;AAEH,EAAA,OAAO,KAAA,CAAM,IAAI,CAAA,CAAA,MAAM;AAAA,IACrB,WAAW,CAAA,CAAE,UAAA;AAAA,IACb,cAAc,CAAA,CAAE,aAAA;AAAA,IAChB,iBAAA,EAAmBA,8BAAA,CAAoB,CAAA,CAAE,cAAc;AAAA,GACzD,CAAE,CAAA;AACJ;AAEA,SAAS,OAAA,CAAQ,MAAY,QAAA,EAAmC;AAC9D,EAAA,MAAM,OAAA,GAAUC,4BAAA,CAAuB,QAAQ,CAAA,GAAI,GAAA;AACnD,EAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AACjD,IAAA,OAAO,KAAK,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,OAAO,UAAU,CAAC,CAAA;AAAA,EAC9D,WAAW,IAAA,CAAK,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACtD,IAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,iBAAA,EAAoB,OAAO,CAAA,OAAA,CAAS,CAAA;AAAA,EACtD;AACA,EAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,kBAAA,EAAqB,OAAO,CAAA,SAAA,CAAW,CAAA;AACzD;;;;"}
+{"version":3,"file":"getDeferredStitchableEntities.cjs.js","sources":["../../../../src/database/operations/stitcher/getDeferredStitchableEntities.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { durationToMilliseconds, HumanDuration } from '@backstage/types';\nimport { Knex } from 'knex';\nimport { DateTime } from 'luxon';\nimport { timestampToDateTime } from '../../conversion';\nimport { DbStitchQueueRow } from '../../tables';\n\n// TODO(freben): There is no retry counter or similar. If items start\n// perpetually crashing during stitching, they'll just get silently retried over\n// and over again, for better or worse. This will be visible in metrics though.\n\n/**\n * Finds entities that are marked for deferred stitching.\n *\n * @remarks\n *\n * This assumes that the stitching strategy is set to deferred.\n *\n * They are expected to already have the stitch_ticket set (by\n * markForStitching) so that their tickets can be returned with each item.\n *\n * All returned items have their next_stitch_at updated to be moved forward by\n * the given timeout duration. This has the effect that they will be picked up\n * for stitching again in the future, if it hasn't completed by that point for\n * some reason (restarts, crashes, etc).\n */\nexport async function getDeferredStitchableEntities(options: {\n  knex: Knex | Knex.Transaction;\n  batchSize: number;\n  stitchTimeout: HumanDuration;\n}): Promise<\n  Array<{\n    entityRef: string;\n    stitchTicket: string;\n    stitchRequestedAt: DateTime; // the time BEFORE moving it forward by the timeout\n  }>\n> {\n  const { knex, batchSize, stitchTimeout } = options;\n  const useLocking = ['mysql', 'mysql2', 'pg'].includes(\n    knex.client.config.client,\n  );\n\n  // The SELECT FOR UPDATE SKIP LOCKED + UPDATE must run inside a single\n  // transaction so that the row locks held by FOR UPDATE persist until\n  // next_stitch_at has been bumped. Without the transaction the locks\n  // are released after the SELECT auto-commits, and another worker can\n  // claim the same rows before the UPDATE runs.\n  const run = async (tx: Knex | Knex.Transaction) => {\n    const items: DbStitchQueueRow[] = await tx('stitch_queue')\n      .select('entity_ref', 'next_stitch_at', 'stitch_ticket')\n      .where('next_stitch_at', '<=', tx.fn.now())\n      .orderBy('next_stitch_at', 'asc')\n      .limit(batchSize)\n      .modify(qb => {\n        if (useLocking) {\n          qb.forUpdate().skipLocked();\n        }\n      });\n\n    if (!items.length) {\n      return [];\n    }\n\n    await tx('stitch_queue')\n      .whereIn(\n        'entity_ref',\n        items.map(i => i.entity_ref),\n      )\n      .update({\n        next_stitch_at: nowPlus(tx, stitchTimeout),\n      });\n\n    return items.map(i => ({\n      entityRef: i.entity_ref,\n      stitchTicket: i.stitch_ticket,\n      stitchRequestedAt: timestampToDateTime(i.next_stitch_at),\n    }));\n  };\n\n  return knex.isTransaction || !useLocking\n    ? await run(knex)\n    : await knex.transaction(run);\n}\n\nfunction nowPlus(knex: Knex, duration: HumanDuration): Knex.Raw {\n  const seconds = durationToMilliseconds(duration) / 1000;\n  if (knex.client.config.client.includes('sqlite3')) {\n    return knex.raw(`datetime('now', ?)`, [`${seconds} seconds`]);\n  } else if (knex.client.config.client.includes('mysql')) {\n    return knex.raw(`now() + interval ${seconds} second`);\n  }\n  return knex.raw(`now() + interval '${seconds} seconds'`);\n}\n"],"names":["timestampToDateTime","durationToMilliseconds"],"mappings":";;;;;AAyCA,eAAsB,8BAA8B,OAAA,EAUlD;AACA,EAAA,MAAM,EAAE,IAAA,EAAM,SAAA,EAAW,aAAA,EAAc,GAAI,OAAA;AAC3C,EAAA,MAAM,UAAA,GAAa,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,QAAA;AAAA,IAC3C,IAAA,CAAK,OAAO,MAAA,CAAO;AAAA,GACrB;AAOA,EAAA,MAAM,GAAA,GAAM,OAAO,EAAA,KAAgC;AACjD,IAAA,MAAM,KAAA,GAA4B,MAAM,EAAA,CAAG,cAAc,CAAA,CACtD,MAAA,CAAO,YAAA,EAAc,gBAAA,EAAkB,eAAe,CAAA,CACtD,KAAA,CAAM,gBAAA,EAAkB,IAAA,EAAM,GAAG,EAAA,CAAG,GAAA,EAAK,CAAA,CACzC,OAAA,CAAQ,gBAAA,EAAkB,KAAK,CAAA,CAC/B,KAAA,CAAM,SAAS,CAAA,CACf,MAAA,CAAO,CAAA,EAAA,KAAM;AACZ,MAAA,IAAI,UAAA,EAAY;AACd,QAAA,EAAA,CAAG,SAAA,GAAY,UAAA,EAAW;AAAA,MAC5B;AAAA,IACF,CAAC,CAAA;AAEH,IAAA,IAAI,CAAC,MAAM,MAAA,EAAQ;AACjB,MAAA,OAAO,EAAC;AAAA,IACV;AAEA,IAAA,MAAM,EAAA,CAAG,cAAc,CAAA,CACpB,OAAA;AAAA,MACC,YAAA;AAAA,MACA,KAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,MAE5B,MAAA,CAAO;AAAA,MACN,cAAA,EAAgB,OAAA,CAAQ,EAAA,EAAI,aAAa;AAAA,KAC1C,CAAA;AAEH,IAAA,OAAO,KAAA,CAAM,IAAI,CAAA,CAAA,MAAM;AAAA,MACrB,WAAW,CAAA,CAAE,UAAA;AAAA,MACb,cAAc,CAAA,CAAE,aAAA;AAAA,MAChB,iBAAA,EAAmBA,8BAAA,CAAoB,CAAA,CAAE,cAAc;AAAA,KACzD,CAAE,CAAA;AAAA,EACJ,CAAA;AAEA,EAAA,OAAO,IAAA,CAAK,aAAA,IAAiB,CAAC,UAAA,GAC1B,MAAM,GAAA,CAAI,IAAI,CAAA,GACd,MAAM,IAAA,CAAK,WAAA,CAAY,GAAG,CAAA;AAChC;AAEA,SAAS,OAAA,CAAQ,MAAY,QAAA,EAAmC;AAC9D,EAAA,MAAM,OAAA,GAAUC,4BAAA,CAAuB,QAAQ,CAAA,GAAI,GAAA;AACnD,EAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AACjD,IAAA,OAAO,KAAK,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,OAAO,UAAU,CAAC,CAAA;AAAA,EAC9D,WAAW,IAAA,CAAK,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACtD,IAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,iBAAA,EAAoB,OAAO,CAAA,OAAA,CAAS,CAAA;AAAA,EACtD;AACA,EAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,kBAAA,EAAqB,OAAO,CAAA,SAAA,CAAW,CAAA;AACzD;;;;"}

package/dist/database/operations/stitcher/markDeferredStitchCompleted.cjs.js

@@ -1,8 +1,15 @@
 'use strict';
 
 async function markDeferredStitchCompleted(option) {
-  const { knex, entityRef, stitchTicket } = option;
-  await knex("stitch_queue").where("entity_ref", "=", entityRef).andWhere("stitch_ticket", "=", stitchTicket).delete();
+  const { knex, entityRef, stitchTicket, result } = option;
+  const deleted = await knex("stitch_queue").where("entity_ref", "=", entityRef).andWhere("stitch_ticket", "=", stitchTicket).delete();
+  if (!deleted) {
+    const update = knex("stitch_queue").where("entity_ref", "=", entityRef).update({ next_stitch_at: knex.fn.now() });
+    if (result === "abandoned") {
+      update.where("next_stitch_at", ">", knex.fn.now());
+    }
+    await update;
+  }
 }
 
 exports.markDeferredStitchCompleted = markDeferredStitchCompleted;

package/dist/database/operations/stitcher/markDeferredStitchCompleted.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"markDeferredStitchCompleted.cjs.js","sources":["../../../../src/database/operations/stitcher/markDeferredStitchCompleted.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport { DbStitchQueueRow } from '../../tables';\n\n/**\n * Marks a single entity as having been stitched.\n *\n * @remarks\n *\n * This assumes that the stitching strategy is set to deferred.\n *\n * The row is only deleted from stitch_queue if the ticket hasn't changed. If\n * it has, it means that a new stitch request has been made, and the entity\n * should be stitched once more some time in the future - or is indeed already\n * being stitched concurrently with ourselves.\n */\nexport async function markDeferredStitchCompleted(option: {\n  knex: Knex | Knex.Transaction;\n  entityRef: string;\n  stitchTicket: string;\n}): Promise<void> {\n  const { knex, entityRef, stitchTicket } = option;\n\n  await knex<DbStitchQueueRow>('stitch_queue')\n    .where('entity_ref', '=', entityRef)\n    .andWhere('stitch_ticket', '=', stitchTicket)\n    .delete();\n}\n"],"names":[],"mappings":";;AA+BA,eAAsB,4BAA4B,MAAA,EAIhC;AAChB,EAAA,MAAM,EAAE,IAAA,EAAM,SAAA,EAAW,YAAA,EAAa,GAAI,MAAA;AAE1C,EAAA,MAAM,IAAA,CAAuB,cAAc,CAAA,CACxC,KAAA,CAAM,YAAA,EAAc,GAAA,EAAK,SAAS,CAAA,CAClC,QAAA,CAAS,eAAA,EAAiB,GAAA,EAAK,YAAY,EAC3C,MAAA,EAAO;AACZ;;;;"}
+{"version":3,"file":"markDeferredStitchCompleted.cjs.js","sources":["../../../../src/database/operations/stitcher/markDeferredStitchCompleted.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport { DbStitchQueueRow } from '../../tables';\n\n/**\n * Marks a single entity as having been stitched.\n *\n * @remarks\n *\n * If the ticket still matches, the stitch_queue entry is deleted — no\n * further stitching is needed.\n *\n * If the ticket changed (a new stitch was requested while this one was\n * in progress), the entry is kept and its next_stitch_at is bumped to\n * now() so the re-stitch becomes immediately eligible for pickup.\n *\n * The `result` parameter controls how the bump behaves when the ticket\n * doesn't match:\n *\n * - `'succeeded'`: The worker wrote its result successfully. A ticket\n *   mismatch means a re-stitch was requested. Bump to now()\n *   unconditionally — we're done, the next worker should start ASAP.\n *\n * - `'abandoned'`: The worker's write was blocked by a stale ticket.\n *   We can't tell whether the ticket changed because of a re-stitch\n *   request (nobody else is active) or because we timed out and\n *   another worker claimed the entry. Bump to now() only if the\n *   timestamp hasn't moved past what we'd have set — i.e. only move\n *   it earlier, never later. This prevents extending the timeout\n *   window of an active worker, while still making overdue entries\n *   eligible immediately.\n */\nexport async function markDeferredStitchCompleted(option: {\n  knex: Knex | Knex.Transaction;\n  entityRef: string;\n  stitchTicket: string;\n  result: 'succeeded' | 'abandoned';\n}): Promise<void> {\n  const { knex, entityRef, stitchTicket, result } = option;\n\n  const deleted = await knex<DbStitchQueueRow>('stitch_queue')\n    .where('entity_ref', '=', entityRef)\n    .andWhere('stitch_ticket', '=', stitchTicket)\n    .delete();\n\n  if (!deleted) {\n    const update = knex<DbStitchQueueRow>('stitch_queue')\n      .where('entity_ref', '=', entityRef)\n      .update({ next_stitch_at: knex.fn.now() });\n\n    if (result === 'abandoned') {\n      // Only move the timestamp earlier, never later — if another\n      // worker pushed it forward, we don't want to undercut their\n      // timeout window.\n      update.where('next_stitch_at', '>', knex.fn.now());\n    }\n\n    await update;\n  }\n}\n"],"names":[],"mappings":";;AA+CA,eAAsB,4BAA4B,MAAA,EAKhC;AAChB,EAAA,MAAM,EAAE,IAAA,EAAM,SAAA,EAAW,YAAA,EAAc,QAAO,GAAI,MAAA;AAElD,EAAA,MAAM,OAAA,GAAU,MAAM,IAAA,CAAuB,cAAc,EACxD,KAAA,CAAM,YAAA,EAAc,GAAA,EAAK,SAAS,EAClC,QAAA,CAAS,eAAA,EAAiB,GAAA,EAAK,YAAY,EAC3C,MAAA,EAAO;AAEV,EAAA,IAAI,CAAC,OAAA,EAAS;AACZ,IAAA,MAAM,SAAS,IAAA,CAAuB,cAAc,CAAA,CACjD,KAAA,CAAM,cAAc,GAAA,EAAK,SAAS,CAAA,CAClC,MAAA,CAAO,EAAE,cAAA,EAAgB,IAAA,CAAK,EAAA,CAAG,GAAA,IAAO,CAAA;AAE3C,IAAA,IAAI,WAAW,WAAA,EAAa;AAI1B,MAAA,MAAA,CAAO,MAAM,gBAAA,EAAkB,GAAA,EAAK,IAAA,CAAK,EAAA,CAAG,KAAK,CAAA;AAAA,IACnD;AAEA,IAAA,MAAM,MAAA;AAAA,EACR;AACF;;;;"}

package/dist/database/operations/stitcher/markForStitching.cjs.js

@@ -13,61 +13,33 @@ async function markForStitching(options) {
   const entityRefs = sortSplit(options.entityRefs);
   const entityIds = sortSplit(options.entityIds);
   const knex = options.knex;
-  const mode = options.strategy.mode;
-  if (mode === "immediate") {
-    for (const chunk of entityRefs) {
-      await knex.table("final_entities").update({
-        hash: "force-stitching"
-      }).whereIn("entity_ref", chunk);
-      await util.retryOnDeadlock(async () => {
-        await knex.table("refresh_state").update({
-          result_hash: "force-stitching",
-          next_update_at: knex.fn.now()
-        }).whereIn("entity_ref", chunk);
-      }, knex);
-    }
-    for (const chunk of entityIds) {
-      await knex.table("final_entities").update({
-        hash: "force-stitching"
-      }).whereIn("entity_id", chunk);
-      await util.retryOnDeadlock(async () => {
-        await knex.table("refresh_state").update({
-          result_hash: "force-stitching",
-          next_update_at: knex.fn.now()
-        }).whereIn("entity_id", chunk);
-      }, knex);
-    }
-  } else if (mode === "deferred") {
-    const ticket = node_crypto.randomUUID();
-    for (const chunk of entityRefs) {
-      await util.retryOnDeadlock(async () => {
-        if (chunk.length > 0) {
-          await knex("stitch_queue").insert(
-            chunk.map((ref) => ({
-              entity_ref: ref,
-              stitch_ticket: ticket,
-              next_stitch_at: knex.fn.now()
-            }))
-          ).onConflict("entity_ref").merge(["next_stitch_at", "stitch_ticket"]);
-        }
-      }, knex);
-    }
-    for (const chunk of entityIds) {
-      await util.retryOnDeadlock(async () => {
-        const refreshStateRows = await knex("refresh_state").select("entity_ref").whereIn("entity_id", chunk);
-        if (refreshStateRows.length > 0) {
-          await knex("stitch_queue").insert(
-            refreshStateRows.map((row) => ({
-              entity_ref: row.entity_ref,
-              stitch_ticket: ticket,
-              next_stitch_at: knex.fn.now()
-            }))
-          ).onConflict("entity_ref").merge(["next_stitch_at", "stitch_ticket"]);
-        }
-      }, knex);
-    }
-  } else {
-    throw new Error(`Unknown stitching strategy mode ${mode}`);
+  const ticket = node_crypto.randomUUID();
+  for (const chunk of entityRefs) {
+    await util.retryOnDeadlock(async () => {
+      if (chunk.length > 0) {
+        await knex("stitch_queue").insert(
+          chunk.map((ref) => ({
+            entity_ref: ref,
+            stitch_ticket: ticket,
+            next_stitch_at: knex.fn.now()
+          }))
+        ).onConflict("entity_ref").merge(["stitch_ticket"]);
+      }
+    }, knex);
+  }
+  for (const chunk of entityIds) {
+    await util.retryOnDeadlock(async () => {
+      const refreshStateRows = await knex("refresh_state").select("entity_ref").whereIn("entity_id", chunk);
+      if (refreshStateRows.length > 0) {
+        await knex("stitch_queue").insert(
+          refreshStateRows.map((row) => ({
+            entity_ref: row.entity_ref,
+            stitch_ticket: ticket,
+            next_stitch_at: knex.fn.now()
+          }))
+        ).onConflict("entity_ref").merge(["stitch_ticket"]);
+      }
+    }, knex);
   }
 }
 function sortSplit(input) {