@backstage/plugin-catalog-backend 4.0.1-next.1 → 4.0.1-next.2

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @backstage/plugin-catalog-backend
 
+## 4.0.1-next.2
+
+### Patch Changes
+
+- 589dcef: Fixed an issue where successfully processed entities could remain unavailable if stitch scheduling was interrupted.
+- a69d971: Improved the reliability of deferred stitching. Pending stitching work is no longer lost when entities are deleted and re-added, and updated instances prevent timed-out stitching attempts from overwriting or removing work taken over by another updated instance. During rolling upgrades from older versions, instances may still perform redundant stitching. On MySQL, protection against overlapping writes remains best-effort.
+- 476f042: Fixed entities disappearing from filtered and sorted catalog queries after a database error during stitching. The final entity and its search index rows are now written together, so a failure part way through is rolled back and retried in full. Previously the entity could be left with a search index that no longer matched it, and because the entity itself looked up to date, every later stitch attempt skipped it. Such an entity stayed readable by direct lookup while missing from list queries that filter or sort, until it was next edited.
+
+  PostgreSQL and MySQL deadlocks trigger prompt, bounded retries of the entire write transaction, skipping the write if newer stitching work has superseded it. If retries are exhausted, queued work remains available for recovery after the stitching timeout.
+
+- Updated dependencies
+  - @backstage/integration@2.1.3-next.1
+  - @backstage/backend-openapi-utils@0.7.3-next.2
+
 ## 4.0.1-next.1
 
 ### Patch Changes

package/dist/database/operations/provider/deleteWithEagerPruningOfChildren.cjs.js

@@ -22,7 +22,6 @@ async function deleteWithEagerPruningOfChildren(options) {
         entityRefs: refsToDelete
       });
       await knex.delete().from("refresh_state").whereIn("entity_ref", refsToDelete);
-      await knex.delete().from("stitch_queue").whereIn("entity_ref", refsToDelete);
     }
     await knex("refresh_state_references").where("source_key", "=", sourceKey).whereIn("target_entity_ref", refs).delete();
     removedCount += orphanEntityRefs.length;

package/dist/database/operations/provider/deleteWithEagerPruningOfChildren.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"deleteWithEagerPruningOfChildren.cjs.js","sources":["../../../../src/database/operations/provider/deleteWithEagerPruningOfChildren.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { DbRefreshStateReferencesRow } from '../../tables';\nimport { markForStitching } from '../stitcher/markForStitching';\n\n/**\n * Given a number of entity refs originally created by a given entity provider\n * (source key), remove those entities from the refresh state, and at the same\n * time recursively remove every child that is a direct or indirect result of\n * processing those entities, if they would have otherwise become orphaned by\n * the removal of their parents.\n */\nexport async function deleteWithEagerPruningOfChildren(options: {\n  knex: Knex | Knex.Transaction;\n  entityRefs: string[];\n  sourceKey: string;\n}): Promise<number> {\n  const { knex, entityRefs, sourceKey } = options;\n\n  // Split up the operation by (large) chunks, so that we do not hit database\n  // limits for the number of permitted bindings on a precompiled statement\n  let removedCount = 0;\n  for (const refs of lodash.chunk(entityRefs, 1000)) {\n    const { orphanEntityRefs } =\n      await findDescendantsThatWouldHaveBeenOrphanedByDeletion({\n        knex: options.knex,\n        refs,\n        sourceKey,\n      });\n\n    // Chunk again - these can be many more than the outer chunk size\n    for (const refsToDelete of lodash.chunk(orphanEntityRefs, 1000)) {\n      await markEntitiesAffectedByDeletionForStitching({\n        knex: options.knex,\n        entityRefs: refsToDelete,\n      });\n      await knex\n        .delete()\n        .from('refresh_state')\n        .whereIn('entity_ref', refsToDelete);\n      await knex\n        .delete()\n        .from('stitch_queue')\n        .whereIn('entity_ref', refsToDelete);\n    }\n\n    // Delete the references that originate only from this entity provider. Note\n    // that there may be more than one entity provider making a \"claim\" for a\n    // given root entity, if they emit with the same location key.\n    await knex<DbRefreshStateReferencesRow>('refresh_state_references')\n      .where('source_key', '=', sourceKey)\n      .whereIn('target_entity_ref', refs)\n      .delete();\n\n    removedCount += orphanEntityRefs.length;\n  }\n\n  return removedCount;\n}\n\nasync function findDescendantsThatWouldHaveBeenOrphanedByDeletion(options: {\n  knex: Knex | Knex.Transaction;\n  refs: string[];\n  sourceKey: string;\n}): Promise<{ orphanEntityRefs: string[] }> {\n  const { knex, refs, sourceKey } = options;\n\n  const orphans: string[] =\n    // First find all nodes that can be reached downwards from the roots\n    // (deletion targets), including the roots themselves, by traversing\n    // down the refresh_state_references table. Note that this query\n    // starts with a condition that source_key = our source key, and\n    // target_entity_ref is one of the deletion targets. This has two\n    // effects: it won't match attempts at deleting something that didn't\n    // originate from us in the first place, and also won't match non-root\n    // entities (source_key would be null for those).\n    //\n    //   KeyA - R1 - R2        Legend:\n    //                 \\       -----------------------------------------\n    //                  R3     Key*    Source key\n    //                 /       R*      Entity ref\n    //   KeyA - R4 - R5        lines   Individual references; sources to\n    //              /                  the left and targets to the right\n    //   KeyB --- R6\n    //\n    // The scenario is that KeyA wants to delete R1.\n    //\n    // The query starts with the KeyA-R1 reference, and then traverses\n    // down to also find R2 and R3. It uses union instead of union all,\n    // because it wants to find the set of unique descendants even if\n    // the tree has unexpected loops etc.\n    await knex\n      .withRecursive('descendants', ['entity_ref'], initial =>\n        initial\n          .select('target_entity_ref')\n          .from('refresh_state_references')\n          .where('source_key', '=', sourceKey)\n          .whereIn('target_entity_ref', refs)\n          .union(recursive =>\n            recursive\n              .select('refresh_state_references.target_entity_ref')\n              .from('descendants')\n              .join(\n                'refresh_state_references',\n                'descendants.entity_ref',\n                'refresh_state_references.source_entity_ref',\n              ),\n          ),\n      )\n      // Then for each descendant, traverse all the way back upwards through\n      // the refresh_state_references table to get an exhaustive list of all\n      // references that are part of keeping that particular descendant\n      // alive.\n      //\n      // Continuing the scenario from above, starting from R3, it goes\n      // upwards to find every pair along every relation line.\n      //\n      //   Top branch:     R2-R3, R1-R2, KeyA-R1\n      //   Middle branch:  R5-R3, R4-R5, KeyA-R4\n      //   Bottom branch:  R6-R5, KeyB-R6\n      //\n      // Note that this all applied to the subject R3. The exact same thing\n      // will be done starting from each other descendant (R2 and R1). They\n      // only have one and two references to find, respectively.\n      //\n      // This query also uses union instead of union all, to get the set of\n      // distinct relations even if the tree has unexpected loops etc.\n      .withRecursive(\n        'ancestors',\n        ['source_key', 'source_entity_ref', 'target_entity_ref', 'subject'],\n        initial =>\n          initial\n            .select(\n              'refresh_state_references.source_key',\n              'refresh_state_references.source_entity_ref',\n              'refresh_state_references.target_entity_ref',\n              'descendants.entity_ref',\n            )\n            .from('descendants')\n            .join(\n              'refresh_state_references',\n              'refresh_state_references.target_entity_ref',\n              'descendants.entity_ref',\n            )\n            .union(recursive =>\n              recursive\n                .select(\n                  'refresh_state_references.source_key',\n                  'refresh_state_references.source_entity_ref',\n                  'refresh_state_references.target_entity_ref',\n                  'ancestors.subject',\n                )\n                .from('ancestors')\n                .join(\n                  'refresh_state_references',\n                  'refresh_state_references.target_entity_ref',\n                  'ancestors.source_entity_ref',\n                ),\n            ),\n      )\n      // Finally, from that list of ancestor relations per descendant, pick\n      // out the ones that are roots (have a source_key). Specifically, find\n      // ones that seem to be be either (1) from another source, or (2)\n      // aren't part of the deletion targets. Those are markers that tell us\n      // that the corresponding descendant should be kept alive and NOT\n      // subject to eager deletion, because there's \"something else\" (not\n      // targeted for deletion) that has references down through the tree to\n      // it.\n      //\n      // Continuing the scenario from above, for R3 we have\n      //\n      //   KeyA-R1, KeyA-R4, KeyB-R6\n      //\n      // This tells us that R3 should be kept alive for two reasons: it's\n      // referenced by a node that isn't being deleted (R4), and also by\n      // another source (KeyB). What about R1 and R2? They both have\n      //\n      //   KeyA-R1\n      //\n      // So those should be deleted, since they are definitely only being\n      // kept alive by something that's about to be deleted.\n      //\n      // Final shape of the tree:\n      //\n      //                  R3\n      //                 /\n      //   KeyA - R4 - R5\n      //              /\n      //   KeyB --- R6\n      .with('retained', ['entity_ref'], notPartOfDeletion =>\n        notPartOfDeletion\n          .select('subject')\n          .from('ancestors')\n          .whereNotNull('ancestors.source_key')\n          .where(foreignKeyOrRef =>\n            foreignKeyOrRef\n              .where('ancestors.source_key', '!=', sourceKey)\n              .orWhereNotIn('ancestors.target_entity_ref', refs),\n          ),\n      )\n      // Return all descendants minus the retained ones\n      .select('descendants.entity_ref AS entity_ref')\n      .from('descendants')\n      .leftOuterJoin(\n        'retained',\n        'retained.entity_ref',\n        'descendants.entity_ref',\n      )\n      .whereNull('retained.entity_ref')\n      .then(rows => rows.map(row => row.entity_ref));\n\n  return { orphanEntityRefs: orphans };\n}\n\nasync function markEntitiesAffectedByDeletionForStitching(options: {\n  knex: Knex | Knex.Transaction;\n  entityRefs: string[];\n}) {\n  const { knex, entityRefs } = options;\n\n  // We want to re-stitch anything that has a relation pointing to the\n  // soon-to-be-deleted entity. In many circumstances we also re-stitch children\n  // in the refresh_state_references graph because their orphan state might\n  // change, but not here - this code by its very definition is meant to not\n  // leave any orphans behind, so we can simplify away that.\n  const affectedIds = await knex\n    .distinct('refresh_state.entity_id AS entity_id')\n    .from('relations')\n    .join(\n      'refresh_state',\n      'relations.source_entity_ref',\n      'refresh_state.entity_ref',\n    )\n    .whereIn('relations.target_entity_ref', entityRefs)\n    .then(rows => rows.map(row => row.entity_id));\n\n  await markForStitching({\n    knex,\n    entityIds: affectedIds,\n  });\n}\n"],"names":["lodash","markForStitching"],"mappings":";;;;;;;;;AA4BA,eAAsB,iCAAiC,OAAA,EAInC;AAClB,EAAA,MAAM,EAAE,IAAA,EAAM,UAAA,EAAY,SAAA,EAAU,GAAI,OAAA;AAIxC,EAAA,IAAI,YAAA,GAAe,CAAA;AACnB,EAAA,KAAA,MAAW,IAAA,IAAQA,uBAAA,CAAO,KAAA,CAAM,UAAA,EAAY,GAAI,CAAA,EAAG;AACjD,IAAA,MAAM,EAAE,gBAAA,EAAiB,GACvB,MAAM,kDAAA,CAAmD;AAAA,MACvD,MAAM,OAAA,CAAQ,IAAA;AAAA,MACd,IAAA;AAAA,MACA;AAAA,KACD,CAAA;AAGH,IAAA,KAAA,MAAW,YAAA,IAAgBA,uBAAA,CAAO,KAAA,CAAM,gBAAA,EAAkB,GAAI,CAAA,EAAG;AAC/D,MAAA,MAAM,0CAAA,CAA2C;AAAA,QAC/C,MAAM,OAAA,CAAQ,IAAA;AAAA,QACd,UAAA,EAAY;AAAA,OACb,CAAA;AACD,MAAA,MAAM,IAAA,CACH,QAAO,CACP,IAAA,CAAK,eAAe,CAAA,CACpB,OAAA,CAAQ,cAAc,YAAY,CAAA;AACrC,MAAA,MAAM,IAAA,CACH,QAAO,CACP,IAAA,CAAK,cAAc,CAAA,CACnB,OAAA,CAAQ,cAAc,YAAY,CAAA;AAAA,IACvC;AAKA,IAAA,MAAM,IAAA,CAAkC,0BAA0B,CAAA,CAC/D,KAAA,CAAM,YAAA,EAAc,GAAA,EAAK,SAAS,CAAA,CAClC,OAAA,CAAQ,mBAAA,EAAqB,IAAI,CAAA,CACjC,MAAA,EAAO;AAEV,IAAA,YAAA,IAAgB,gBAAA,CAAiB,MAAA;AAAA,EACnC;AAEA,EAAA,OAAO,YAAA;AACT;AAEA,eAAe,mDAAmD,OAAA,EAItB;AAC1C,EAAA,MAAM,EAAE,IAAA,EAAM,IAAA,EAAM,SAAA,EAAU,GAAI,OAAA;AAElC,EAAA,MAAM,OAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAwBJ,MAAM,IAAA,CACH,aAAA;AAAA,MAAc,aAAA;AAAA,MAAe,CAAC,YAAY,CAAA;AAAA,MAAG,aAC5C,OAAA,CACG,MAAA,CAAO,mBAAmB,CAAA,CAC1B,KAAK,0BAA0B,CAAA,CAC/B,KAAA,CAAM,YAAA,EAAc,KAAK,SAAS,CAAA,CAClC,OAAA,CAAQ,mBAAA,EAAqB,IAAI,CAAA,CACjC,KAAA;AAAA,QAAM,eACL,SAAA,CACG,MAAA,CAAO,4CAA4C,CAAA,CACnD,IAAA,CAAK,aAAa,CAAA,CAClB,IAAA;AAAA,UACC,0BAAA;AAAA,UACA,wBAAA;AAAA,UACA;AAAA;AACF;AACJ,KACJ,CAmBC,aAAA;AAAA,MACC,WAAA;AAAA,MACA,CAAC,YAAA,EAAc,mBAAA,EAAqB,mBAAA,EAAqB,SAAS,CAAA;AAAA,MAClE,aACE,OAAA,CACG,MAAA;AAAA,QACC,qCAAA;AAAA,QACA,4CAAA;AAAA,QACA,4CAAA;AAAA,QACA;AAAA,OACF,CACC,IAAA,CAAK,aAAa,CAAA,CAClB,IAAA;AAAA,QACC,0BAAA;AAAA,QACA,4CAAA;AAAA,QACA;AAAA,OACF,CACC,KAAA;AAAA,QAAM,eACL,SAAA,CACG,MAAA;AAAA,UACC,qCAAA;AAAA,UACA,4CAAA;AAAA,UACA,4CAAA;AAAA,UACA;AAAA,SACF,CACC,IAAA,CAAK,WAAW,CAAA,CAChB,IAAA;AAAA,UACC,0BAAA;AAAA,UACA,4CAAA;AAAA,UACA;AAAA;AACF;AACJ,KACN,CA8BC,IAAA;AAAA,MAAK,UAAA;AAAA,MAAY,CAAC,YAAY,CAAA;AAAA,MAAG,CAAA,iBAAA,KAChC,iBAAA,CACG,MAAA,CAAO,SAAS,CAAA,CAChB,KAAK,WAAW,CAAA,CAChB,YAAA,CAAa,sBAAsB,CAAA,CACnC,KAAA;AAAA,QAAM,CAAA,eAAA,KACL,gBACG,KAAA,CAAM,sBAAA,EAAwB,MAAM,SAAS,CAAA,CAC7C,YAAA,CAAa,6BAAA,EAA+B,IAAI;AAAA;AACrD,MAGH,MAAA,CAAO,sCAAsC,CAAA,CAC7C,IAAA,CAAK,aAAa,CAAA,CAClB,aAAA;AAAA,MACC,UAAA;AAAA,MACA,qBAAA;AAAA,MACA;AAAA,KACF,CACC,SAAA,CAAU,qBAAqB,CAAA,CAC/B,IAAA,CAAK,CAAA,IAAA,KAAQ,IAAA,CAAK,GAAA,CAAI,CAAA,GAAA,KAAO,GAAA,CAAI,UAAU,CAAC;AAAA,GAAA;AAEjD,EAAA,OAAO,EAAE,kBAAkB,OAAA,EAAQ;AACrC;AAEA,eAAe,2CAA2C,OAAA,EAGvD;AACD,EAAA,MAAM,EAAE,IAAA,EAAM,UAAA,EAAW,GAAI,OAAA;AAO7B,EAAA,MAAM,WAAA,GAAc,MAAM,IAAA,CACvB,QAAA,CAAS,sCAAsC,CAAA,CAC/C,IAAA,CAAK,WAAW,CAAA,CAChB,IAAA;AAAA,IACC,eAAA;AAAA,IACA,6BAAA;AAAA,IACA;AAAA,GACF,CACC,OAAA,CAAQ,6BAAA,EAA+B,UAAU,CAAA,CACjD,IAAA,CAAK,CAAA,IAAA,KAAQ,IAAA,CAAK,GAAA,CAAI,CAAA,GAAA,KAAO,GAAA,CAAI,SAAS,CAAC,CAAA;AAE9C,EAAA,MAAMC,iCAAA,CAAiB;AAAA,IACrB,IAAA;AAAA,IACA,SAAA,EAAW;AAAA,GACZ,CAAA;AACH;;"}
+{"version":3,"file":"deleteWithEagerPruningOfChildren.cjs.js","sources":["../../../../src/database/operations/provider/deleteWithEagerPruningOfChildren.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { DbRefreshStateReferencesRow } from '../../tables';\nimport { markForStitching } from '../stitcher/markForStitching';\n\n/**\n * Given a number of entity refs originally created by a given entity provider\n * (source key), remove those entities from the refresh state, and at the same\n * time recursively remove every child that is a direct or indirect result of\n * processing those entities, if they would have otherwise become orphaned by\n * the removal of their parents.\n */\nexport async function deleteWithEagerPruningOfChildren(options: {\n  knex: Knex | Knex.Transaction;\n  entityRefs: string[];\n  sourceKey: string;\n}): Promise<number> {\n  const { knex, entityRefs, sourceKey } = options;\n\n  // Split up the operation by (large) chunks, so that we do not hit database\n  // limits for the number of permitted bindings on a precompiled statement\n  let removedCount = 0;\n  for (const refs of lodash.chunk(entityRefs, 1000)) {\n    const { orphanEntityRefs } =\n      await findDescendantsThatWouldHaveBeenOrphanedByDeletion({\n        knex: options.knex,\n        refs,\n        sourceKey,\n      });\n\n    // Chunk again - these can be many more than the outer chunk size\n    for (const refsToDelete of lodash.chunk(orphanEntityRefs, 1000)) {\n      await markEntitiesAffectedByDeletionForStitching({\n        knex: options.knex,\n        entityRefs: refsToDelete,\n      });\n      await knex\n        .delete()\n        .from('refresh_state')\n        .whereIn('entity_ref', refsToDelete);\n      // Leave pending stitches intact. The stitcher safely settles missing\n      // entities, while a concurrent reintroduction may still need this work.\n    }\n\n    // Delete the references that originate only from this entity provider. Note\n    // that there may be more than one entity provider making a \"claim\" for a\n    // given root entity, if they emit with the same location key.\n    await knex<DbRefreshStateReferencesRow>('refresh_state_references')\n      .where('source_key', '=', sourceKey)\n      .whereIn('target_entity_ref', refs)\n      .delete();\n\n    removedCount += orphanEntityRefs.length;\n  }\n\n  return removedCount;\n}\n\nasync function findDescendantsThatWouldHaveBeenOrphanedByDeletion(options: {\n  knex: Knex | Knex.Transaction;\n  refs: string[];\n  sourceKey: string;\n}): Promise<{ orphanEntityRefs: string[] }> {\n  const { knex, refs, sourceKey } = options;\n\n  const orphans: string[] =\n    // First find all nodes that can be reached downwards from the roots\n    // (deletion targets), including the roots themselves, by traversing\n    // down the refresh_state_references table. Note that this query\n    // starts with a condition that source_key = our source key, and\n    // target_entity_ref is one of the deletion targets. This has two\n    // effects: it won't match attempts at deleting something that didn't\n    // originate from us in the first place, and also won't match non-root\n    // entities (source_key would be null for those).\n    //\n    //   KeyA - R1 - R2        Legend:\n    //                 \\       -----------------------------------------\n    //                  R3     Key*    Source key\n    //                 /       R*      Entity ref\n    //   KeyA - R4 - R5        lines   Individual references; sources to\n    //              /                  the left and targets to the right\n    //   KeyB --- R6\n    //\n    // The scenario is that KeyA wants to delete R1.\n    //\n    // The query starts with the KeyA-R1 reference, and then traverses\n    // down to also find R2 and R3. It uses union instead of union all,\n    // because it wants to find the set of unique descendants even if\n    // the tree has unexpected loops etc.\n    await knex\n      .withRecursive('descendants', ['entity_ref'], initial =>\n        initial\n          .select('target_entity_ref')\n          .from('refresh_state_references')\n          .where('source_key', '=', sourceKey)\n          .whereIn('target_entity_ref', refs)\n          .union(recursive =>\n            recursive\n              .select('refresh_state_references.target_entity_ref')\n              .from('descendants')\n              .join(\n                'refresh_state_references',\n                'descendants.entity_ref',\n                'refresh_state_references.source_entity_ref',\n              ),\n          ),\n      )\n      // Then for each descendant, traverse all the way back upwards through\n      // the refresh_state_references table to get an exhaustive list of all\n      // references that are part of keeping that particular descendant\n      // alive.\n      //\n      // Continuing the scenario from above, starting from R3, it goes\n      // upwards to find every pair along every relation line.\n      //\n      //   Top branch:     R2-R3, R1-R2, KeyA-R1\n      //   Middle branch:  R5-R3, R4-R5, KeyA-R4\n      //   Bottom branch:  R6-R5, KeyB-R6\n      //\n      // Note that this all applied to the subject R3. The exact same thing\n      // will be done starting from each other descendant (R2 and R1). They\n      // only have one and two references to find, respectively.\n      //\n      // This query also uses union instead of union all, to get the set of\n      // distinct relations even if the tree has unexpected loops etc.\n      .withRecursive(\n        'ancestors',\n        ['source_key', 'source_entity_ref', 'target_entity_ref', 'subject'],\n        initial =>\n          initial\n            .select(\n              'refresh_state_references.source_key',\n              'refresh_state_references.source_entity_ref',\n              'refresh_state_references.target_entity_ref',\n              'descendants.entity_ref',\n            )\n            .from('descendants')\n            .join(\n              'refresh_state_references',\n              'refresh_state_references.target_entity_ref',\n              'descendants.entity_ref',\n            )\n            .union(recursive =>\n              recursive\n                .select(\n                  'refresh_state_references.source_key',\n                  'refresh_state_references.source_entity_ref',\n                  'refresh_state_references.target_entity_ref',\n                  'ancestors.subject',\n                )\n                .from('ancestors')\n                .join(\n                  'refresh_state_references',\n                  'refresh_state_references.target_entity_ref',\n                  'ancestors.source_entity_ref',\n                ),\n            ),\n      )\n      // Finally, from that list of ancestor relations per descendant, pick\n      // out the ones that are roots (have a source_key). Specifically, find\n      // ones that seem to be be either (1) from another source, or (2)\n      // aren't part of the deletion targets. Those are markers that tell us\n      // that the corresponding descendant should be kept alive and NOT\n      // subject to eager deletion, because there's \"something else\" (not\n      // targeted for deletion) that has references down through the tree to\n      // it.\n      //\n      // Continuing the scenario from above, for R3 we have\n      //\n      //   KeyA-R1, KeyA-R4, KeyB-R6\n      //\n      // This tells us that R3 should be kept alive for two reasons: it's\n      // referenced by a node that isn't being deleted (R4), and also by\n      // another source (KeyB). What about R1 and R2? They both have\n      //\n      //   KeyA-R1\n      //\n      // So those should be deleted, since they are definitely only being\n      // kept alive by something that's about to be deleted.\n      //\n      // Final shape of the tree:\n      //\n      //                  R3\n      //                 /\n      //   KeyA - R4 - R5\n      //              /\n      //   KeyB --- R6\n      .with('retained', ['entity_ref'], notPartOfDeletion =>\n        notPartOfDeletion\n          .select('subject')\n          .from('ancestors')\n          .whereNotNull('ancestors.source_key')\n          .where(foreignKeyOrRef =>\n            foreignKeyOrRef\n              .where('ancestors.source_key', '!=', sourceKey)\n              .orWhereNotIn('ancestors.target_entity_ref', refs),\n          ),\n      )\n      // Return all descendants minus the retained ones\n      .select('descendants.entity_ref AS entity_ref')\n      .from('descendants')\n      .leftOuterJoin(\n        'retained',\n        'retained.entity_ref',\n        'descendants.entity_ref',\n      )\n      .whereNull('retained.entity_ref')\n      .then(rows => rows.map(row => row.entity_ref));\n\n  return { orphanEntityRefs: orphans };\n}\n\nasync function markEntitiesAffectedByDeletionForStitching(options: {\n  knex: Knex | Knex.Transaction;\n  entityRefs: string[];\n}) {\n  const { knex, entityRefs } = options;\n\n  // We want to re-stitch anything that has a relation pointing to the\n  // soon-to-be-deleted entity. In many circumstances we also re-stitch children\n  // in the refresh_state_references graph because their orphan state might\n  // change, but not here - this code by its very definition is meant to not\n  // leave any orphans behind, so we can simplify away that.\n  const affectedIds = await knex\n    .distinct('refresh_state.entity_id AS entity_id')\n    .from('relations')\n    .join(\n      'refresh_state',\n      'relations.source_entity_ref',\n      'refresh_state.entity_ref',\n    )\n    .whereIn('relations.target_entity_ref', entityRefs)\n    .then(rows => rows.map(row => row.entity_id));\n\n  await markForStitching({\n    knex,\n    entityIds: affectedIds,\n  });\n}\n"],"names":["lodash","markForStitching"],"mappings":";;;;;;;;;AA4BA,eAAsB,iCAAiC,OAAA,EAInC;AAClB,EAAA,MAAM,EAAE,IAAA,EAAM,UAAA,EAAY,SAAA,EAAU,GAAI,OAAA;AAIxC,EAAA,IAAI,YAAA,GAAe,CAAA;AACnB,EAAA,KAAA,MAAW,IAAA,IAAQA,uBAAA,CAAO,KAAA,CAAM,UAAA,EAAY,GAAI,CAAA,EAAG;AACjD,IAAA,MAAM,EAAE,gBAAA,EAAiB,GACvB,MAAM,kDAAA,CAAmD;AAAA,MACvD,MAAM,OAAA,CAAQ,IAAA;AAAA,MACd,IAAA;AAAA,MACA;AAAA,KACD,CAAA;AAGH,IAAA,KAAA,MAAW,YAAA,IAAgBA,uBAAA,CAAO,KAAA,CAAM,gBAAA,EAAkB,GAAI,CAAA,EAAG;AAC/D,MAAA,MAAM,0CAAA,CAA2C;AAAA,QAC/C,MAAM,OAAA,CAAQ,IAAA;AAAA,QACd,UAAA,EAAY;AAAA,OACb,CAAA;AACD,MAAA,MAAM,IAAA,CACH,QAAO,CACP,IAAA,CAAK,eAAe,CAAA,CACpB,OAAA,CAAQ,cAAc,YAAY,CAAA;AAAA,IAGvC;AAKA,IAAA,MAAM,IAAA,CAAkC,0BAA0B,CAAA,CAC/D,KAAA,CAAM,YAAA,EAAc,GAAA,EAAK,SAAS,CAAA,CAClC,OAAA,CAAQ,mBAAA,EAAqB,IAAI,CAAA,CACjC,MAAA,EAAO;AAEV,IAAA,YAAA,IAAgB,gBAAA,CAAiB,MAAA;AAAA,EACnC;AAEA,EAAA,OAAO,YAAA;AACT;AAEA,eAAe,mDAAmD,OAAA,EAItB;AAC1C,EAAA,MAAM,EAAE,IAAA,EAAM,IAAA,EAAM,SAAA,EAAU,GAAI,OAAA;AAElC,EAAA,MAAM,OAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAwBJ,MAAM,IAAA,CACH,aAAA;AAAA,MAAc,aAAA;AAAA,MAAe,CAAC,YAAY,CAAA;AAAA,MAAG,aAC5C,OAAA,CACG,MAAA,CAAO,mBAAmB,CAAA,CAC1B,KAAK,0BAA0B,CAAA,CAC/B,KAAA,CAAM,YAAA,EAAc,KAAK,SAAS,CAAA,CAClC,OAAA,CAAQ,mBAAA,EAAqB,IAAI,CAAA,CACjC,KAAA;AAAA,QAAM,eACL,SAAA,CACG,MAAA,CAAO,4CAA4C,CAAA,CACnD,IAAA,CAAK,aAAa,CAAA,CAClB,IAAA;AAAA,UACC,0BAAA;AAAA,UACA,wBAAA;AAAA,UACA;AAAA;AACF;AACJ,KACJ,CAmBC,aAAA;AAAA,MACC,WAAA;AAAA,MACA,CAAC,YAAA,EAAc,mBAAA,EAAqB,mBAAA,EAAqB,SAAS,CAAA;AAAA,MAClE,aACE,OAAA,CACG,MAAA;AAAA,QACC,qCAAA;AAAA,QACA,4CAAA;AAAA,QACA,4CAAA;AAAA,QACA;AAAA,OACF,CACC,IAAA,CAAK,aAAa,CAAA,CAClB,IAAA;AAAA,QACC,0BAAA;AAAA,QACA,4CAAA;AAAA,QACA;AAAA,OACF,CACC,KAAA;AAAA,QAAM,eACL,SAAA,CACG,MAAA;AAAA,UACC,qCAAA;AAAA,UACA,4CAAA;AAAA,UACA,4CAAA;AAAA,UACA;AAAA,SACF,CACC,IAAA,CAAK,WAAW,CAAA,CAChB,IAAA;AAAA,UACC,0BAAA;AAAA,UACA,4CAAA;AAAA,UACA;AAAA;AACF;AACJ,KACN,CA8BC,IAAA;AAAA,MAAK,UAAA;AAAA,MAAY,CAAC,YAAY,CAAA;AAAA,MAAG,CAAA,iBAAA,KAChC,iBAAA,CACG,MAAA,CAAO,SAAS,CAAA,CAChB,KAAK,WAAW,CAAA,CAChB,YAAA,CAAa,sBAAsB,CAAA,CACnC,KAAA;AAAA,QAAM,CAAA,eAAA,KACL,gBACG,KAAA,CAAM,sBAAA,EAAwB,MAAM,SAAS,CAAA,CAC7C,YAAA,CAAa,6BAAA,EAA+B,IAAI;AAAA;AACrD,MAGH,MAAA,CAAO,sCAAsC,CAAA,CAC7C,IAAA,CAAK,aAAa,CAAA,CAClB,aAAA;AAAA,MACC,UAAA;AAAA,MACA,qBAAA;AAAA,MACA;AAAA,KACF,CACC,SAAA,CAAU,qBAAqB,CAAA,CAC/B,IAAA,CAAK,CAAA,IAAA,KAAQ,IAAA,CAAK,GAAA,CAAI,CAAA,GAAA,KAAO,GAAA,CAAI,UAAU,CAAC;AAAA,GAAA;AAEjD,EAAA,OAAO,EAAE,kBAAkB,OAAA,EAAQ;AACrC;AAEA,eAAe,2CAA2C,OAAA,EAGvD;AACD,EAAA,MAAM,EAAE,IAAA,EAAM,UAAA,EAAW,GAAI,OAAA;AAO7B,EAAA,MAAM,WAAA,GAAc,MAAM,IAAA,CACvB,QAAA,CAAS,sCAAsC,CAAA,CAC/C,IAAA,CAAK,WAAW,CAAA,CAChB,IAAA;AAAA,IACC,eAAA;AAAA,IACA,6BAAA;AAAA,IACA;AAAA,GACF,CACC,OAAA,CAAQ,6BAAA,EAA+B,UAAU,CAAA,CACjD,IAAA,CAAK,CAAA,IAAA,KAAQ,IAAA,CAAK,GAAA,CAAI,CAAA,GAAA,KAAO,GAAA,CAAI,SAAS,CAAC,CAAA;AAE9C,EAAA,MAAMC,iCAAA,CAAiB;AAAA,IACrB,IAAA;AAAA,IACA,SAAA,EAAW;AAAA,GACZ,CAAA;AACH;;"}

package/dist/database/operations/stitcher/getDeferredStitchableEntities.cjs.js

@@ -1,6 +1,7 @@
 'use strict';
 
 var types = require('@backstage/types');
+var node_crypto = require('node:crypto');
 var conversion = require('../../conversion.cjs.js');
 
 async function getDeferredStitchableEntities(options) {
@@ -17,19 +18,45 @@ async function getDeferredStitchableEntities(options) {
     if (!items.length) {
       return [];
     }
-    await tx("stitch_queue").whereIn(
+    const stitchTicket = node_crypto.randomUUID();
+    const update = tx("stitch_queue").whereIn(
       "entity_ref",
       items.map((i) => i.entity_ref)
     ).update({
-      next_stitch_at: nowPlus(tx, stitchTimeout)
+      next_stitch_at: nowPlus(tx, stitchTimeout),
+      stitch_ticket: stitchTicket
     });
+    let leases;
+    if (tx.client.config.client === "pg") {
+      leases = await update.returning([
+        "entity_ref",
+        tx.raw("next_stitch_at::text as next_stitch_at")
+      ]);
+    } else if (String(tx.client.config.client).includes("mysql")) {
+      await update;
+      leases = await tx("stitch_queue").select(
+        "entity_ref",
+        tx.raw("CAST(next_stitch_at AS CHAR) as next_stitch_at")
+      ).whereIn(
+        "entity_ref",
+        items.map((i) => i.entity_ref)
+      );
+    } else {
+      leases = await update.returning(["entity_ref", "next_stitch_at"]);
+    }
+    const leaseByRef = new Map(
+      leases.map((i) => [i.entity_ref, i.next_stitch_at])
+    );
     return items.map((i) => ({
       entityRef: i.entity_ref,
-      stitchTicket: i.stitch_ticket,
-      stitchRequestedAt: conversion.timestampToDateTime(i.next_stitch_at)
+      stitchTicket,
+      stitchRequestedAt: conversion.timestampToDateTime(i.next_stitch_at),
+      stitchLeaseExpiresAt: leaseByRef.get(
+        i.entity_ref
+      )
     }));
   };
-  return knex.isTransaction || !useLocking ? await run(knex) : await knex.transaction(run);
+  return knex.isTransaction ? await run(knex) : await knex.transaction(run);
 }
 function nowPlus(knex, duration) {
   const seconds = types.durationToMilliseconds(duration) / 1e3;

package/dist/database/operations/stitcher/getDeferredStitchableEntities.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"getDeferredStitchableEntities.cjs.js","sources":["../../../../src/database/operations/stitcher/getDeferredStitchableEntities.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { durationToMilliseconds, HumanDuration } from '@backstage/types';\nimport { Knex } from 'knex';\nimport { DateTime } from 'luxon';\nimport { timestampToDateTime } from '../../conversion';\nimport { DbStitchQueueRow } from '../../tables';\n\n// TODO(freben): There is no retry counter or similar. If items start\n// perpetually crashing during stitching, they'll just get silently retried over\n// and over again, for better or worse. This will be visible in metrics though.\n\n/**\n * Finds entities that are marked for deferred stitching.\n *\n * @remarks\n *\n * This assumes that the stitching strategy is set to deferred.\n *\n * They are expected to already have the stitch_ticket set (by\n * markForStitching) so that their tickets can be returned with each item.\n *\n * All returned items have their next_stitch_at updated to be moved forward by\n * the given timeout duration. This has the effect that they will be picked up\n * for stitching again in the future, if it hasn't completed by that point for\n * some reason (restarts, crashes, etc).\n */\nexport async function getDeferredStitchableEntities(options: {\n  knex: Knex | Knex.Transaction;\n  batchSize: number;\n  stitchTimeout: HumanDuration;\n}): Promise<\n  Array<{\n    entityRef: string;\n    stitchTicket: string;\n    stitchRequestedAt: DateTime; // the time BEFORE moving it forward by the timeout\n  }>\n> {\n  const { knex, batchSize, stitchTimeout } = options;\n  const useLocking = ['mysql', 'mysql2', 'pg'].includes(\n    knex.client.config.client,\n  );\n\n  // The SELECT FOR UPDATE SKIP LOCKED + UPDATE must run inside a single\n  // transaction so that the row locks held by FOR UPDATE persist until\n  // next_stitch_at has been bumped. Without the transaction the locks\n  // are released after the SELECT auto-commits, and another worker can\n  // claim the same rows before the UPDATE runs.\n  const run = async (tx: Knex | Knex.Transaction) => {\n    const items: DbStitchQueueRow[] = await tx('stitch_queue')\n      .select('entity_ref', 'next_stitch_at', 'stitch_ticket')\n      .where('next_stitch_at', '<=', tx.fn.now())\n      .orderBy('next_stitch_at', 'asc')\n      .limit(batchSize)\n      .modify(qb => {\n        if (useLocking) {\n          qb.forUpdate().skipLocked();\n        }\n      });\n\n    if (!items.length) {\n      return [];\n    }\n\n    await tx('stitch_queue')\n      .whereIn(\n        'entity_ref',\n        items.map(i => i.entity_ref),\n      )\n      .update({\n        next_stitch_at: nowPlus(tx, stitchTimeout),\n      });\n\n    return items.map(i => ({\n      entityRef: i.entity_ref,\n      stitchTicket: i.stitch_ticket,\n      stitchRequestedAt: timestampToDateTime(i.next_stitch_at),\n    }));\n  };\n\n  return knex.isTransaction || !useLocking\n    ? await run(knex)\n    : await knex.transaction(run);\n}\n\nfunction nowPlus(knex: Knex, duration: HumanDuration): Knex.Raw {\n  const seconds = durationToMilliseconds(duration) / 1000;\n  if (knex.client.config.client.includes('sqlite3')) {\n    return knex.raw(`datetime('now', ?)`, [`${seconds} seconds`]);\n  } else if (knex.client.config.client.includes('mysql')) {\n    return knex.raw(`now() + interval ${seconds} second`);\n  }\n  return knex.raw(`now() + interval '${seconds} seconds'`);\n}\n"],"names":["timestampToDateTime","durationToMilliseconds"],"mappings":";;;;;AAyCA,eAAsB,8BAA8B,OAAA,EAUlD;AACA,EAAA,MAAM,EAAE,IAAA,EAAM,SAAA,EAAW,aAAA,EAAc,GAAI,OAAA;AAC3C,EAAA,MAAM,UAAA,GAAa,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,QAAA;AAAA,IAC3C,IAAA,CAAK,OAAO,MAAA,CAAO;AAAA,GACrB;AAOA,EAAA,MAAM,GAAA,GAAM,OAAO,EAAA,KAAgC;AACjD,IAAA,MAAM,KAAA,GAA4B,MAAM,EAAA,CAAG,cAAc,CAAA,CACtD,MAAA,CAAO,YAAA,EAAc,gBAAA,EAAkB,eAAe,CAAA,CACtD,KAAA,CAAM,gBAAA,EAAkB,IAAA,EAAM,GAAG,EAAA,CAAG,GAAA,EAAK,CAAA,CACzC,OAAA,CAAQ,gBAAA,EAAkB,KAAK,CAAA,CAC/B,KAAA,CAAM,SAAS,CAAA,CACf,MAAA,CAAO,CAAA,EAAA,KAAM;AACZ,MAAA,IAAI,UAAA,EAAY;AACd,QAAA,EAAA,CAAG,SAAA,GAAY,UAAA,EAAW;AAAA,MAC5B;AAAA,IACF,CAAC,CAAA;AAEH,IAAA,IAAI,CAAC,MAAM,MAAA,EAAQ;AACjB,MAAA,OAAO,EAAC;AAAA,IACV;AAEA,IAAA,MAAM,EAAA,CAAG,cAAc,CAAA,CACpB,OAAA;AAAA,MACC,YAAA;AAAA,MACA,KAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,MAE5B,MAAA,CAAO;AAAA,MACN,cAAA,EAAgB,OAAA,CAAQ,EAAA,EAAI,aAAa;AAAA,KAC1C,CAAA;AAEH,IAAA,OAAO,KAAA,CAAM,IAAI,CAAA,CAAA,MAAM;AAAA,MACrB,WAAW,CAAA,CAAE,UAAA;AAAA,MACb,cAAc,CAAA,CAAE,aAAA;AAAA,MAChB,iBAAA,EAAmBA,8BAAA,CAAoB,CAAA,CAAE,cAAc;AAAA,KACzD,CAAE,CAAA;AAAA,EACJ,CAAA;AAEA,EAAA,OAAO,IAAA,CAAK,aAAA,IAAiB,CAAC,UAAA,GAC1B,MAAM,GAAA,CAAI,IAAI,CAAA,GACd,MAAM,IAAA,CAAK,WAAA,CAAY,GAAG,CAAA;AAChC;AAEA,SAAS,OAAA,CAAQ,MAAY,QAAA,EAAmC;AAC9D,EAAA,MAAM,OAAA,GAAUC,4BAAA,CAAuB,QAAQ,CAAA,GAAI,GAAA;AACnD,EAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AACjD,IAAA,OAAO,KAAK,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,OAAO,UAAU,CAAC,CAAA;AAAA,EAC9D,WAAW,IAAA,CAAK,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACtD,IAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,iBAAA,EAAoB,OAAO,CAAA,OAAA,CAAS,CAAA;AAAA,EACtD;AACA,EAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,kBAAA,EAAqB,OAAO,CAAA,SAAA,CAAW,CAAA;AACzD;;"}
+{"version":3,"file":"getDeferredStitchableEntities.cjs.js","sources":["../../../../src/database/operations/stitcher/getDeferredStitchableEntities.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { durationToMilliseconds, HumanDuration } from '@backstage/types';\nimport { Knex } from 'knex';\nimport { DateTime } from 'luxon';\nimport { randomUUID } from 'node:crypto';\nimport { timestampToDateTime } from '../../conversion';\nimport { DbStitchQueueRow } from '../../tables';\n\n/**\n * Opaque lease value captured by a deferred stitch claim. Pass it unchanged\n * back to completion using the same database. Do not parse or normalize it:\n * PostgreSQL requires its exact timestamp text to retain microsecond precision;\n * MySQL uses timestamp text to avoid timezone/DST conversion through a Date.\n * SQLite also returns its stored timestamp text.\n */\nexport type StitchLeaseExpiresAt = DbStitchQueueRow['next_stitch_at'] & {\n  readonly __stitchLeaseExpiresAt: unique symbol;\n};\n\n// TODO(freben): There is no retry counter or similar. If items start\n// perpetually crashing during stitching, they'll just get silently retried over\n// and over again, for better or worse. This will be visible in metrics though.\n\n/**\n * Finds entities that are marked for deferred stitching.\n *\n * @remarks\n *\n * This assumes that the stitching strategy is set to deferred.\n *\n * Each claim replaces stitch_ticket, fencing workers whose leases expired even\n * when no new stitch request was made. A ticket may be shared across a batch;\n * it is always checked together with entity_ref.\n *\n * All returned items have their next_stitch_at updated to be moved forward by\n * the given timeout duration. This has the effect that they will be picked up\n * for stitching again in the future, if it hasn't completed by that point for\n * some reason (restarts, crashes, etc).\n */\nexport async function getDeferredStitchableEntities(options: {\n  knex: Knex | Knex.Transaction;\n  batchSize: number;\n  stitchTimeout: HumanDuration;\n}): Promise<\n  Array<{\n    entityRef: string;\n    stitchTicket: string;\n    stitchRequestedAt: DateTime; // the time BEFORE moving it forward by the timeout\n    stitchLeaseExpiresAt: StitchLeaseExpiresAt;\n  }>\n> {\n  const { knex, batchSize, stitchTimeout } = options;\n  const useLocking = ['mysql', 'mysql2', 'pg'].includes(\n    knex.client.config.client,\n  );\n\n  // The SELECT FOR UPDATE SKIP LOCKED + UPDATE must run inside a single\n  // transaction so that the row locks held by FOR UPDATE persist until\n  // next_stitch_at has been bumped. Without the transaction the locks\n  // are released after the SELECT auto-commits, and another worker can\n  // claim the same rows before the UPDATE runs.\n  // SQLite also keeps selection and ticket/lease replacement in one transaction,\n  // without attempting the row-locking clauses that it does not support.\n  const run = async (tx: Knex | Knex.Transaction) => {\n    const items: DbStitchQueueRow[] = await tx('stitch_queue')\n      .select('entity_ref', 'next_stitch_at', 'stitch_ticket')\n      .where('next_stitch_at', '<=', tx.fn.now())\n      .orderBy('next_stitch_at', 'asc')\n      .limit(batchSize)\n      .modify(qb => {\n        if (useLocking) {\n          qb.forUpdate().skipLocked();\n        }\n      });\n\n    if (!items.length) {\n      return [];\n    }\n\n    const stitchTicket = randomUUID();\n    const update = tx<DbStitchQueueRow>('stitch_queue')\n      .whereIn(\n        'entity_ref',\n        items.map(i => i.entity_ref),\n      )\n      .update({\n        next_stitch_at: nowPlus(tx, stitchTimeout),\n        stitch_ticket: stitchTicket,\n      });\n\n    // Preserve PostgreSQL's full timestamp precision for the completion guard;\n    // converting it to a JavaScript Date would truncate microseconds.\n    let leases: Pick<DbStitchQueueRow, 'entity_ref' | 'next_stitch_at'>[];\n    if (tx.client.config.client === 'pg') {\n      leases = await update.returning([\n        'entity_ref',\n        tx.raw('next_stitch_at::text as next_stitch_at'),\n      ]);\n    } else if (String(tx.client.config.client).includes('mysql')) {\n      // MySQL has no UPDATE RETURNING; read the lease on the same locked\n      // connection before committing the claim. Cast to text so the driver\n      // cannot normalize the value through a Date across a local DST gap.\n      await update;\n      leases = await tx<DbStitchQueueRow>('stitch_queue')\n        .select<Pick<DbStitchQueueRow, 'entity_ref' | 'next_stitch_at'>[]>(\n          'entity_ref',\n          tx.raw('CAST(next_stitch_at AS CHAR) as next_stitch_at'),\n        )\n        .whereIn(\n          'entity_ref',\n          items.map(i => i.entity_ref),\n        );\n    } else {\n      leases = await update.returning(['entity_ref', 'next_stitch_at']);\n    }\n    const leaseByRef = new Map(\n      leases.map(i => [i.entity_ref, i.next_stitch_at]),\n    );\n\n    return items.map(i => ({\n      entityRef: i.entity_ref,\n      stitchTicket,\n      stitchRequestedAt: timestampToDateTime(i.next_stitch_at),\n      stitchLeaseExpiresAt: leaseByRef.get(\n        i.entity_ref,\n      )! as StitchLeaseExpiresAt,\n    }));\n  };\n\n  return knex.isTransaction ? await run(knex) : await knex.transaction(run);\n}\n\nfunction nowPlus(knex: Knex, duration: HumanDuration): Knex.Raw {\n  const seconds = durationToMilliseconds(duration) / 1000;\n  if (knex.client.config.client.includes('sqlite3')) {\n    return knex.raw(`datetime('now', ?)`, [`${seconds} seconds`]);\n  } else if (knex.client.config.client.includes('mysql')) {\n    return knex.raw(`now() + interval ${seconds} second`);\n  }\n  return knex.raw(`now() + interval '${seconds} seconds'`);\n}\n"],"names":["randomUUID","timestampToDateTime","durationToMilliseconds"],"mappings":";;;;;;AAsDA,eAAsB,8BAA8B,OAAA,EAWlD;AACA,EAAA,MAAM,EAAE,IAAA,EAAM,SAAA,EAAW,aAAA,EAAc,GAAI,OAAA;AAC3C,EAAA,MAAM,UAAA,GAAa,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,QAAA;AAAA,IAC3C,IAAA,CAAK,OAAO,MAAA,CAAO;AAAA,GACrB;AASA,EAAA,MAAM,GAAA,GAAM,OAAO,EAAA,KAAgC;AACjD,IAAA,MAAM,KAAA,GAA4B,MAAM,EAAA,CAAG,cAAc,CAAA,CACtD,MAAA,CAAO,YAAA,EAAc,gBAAA,EAAkB,eAAe,CAAA,CACtD,KAAA,CAAM,gBAAA,EAAkB,IAAA,EAAM,GAAG,EAAA,CAAG,GAAA,EAAK,CAAA,CACzC,OAAA,CAAQ,gBAAA,EAAkB,KAAK,CAAA,CAC/B,KAAA,CAAM,SAAS,CAAA,CACf,MAAA,CAAO,CAAA,EAAA,KAAM;AACZ,MAAA,IAAI,UAAA,EAAY;AACd,QAAA,EAAA,CAAG,SAAA,GAAY,UAAA,EAAW;AAAA,MAC5B;AAAA,IACF,CAAC,CAAA;AAEH,IAAA,IAAI,CAAC,MAAM,MAAA,EAAQ;AACjB,MAAA,OAAO,EAAC;AAAA,IACV;AAEA,IAAA,MAAM,eAAeA,sBAAA,EAAW;AAChC,IAAA,MAAM,MAAA,GAAS,EAAA,CAAqB,cAAc,CAAA,CAC/C,OAAA;AAAA,MACC,YAAA;AAAA,MACA,KAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,MAE5B,MAAA,CAAO;AAAA,MACN,cAAA,EAAgB,OAAA,CAAQ,EAAA,EAAI,aAAa,CAAA;AAAA,MACzC,aAAA,EAAe;AAAA,KAChB,CAAA;AAIH,IAAA,IAAI,MAAA;AACJ,IAAA,IAAI,EAAA,CAAG,MAAA,CAAO,MAAA,CAAO,MAAA,KAAW,IAAA,EAAM;AACpC,MAAA,MAAA,GAAS,MAAM,OAAO,SAAA,CAAU;AAAA,QAC9B,YAAA;AAAA,QACA,EAAA,CAAG,IAAI,wCAAwC;AAAA,OAChD,CAAA;AAAA,IACH,CAAA,MAAA,IAAW,OAAO,EAAA,CAAG,MAAA,CAAO,OAAO,MAAM,CAAA,CAAE,QAAA,CAAS,OAAO,CAAA,EAAG;AAI5D,MAAA,MAAM,MAAA;AACN,MAAA,MAAA,GAAS,MAAM,EAAA,CAAqB,cAAc,CAAA,CAC/C,MAAA;AAAA,QACC,YAAA;AAAA,QACA,EAAA,CAAG,IAAI,gDAAgD;AAAA,OACzD,CACC,OAAA;AAAA,QACC,YAAA;AAAA,QACA,KAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,OAC7B;AAAA,IACJ,CAAA,MAAO;AACL,MAAA,MAAA,GAAS,MAAM,MAAA,CAAO,SAAA,CAAU,CAAC,YAAA,EAAc,gBAAgB,CAAC,CAAA;AAAA,IAClE;AACA,IAAA,MAAM,aAAa,IAAI,GAAA;AAAA,MACrB,MAAA,CAAO,IAAI,CAAA,CAAA,KAAK,CAAC,EAAE,UAAA,EAAY,CAAA,CAAE,cAAc,CAAC;AAAA,KAClD;AAEA,IAAA,OAAO,KAAA,CAAM,IAAI,CAAA,CAAA,MAAM;AAAA,MACrB,WAAW,CAAA,CAAE,UAAA;AAAA,MACb,YAAA;AAAA,MACA,iBAAA,EAAmBC,8BAAA,CAAoB,CAAA,CAAE,cAAc,CAAA;AAAA,MACvD,sBAAsB,UAAA,CAAW,GAAA;AAAA,QAC/B,CAAA,CAAE;AAAA;AACJ,KACF,CAAE,CAAA;AAAA,EACJ,CAAA;AAEA,EAAA,OAAO,IAAA,CAAK,gBAAgB,MAAM,GAAA,CAAI,IAAI,CAAA,GAAI,MAAM,IAAA,CAAK,WAAA,CAAY,GAAG,CAAA;AAC1E;AAEA,SAAS,OAAA,CAAQ,MAAY,QAAA,EAAmC;AAC9D,EAAA,MAAM,OAAA,GAAUC,4BAAA,CAAuB,QAAQ,CAAA,GAAI,GAAA;AACnD,EAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AACjD,IAAA,OAAO,KAAK,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,OAAO,UAAU,CAAC,CAAA;AAAA,EAC9D,WAAW,IAAA,CAAK,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACtD,IAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,iBAAA,EAAoB,OAAO,CAAA,OAAA,CAAS,CAAA;AAAA,EACtD;AACA,EAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,kBAAA,EAAqB,OAAO,CAAA,SAAA,CAAW,CAAA;AACzD;;"}

package/dist/database/operations/stitcher/markDeferredStitchCompleted.cjs.js

@@ -1,14 +1,10 @@
 'use strict';
 
 async function markDeferredStitchCompleted(option) {
-  const { knex, entityRef, stitchTicket, result } = option;
-  const deleted = await knex("stitch_queue").where("entity_ref", "=", entityRef).andWhere("stitch_ticket", "=", stitchTicket).delete();
+  const { knex, entityRef, stitchTicket, stitchLeaseExpiresAt } = option;
+  const deleted = await knex("stitch_queue").where("entity_ref", "=", entityRef).andWhere("stitch_ticket", "=", stitchTicket).andWhere("next_stitch_at", "=", stitchLeaseExpiresAt).delete();
   if (!deleted) {
-    const update = knex("stitch_queue").where("entity_ref", "=", entityRef).update({ next_stitch_at: knex.fn.now() });
-    if (result === "abandoned") {
-      update.where("next_stitch_at", ">", knex.fn.now());
-    }
-    await update;
+    await knex("stitch_queue").where("entity_ref", "=", entityRef).where("next_stitch_at", "=", stitchLeaseExpiresAt).update({ next_stitch_at: knex.fn.now() });
   }
 }
 

package/dist/database/operations/stitcher/markDeferredStitchCompleted.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"markDeferredStitchCompleted.cjs.js","sources":["../../../../src/database/operations/stitcher/markDeferredStitchCompleted.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport { DbStitchQueueRow } from '../../tables';\n\n/**\n * Marks a single entity as having been stitched.\n *\n * @remarks\n *\n * If the ticket still matches, the stitch_queue entry is deleted — no\n * further stitching is needed.\n *\n * If the ticket changed (a new stitch was requested while this one was\n * in progress), the entry is kept and its next_stitch_at is bumped to\n * now() so the re-stitch becomes immediately eligible for pickup.\n *\n * The `result` parameter controls how the bump behaves when the ticket\n * doesn't match:\n *\n * - `'succeeded'`: The worker wrote its result successfully. A ticket\n *   mismatch means a re-stitch was requested. Bump to now()\n *   unconditionally — we're done, the next worker should start ASAP.\n *\n * - `'abandoned'`: The worker's write was blocked by a stale ticket.\n *   We can't tell whether the ticket changed because of a re-stitch\n *   request (nobody else is active) or because we timed out and\n *   another worker claimed the entry. Bump to now() only if the\n *   timestamp hasn't moved past what we'd have set — i.e. only move\n *   it earlier, never later. This prevents extending the timeout\n *   window of an active worker, while still making overdue entries\n *   eligible immediately.\n */\nexport async function markDeferredStitchCompleted(option: {\n  knex: Knex | Knex.Transaction;\n  entityRef: string;\n  stitchTicket: string;\n  result: 'succeeded' | 'abandoned';\n}): Promise<void> {\n  const { knex, entityRef, stitchTicket, result } = option;\n\n  const deleted = await knex<DbStitchQueueRow>('stitch_queue')\n    .where('entity_ref', '=', entityRef)\n    .andWhere('stitch_ticket', '=', stitchTicket)\n    .delete();\n\n  if (!deleted) {\n    const update = knex<DbStitchQueueRow>('stitch_queue')\n      .where('entity_ref', '=', entityRef)\n      .update({ next_stitch_at: knex.fn.now() });\n\n    if (result === 'abandoned') {\n      // Only move the timestamp earlier, never later — if another\n      // worker pushed it forward, we don't want to undercut their\n      // timeout window.\n      update.where('next_stitch_at', '>', knex.fn.now());\n    }\n\n    await update;\n  }\n}\n"],"names":[],"mappings":";;AA+CA,eAAsB,4BAA4B,MAAA,EAKhC;AAChB,EAAA,MAAM,EAAE,IAAA,EAAM,SAAA,EAAW,YAAA,EAAc,QAAO,GAAI,MAAA;AAElD,EAAA,MAAM,OAAA,GAAU,MAAM,IAAA,CAAuB,cAAc,EACxD,KAAA,CAAM,YAAA,EAAc,GAAA,EAAK,SAAS,EAClC,QAAA,CAAS,eAAA,EAAiB,GAAA,EAAK,YAAY,EAC3C,MAAA,EAAO;AAEV,EAAA,IAAI,CAAC,OAAA,EAAS;AACZ,IAAA,MAAM,SAAS,IAAA,CAAuB,cAAc,CAAA,CACjD,KAAA,CAAM,cAAc,GAAA,EAAK,SAAS,CAAA,CAClC,MAAA,CAAO,EAAE,cAAA,EAAgB,IAAA,CAAK,EAAA,CAAG,GAAA,IAAO,CAAA;AAE3C,IAAA,IAAI,WAAW,WAAA,EAAa;AAI1B,MAAA,MAAA,CAAO,MAAM,gBAAA,EAAkB,GAAA,EAAK,IAAA,CAAK,EAAA,CAAG,KAAK,CAAA;AAAA,IACnD;AAEA,IAAA,MAAM,MAAA;AAAA,EACR;AACF;;"}
+{"version":3,"file":"markDeferredStitchCompleted.cjs.js","sources":["../../../../src/database/operations/stitcher/markDeferredStitchCompleted.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport { DbStitchQueueRow } from '../../tables';\nimport { StitchLeaseExpiresAt } from './getDeferredStitchableEntities';\n\n/**\n * Marks a single entity as having been stitched.\n *\n * @remarks\n *\n * If the ticket and captured lease still match, the stitch_queue entry is\n * deleted — no further stitching is needed. Both guards are needed during\n * mixed-version rollouts: old workers reclaim without replacing the ticket.\n *\n * A changed ticket with the same lease means a new request arrived during this\n * attempt. Make it immediately eligible, whether this attempt succeeded or was\n * abandoned. A changed lease belongs to a successor and must not be disturbed.\n * The captured lease is opaque and must be passed through unchanged from\n * getDeferredStitchableEntities, not reconstructed from a Date or ISO string.\n */\nexport async function markDeferredStitchCompleted(option: {\n  knex: Knex | Knex.Transaction;\n  entityRef: string;\n  stitchTicket: string;\n  stitchLeaseExpiresAt: StitchLeaseExpiresAt;\n}): Promise<void> {\n  const { knex, entityRef, stitchTicket, stitchLeaseExpiresAt } = option;\n\n  const deleted = await knex<DbStitchQueueRow>('stitch_queue')\n    .where('entity_ref', '=', entityRef)\n    .andWhere('stitch_ticket', '=', stitchTicket)\n    .andWhere('next_stitch_at', '=', stitchLeaseExpiresAt)\n    .delete();\n\n  if (!deleted) {\n    await knex<DbStitchQueueRow>('stitch_queue')\n      .where('entity_ref', '=', entityRef)\n      .where('next_stitch_at', '=', stitchLeaseExpiresAt)\n      .update({ next_stitch_at: knex.fn.now() });\n  }\n}\n"],"names":[],"mappings":";;AAmCA,eAAsB,4BAA4B,MAAA,EAKhC;AAChB,EAAA,MAAM,EAAE,IAAA,EAAM,SAAA,EAAW,YAAA,EAAc,sBAAqB,GAAI,MAAA;AAEhE,EAAA,MAAM,OAAA,GAAU,MAAM,IAAA,CAAuB,cAAc,EACxD,KAAA,CAAM,YAAA,EAAc,KAAK,SAAS,CAAA,CAClC,SAAS,eAAA,EAAiB,GAAA,EAAK,YAAY,CAAA,CAC3C,QAAA,CAAS,kBAAkB,GAAA,EAAK,oBAAoB,EACpD,MAAA,EAAO;AAEV,EAAA,IAAI,CAAC,OAAA,EAAS;AACZ,IAAA,MAAM,IAAA,CAAuB,cAAc,CAAA,CACxC,KAAA,CAAM,cAAc,GAAA,EAAK,SAAS,EAClC,KAAA,CAAM,gBAAA,EAAkB,KAAK,oBAAoB,CAAA,CACjD,OAAO,EAAE,cAAA,EAAgB,KAAK,EAAA,CAAG,GAAA,IAAO,CAAA;AAAA,EAC7C;AACF;;"}

package/dist/database/operations/stitcher/performStitching.cjs.js

@@ -7,11 +7,14 @@ var stableStringify = require('fast-json-stable-stringify');
 var buildEntitySearch = require('./buildEntitySearch.cjs.js');
 var markDeferredStitchCompleted = require('./markDeferredStitchCompleted.cjs.js');
 var syncSearchRows = require('./syncSearchRows.cjs.js');
+var util = require('../../util.cjs.js');
 
 function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e : { default: e }; }
 
 var stableStringify__default = /*#__PURE__*/_interopDefaultCompat(stableStringify);
 
+class StitchPublicationSupersededError extends Error {
+}
 function generateStableHash(entity) {
   return node_crypto.createHash("sha1").update(stableStringify__default.default({ ...entity })).digest("hex");
 }
@@ -20,7 +23,7 @@ const scriptProtocolPattern = (
   /^[\u0000-\u001F ]*j[\r\n\t]*a[\r\n\t]*v[\r\n\t]*a[\r\n\t]*s[\r\n\t]*c[\r\n\t]*r[\r\n\t]*i[\r\n\t]*p[\r\n\t]*t[\r\n\t]*\:/i
 );
 async function performStitching(options) {
-  const { knex, logger, entityRef, stitchTicket } = options;
+  const { knex, logger, entityRef, stitchTicket, stitchLeaseExpiresAt } = options;
   let stitchResult;
   try {
     const [processedResult, relationsResult] = await Promise.all([
@@ -113,43 +116,68 @@ async function performStitching(options) {
     }
     const searchEntries = buildEntitySearch.buildEntitySearch(entityId, entity);
     const isMySQL = String(knex.client.config.client).includes("mysql");
-    if (isMySQL) {
-      const ticketValid = await knex("stitch_queue").where("entity_ref", entityRef).where("stitch_ticket", stitchTicket).first();
-      if (!ticketValid) {
-        logger.debug(
-          `Entity ${entityRef} is already stitched, skipping write.`
-        );
-        stitchResult = "abandoned";
-        return "abandoned";
-      }
-    }
-    let upsert = knex("final_entities").insert({
-      entity_id: entityId,
-      entity_ref: entityRef,
-      final_entity: JSON.stringify(entity),
-      hash,
-      last_updated_at: knex.fn.now()
-    }).onConflict("entity_id").merge(["final_entity", "hash", "last_updated_at"]);
-    if (!isMySQL) {
-      upsert = upsert.where(
-        knex.raw(
-          "exists (select 1 from stitch_queue where entity_ref = ? and stitch_ticket = ?)",
-          [entityRef, stitchTicket]
-        )
-      );
-    }
-    await upsert;
-    if (!isMySQL) {
-      const written = await knex("final_entities").where("entity_id", entityId).where("hash", hash).select(knex.raw("1")).first();
-      if (!written) {
-        logger.debug(
-          `Entity ${entityRef} is already stitched, skipping write.`
-        );
-        stitchResult = "abandoned";
+    const isPostgres = knex.client.config.client === "pg";
+    const writeOutcome = await util.retryOnDeadlock(
+      () => knex.transaction(async (tx) => {
+        if (!isPostgres) {
+          const ticketValid = await tx("stitch_queue").where("entity_ref", entityRef).where("stitch_ticket", stitchTicket).first();
+          if (!ticketValid) return "abandoned";
+        }
+        let upsert = tx("final_entities").insert(
+          !isPostgres ? {
+            entity_id: entityId,
+            entity_ref: entityRef,
+            final_entity: JSON.stringify(entity),
+            hash,
+            last_updated_at: tx.fn.now()
+          } : tx.raw(
+            "(entity_id, entity_ref, final_entity, hash, last_updated_at) select ?, ?, ?, ?, CURRENT_TIMESTAMP where exists (select 1 from stitch_queue where entity_ref = ? and stitch_ticket = ?)",
+            [
+              entityId,
+              entityRef,
+              JSON.stringify(entity),
+              hash,
+              entityRef,
+              stitchTicket
+            ]
+          )
+        ).onConflict("entity_id").merge(["final_entity", "hash", "last_updated_at"]);
+        if (!isMySQL) {
+          upsert = upsert.where(
+            tx.raw(
+              "exists (select 1 from stitch_queue where entity_ref = ? and stitch_ticket = ?)",
+              [entityRef, stitchTicket]
+            )
+          );
+        }
+        await upsert;
+        if (!isMySQL) {
+          const written = await tx("final_entities").where("entity_id", entityId).where("hash", hash).modify((qb) => {
+            if (isPostgres) {
+              qb.whereExists(
+                tx("stitch_queue").select(tx.raw("1")).where("entity_ref", entityRef).where("stitch_ticket", stitchTicket).where("next_stitch_at", stitchLeaseExpiresAt)
+              );
+            }
+          }).select(tx.raw("1")).first();
+          if (!written) {
+            throw new StitchPublicationSupersededError();
+          }
+        }
+        await syncSearchRows.syncSearchRows(tx, entityId, searchEntries);
+        return "changed";
+      }),
+      knex
+    ).catch((error) => {
+      if (error instanceof StitchPublicationSupersededError) {
         return "abandoned";
       }
+      throw error;
+    });
+    if (writeOutcome === "abandoned") {
+      logger.debug(`Entity ${entityRef} is already stitched, skipping write.`);
+      stitchResult = "abandoned";
+      return "abandoned";
     }
-    await syncSearchRows.syncSearchRows(knex, entityId, searchEntries);
     stitchResult = "succeeded";
     return "changed";
   } catch (error) {
@@ -160,7 +188,7 @@ async function performStitching(options) {
         knex,
         entityRef,
         stitchTicket,
-        result: stitchResult
+        stitchLeaseExpiresAt
       });
     }
   }

package/dist/database/operations/stitcher/performStitching.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"performStitching.cjs.js","sources":["../../../../src/database/operations/stitcher/performStitching.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from '@backstage/catalog-client';\nimport {\n  ANNOTATION_EDIT_URL,\n  ANNOTATION_VIEW_URL,\n  Entity,\n  EntityRelation,\n} from '@backstage/catalog-model';\nimport { AlphaEntity, EntityStatusItem } from '@backstage/catalog-model/alpha';\nimport { SerializedError } from '@backstage/errors';\nimport { Knex } from 'knex';\nimport { createHash } from 'node:crypto';\nimport stableStringify from 'fast-json-stable-stringify';\nimport { DbFinalEntitiesRow, DbStitchQueueRow } from '../../tables';\nimport { buildEntitySearch } from './buildEntitySearch';\nimport { markDeferredStitchCompleted } from './markDeferredStitchCompleted';\nimport { syncSearchRows } from './syncSearchRows';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\nfunction generateStableHash(entity: Entity) {\n  return createHash('sha1')\n    .update(stableStringify({ ...entity }))\n    .digest('hex');\n}\n\n// See https://github.com/facebook/react/blob/f0cf832e1d0c8544c36aa8b310960885a11a847c/packages/react-dom-bindings/src/shared/sanitizeURL.js\nconst scriptProtocolPattern =\n  // eslint-disable-next-line no-control-regex\n  /^[\\u0000-\\u001F ]*j[\\r\\n\\t]*a[\\r\\n\\t]*v[\\r\\n\\t]*a[\\r\\n\\t]*s[\\r\\n\\t]*c[\\r\\n\\t]*r[\\r\\n\\t]*i[\\r\\n\\t]*p[\\r\\n\\t]*t[\\r\\n\\t]*\\:/i;\n\n/**\n * Performs the act of stitching - to take all of the various outputs from the\n * ingestion process, and stitching them together into the final entity JSON\n * shape.\n */\nexport async function performStitching(options: {\n  knex: Knex | Knex.Transaction;\n  logger: LoggerService;\n  entityRef: string;\n  stitchTicket: string;\n}): Promise<'changed' | 'unchanged' | 'abandoned'> {\n  const { knex, logger, entityRef, stitchTicket } = options;\n\n  // The stitch queue is cleaned up on ANY completion — either by deleting\n  // the entry (ticket unchanged) or bumping next_stitch_at (re-stitch\n  // requested). Exceptions are the only case where we skip cleanup, so\n  // the entity gets retried at a later time.\n  let stitchResult: 'succeeded' | 'abandoned' | undefined;\n\n  try {\n    // Selecting from refresh_state (with an optional left join to\n    // final_entities for the previous hash) should yield exactly one row,\n    // except in abnormal cases where the entity was deleted between the\n    // stitch request and now.\n    const [processedResult, relationsResult] = await Promise.all([\n      knex\n        .with('incoming_references', function incomingReferences(builder) {\n          return builder\n            .from('refresh_state_references')\n            .where({ target_entity_ref: entityRef })\n            .count({ count: '*' });\n        })\n        .select({\n          entityId: 'refresh_state.entity_id',\n          processedEntity: 'refresh_state.processed_entity',\n          errors: 'refresh_state.errors',\n          incomingReferenceCount: 'incoming_references.count',\n          previousHash: 'final_entities.hash',\n        })\n        .from('refresh_state')\n        .where({ 'refresh_state.entity_ref': entityRef })\n        .crossJoin(knex.raw('incoming_references'))\n        .leftOuterJoin('final_entities', {\n          'final_entities.entity_id': 'refresh_state.entity_id',\n        }),\n      knex\n        .distinct({\n          relationType: 'type',\n          relationTarget: 'target_entity_ref',\n        })\n        .from('relations')\n        .where({ source_entity_ref: entityRef })\n        .orderBy('relationType', 'asc')\n        .orderBy('relationTarget', 'asc'),\n    ]);\n\n    // If there were no rows returned, it would mean that there was no\n    // matching row even in the refresh_state. This can happen for example\n    // if we emit a relation to something that hasn't been ingested yet.\n    // It's safe to ignore this stitch attempt in that case.\n    if (!processedResult.length) {\n      logger.debug(\n        `Unable to stitch ${entityRef}, item does not exist in refresh state table`,\n      );\n      stitchResult = 'abandoned';\n      return 'abandoned';\n    }\n\n    const {\n      entityId,\n      processedEntity,\n      errors,\n      incomingReferenceCount,\n      previousHash,\n    } = processedResult[0];\n\n    // If there was no processed entity in place, the target hasn't been\n    // through the processing steps yet. It's safe to ignore this stitch\n    // attempt in that case, since another stitch will be triggered when\n    // that processing has finished.\n    if (!processedEntity) {\n      logger.debug(\n        `Unable to stitch ${entityRef}, the entity has not yet been processed`,\n      );\n      stitchResult = 'abandoned';\n      return 'abandoned';\n    }\n\n    // Grab the processed entity and stitch all of the relevant data into\n    // it\n    const entity = JSON.parse(processedEntity) as AlphaEntity;\n    const isOrphan = Number(incomingReferenceCount) === 0;\n    let statusItems: EntityStatusItem[] = [];\n\n    if (isOrphan) {\n      logger.debug(`${entityRef} is an orphan`);\n      entity.metadata.annotations = {\n        ...entity.metadata.annotations,\n        ['backstage.io/orphan']: 'true',\n      };\n    }\n    if (errors) {\n      const parsedErrors = JSON.parse(errors) as SerializedError[];\n      if (Array.isArray(parsedErrors) && parsedErrors.length) {\n        statusItems = parsedErrors.map(e => ({\n          type: ENTITY_STATUS_CATALOG_PROCESSING_TYPE,\n          level: 'error',\n          message: `${e.name}: ${e.message}`,\n          error: e,\n        }));\n      }\n    }\n    // We opt to do this check here as we otherwise can't guarantee that it will be run after all processors\n    for (const annotation of [ANNOTATION_VIEW_URL, ANNOTATION_EDIT_URL]) {\n      const value = entity.metadata.annotations?.[annotation];\n      if (typeof value === 'string' && scriptProtocolPattern.test(value)) {\n        entity.metadata.annotations![annotation] =\n          'https://backstage.io/annotation-rejected-for-security-reasons';\n      }\n    }\n\n    // TODO: entityRef is lower case and should be uppercase in the final\n    // result\n    entity.relations = relationsResult\n      .filter(row => row.relationType /* exclude null row, if relevant */)\n      .map<EntityRelation>(row => ({\n        type: row.relationType!,\n        targetRef: row.relationTarget!,\n      }));\n    if (statusItems.length) {\n      entity.status = {\n        ...entity.status,\n        items: [...(entity.status?.items ?? []), ...statusItems],\n      };\n    }\n\n    // If the output entity was actually not changed, just abort\n    const hash = generateStableHash(entity);\n    if (hash === previousHash) {\n      logger.debug(`Skipped stitching of ${entityRef}, no changes`);\n      stitchResult = 'succeeded';\n      return 'unchanged';\n    }\n\n    entity.metadata.uid = entityId;\n    if (!entity.metadata.etag) {\n      // If the original data source did not have its own etag handling,\n      // use the hash as a good-quality etag\n      entity.metadata.etag = hash;\n    }\n\n    // This may throw if the entity is invalid, so we call it before\n    // the final_entities write, even though we may end up not needing\n    // to write the search index.\n    const searchEntries = buildEntitySearch(entityId, entity);\n\n    // Guard against concurrent stitchers: if our stitch_ticket no longer\n    // matches stitch_queue, another worker has newer data and we should\n    // not overwrite it. On PostgreSQL and SQLite, this is done atomically\n    // via a WHERE on the upsert merge path. MySQL does not support\n    // ON CONFLICT ... DO UPDATE ... WHERE, so it uses a separate check\n    // with a negligible TOCTOU window (self-corrects on the next ~1s\n    // stitch cycle).\n    const isMySQL = String(knex.client.config.client).includes('mysql');\n\n    if (isMySQL) {\n      const ticketValid = await knex<DbStitchQueueRow>('stitch_queue')\n        .where('entity_ref', entityRef)\n        .where('stitch_ticket', stitchTicket)\n        .first();\n      if (!ticketValid) {\n        logger.debug(\n          `Entity ${entityRef} is already stitched, skipping write.`,\n        );\n        stitchResult = 'abandoned';\n        return 'abandoned';\n      }\n    }\n\n    let upsert = knex<DbFinalEntitiesRow>('final_entities')\n      .insert({\n        entity_id: entityId,\n        entity_ref: entityRef,\n        final_entity: JSON.stringify(entity),\n        hash,\n        last_updated_at: knex.fn.now(),\n      })\n      .onConflict('entity_id')\n      .merge(['final_entity', 'hash', 'last_updated_at']);\n\n    if (!isMySQL) {\n      upsert = upsert.where(\n        knex.raw(\n          'exists (select 1 from stitch_queue where entity_ref = ? and stitch_ticket = ?)',\n          [entityRef, stitchTicket],\n        ),\n      );\n    }\n\n    await upsert;\n\n    // Verify the write took effect. INSERT return values vary across\n    // database engines (row IDs vs row counts vs empty arrays), so we\n    // check the hash directly — we already know hash !== previousHash\n    // from the check above, so a mismatch means the write was blocked.\n    if (!isMySQL) {\n      const written = await knex<DbFinalEntitiesRow>('final_entities')\n        .where('entity_id', entityId)\n        .where('hash', hash)\n        .select(knex.raw('1'))\n        .first();\n      if (!written) {\n        logger.debug(\n          `Entity ${entityRef} is already stitched, skipping write.`,\n        );\n        stitchResult = 'abandoned';\n        return 'abandoned';\n      }\n    }\n\n    await syncSearchRows(knex, entityId, searchEntries);\n\n    stitchResult = 'succeeded';\n    return 'changed';\n  } catch (error) {\n    throw error;\n  } finally {\n    if (stitchResult) {\n      await markDeferredStitchCompleted({\n        knex: knex,\n        entityRef,\n        stitchTicket,\n        result: stitchResult,\n      });\n    }\n  }\n}\n"],"names":["createHash","stableStringify","ENTITY_STATUS_CATALOG_PROCESSING_TYPE","ANNOTATION_VIEW_URL","ANNOTATION_EDIT_URL","buildEntitySearch","syncSearchRows","markDeferredStitchCompleted"],"mappings":";;;;;;;;;;;;;;AAkCA,SAAS,mBAAmB,MAAA,EAAgB;AAC1C,EAAA,OAAOA,sBAAA,CAAW,MAAM,CAAA,CACrB,MAAA,CAAOC,gCAAA,CAAgB,EAAE,GAAG,MAAA,EAAQ,CAAC,CAAA,CACrC,MAAA,CAAO,KAAK,CAAA;AACjB;AAGA,MAAM,qBAAA;AAAA;AAAA,EAEJ;AAAA,CAAA;AAOF,eAAsB,iBAAiB,OAAA,EAKY;AACjD,EAAA,MAAM,EAAE,IAAA,EAAM,MAAA,EAAQ,SAAA,EAAW,cAAa,GAAI,OAAA;AAMlD,EAAA,IAAI,YAAA;AAEJ,EAAA,IAAI;AAKF,IAAA,MAAM,CAAC,eAAA,EAAiB,eAAe,CAAA,GAAI,MAAM,QAAQ,GAAA,CAAI;AAAA,MAC3D,IAAA,CACG,IAAA,CAAK,qBAAA,EAAuB,SAAS,mBAAmB,OAAA,EAAS;AAChE,QAAA,OAAO,OAAA,CACJ,IAAA,CAAK,0BAA0B,CAAA,CAC/B,MAAM,EAAE,iBAAA,EAAmB,SAAA,EAAW,CAAA,CACtC,KAAA,CAAM,EAAE,KAAA,EAAO,KAAK,CAAA;AAAA,MACzB,CAAC,EACA,MAAA,CAAO;AAAA,QACN,QAAA,EAAU,yBAAA;AAAA,QACV,eAAA,EAAiB,gCAAA;AAAA,QACjB,MAAA,EAAQ,sBAAA;AAAA,QACR,sBAAA,EAAwB,2BAAA;AAAA,QACxB,YAAA,EAAc;AAAA,OACf,CAAA,CACA,IAAA,CAAK,eAAe,CAAA,CACpB,KAAA,CAAM,EAAE,0BAAA,EAA4B,SAAA,EAAW,CAAA,CAC/C,UAAU,IAAA,CAAK,GAAA,CAAI,qBAAqB,CAAC,CAAA,CACzC,cAAc,gBAAA,EAAkB;AAAA,QAC/B,0BAAA,EAA4B;AAAA,OAC7B,CAAA;AAAA,MACH,KACG,QAAA,CAAS;AAAA,QACR,YAAA,EAAc,MAAA;AAAA,QACd,cAAA,EAAgB;AAAA,OACjB,CAAA,CACA,IAAA,CAAK,WAAW,CAAA,CAChB,MAAM,EAAE,iBAAA,EAAmB,SAAA,EAAW,EACtC,OAAA,CAAQ,cAAA,EAAgB,KAAK,CAAA,CAC7B,OAAA,CAAQ,kBAAkB,KAAK;AAAA,KACnC,CAAA;AAMD,IAAA,IAAI,CAAC,gBAAgB,MAAA,EAAQ;AAC3B,MAAA,MAAA,CAAO,KAAA;AAAA,QACL,oBAAoB,SAAS,CAAA,4CAAA;AAAA,OAC/B;AACA,MAAA,YAAA,GAAe,WAAA;AACf,MAAA,OAAO,WAAA;AAAA,IACT;AAEA,IAAA,MAAM;AAAA,MACJ,QAAA;AAAA,MACA,eAAA;AAAA,MACA,MAAA;AAAA,MACA,sBAAA;AAAA,MACA;AAAA,KACF,GAAI,gBAAgB,CAAC,CAAA;AAMrB,IAAA,IAAI,CAAC,eAAA,EAAiB;AACpB,MAAA,MAAA,CAAO,KAAA;AAAA,QACL,oBAAoB,SAAS,CAAA,uCAAA;AAAA,OAC/B;AACA,MAAA,YAAA,GAAe,WAAA;AACf,MAAA,OAAO,WAAA;AAAA,IACT;AAIA,IAAA,MAAM,MAAA,GAAS,IAAA,CAAK,KAAA,CAAM,eAAe,CAAA;AACzC,IAAA,MAAM,QAAA,GAAW,MAAA,CAAO,sBAAsB,CAAA,KAAM,CAAA;AACpD,IAAA,IAAI,cAAkC,EAAC;AAEvC,IAAA,IAAI,QAAA,EAAU;AACZ,MAAA,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,SAAS,CAAA,aAAA,CAAe,CAAA;AACxC,MAAA,MAAA,CAAO,SAAS,WAAA,GAAc;AAAA,QAC5B,GAAG,OAAO,QAAA,CAAS,WAAA;AAAA,QACnB,CAAC,qBAAqB,GAAG;AAAA,OAC3B;AAAA,IACF;AACA,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,MAAM,YAAA,GAAe,IAAA,CAAK,KAAA,CAAM,MAAM,CAAA;AACtC,MAAA,IAAI,KAAA,CAAM,OAAA,CAAQ,YAAY,CAAA,IAAK,aAAa,MAAA,EAAQ;AACtD,QAAA,WAAA,GAAc,YAAA,CAAa,IAAI,CAAA,CAAA,MAAM;AAAA,UACnC,IAAA,EAAMC,mDAAA;AAAA,UACN,KAAA,EAAO,OAAA;AAAA,UACP,SAAS,CAAA,EAAG,CAAA,CAAE,IAAI,CAAA,EAAA,EAAK,EAAE,OAAO,CAAA,CAAA;AAAA,UAChC,KAAA,EAAO;AAAA,SACT,CAAE,CAAA;AAAA,MACJ;AAAA,IACF;AAEA,IAAA,KAAA,MAAW,UAAA,IAAc,CAACC,gCAAA,EAAqBC,gCAAmB,CAAA,EAAG;AACnE,MAAA,MAAM,KAAA,GAAQ,MAAA,CAAO,QAAA,CAAS,WAAA,GAAc,UAAU,CAAA;AACtD,MAAA,IAAI,OAAO,KAAA,KAAU,QAAA,IAAY,qBAAA,CAAsB,IAAA,CAAK,KAAK,CAAA,EAAG;AAClE,QAAA,MAAA,CAAO,QAAA,CAAS,WAAA,CAAa,UAAU,CAAA,GACrC,+DAAA;AAAA,MACJ;AAAA,IACF;AAIA,IAAA,MAAA,CAAO,YAAY,eAAA,CAChB,MAAA;AAAA,MAAO,SAAO,GAAA,CAAI;AAAA;AAAA,KAAgD,CAClE,IAAoB,CAAA,GAAA,MAAQ;AAAA,MAC3B,MAAM,GAAA,CAAI,YAAA;AAAA,MACV,WAAW,GAAA,CAAI;AAAA,KACjB,CAAE,CAAA;AACJ,IAAA,IAAI,YAAY,MAAA,EAAQ;AACtB,MAAA,MAAA,CAAO,MAAA,GAAS;AAAA,QACd,GAAG,MAAA,CAAO,MAAA;AAAA,QACV,KAAA,EAAO,CAAC,GAAI,MAAA,CAAO,QAAQ,KAAA,IAAS,EAAC,EAAI,GAAG,WAAW;AAAA,OACzD;AAAA,IACF;AAGA,IAAA,MAAM,IAAA,GAAO,mBAAmB,MAAM,CAAA;AACtC,IAAA,IAAI,SAAS,YAAA,EAAc;AACzB,MAAA,MAAA,CAAO,KAAA,CAAM,CAAA,qBAAA,EAAwB,SAAS,CAAA,YAAA,CAAc,CAAA;AAC5D,MAAA,YAAA,GAAe,WAAA;AACf,MAAA,OAAO,WAAA;AAAA,IACT;AAEA,IAAA,MAAA,CAAO,SAAS,GAAA,GAAM,QAAA;AACtB,IAAA,IAAI,CAAC,MAAA,CAAO,QAAA,CAAS,IAAA,EAAM;AAGzB,MAAA,MAAA,CAAO,SAAS,IAAA,GAAO,IAAA;AAAA,IACzB;AAKA,IAAA,MAAM,aAAA,GAAgBC,mCAAA,CAAkB,QAAA,EAAU,MAAM,CAAA;AASxD,IAAA,MAAM,OAAA,GAAU,OAAO,IAAA,CAAK,MAAA,CAAO,OAAO,MAAM,CAAA,CAAE,SAAS,OAAO,CAAA;AAElE,IAAA,IAAI,OAAA,EAAS;AACX,MAAA,MAAM,WAAA,GAAc,MAAM,IAAA,CAAuB,cAAc,CAAA,CAC5D,KAAA,CAAM,YAAA,EAAc,SAAS,CAAA,CAC7B,KAAA,CAAM,eAAA,EAAiB,YAAY,EACnC,KAAA,EAAM;AACT,MAAA,IAAI,CAAC,WAAA,EAAa;AAChB,QAAA,MAAA,CAAO,KAAA;AAAA,UACL,UAAU,SAAS,CAAA,qCAAA;AAAA,SACrB;AACA,QAAA,YAAA,GAAe,WAAA;AACf,QAAA,OAAO,WAAA;AAAA,MACT;AAAA,IACF;AAEA,IAAA,IAAI,MAAA,GAAS,IAAA,CAAyB,gBAAgB,CAAA,CACnD,MAAA,CAAO;AAAA,MACN,SAAA,EAAW,QAAA;AAAA,MACX,UAAA,EAAY,SAAA;AAAA,MACZ,YAAA,EAAc,IAAA,CAAK,SAAA,CAAU,MAAM,CAAA;AAAA,MACnC,IAAA;AAAA,MACA,eAAA,EAAiB,IAAA,CAAK,EAAA,CAAG,GAAA;AAAI,KAC9B,CAAA,CACA,UAAA,CAAW,WAAW,CAAA,CACtB,MAAM,CAAC,cAAA,EAAgB,MAAA,EAAQ,iBAAiB,CAAC,CAAA;AAEpD,IAAA,IAAI,CAAC,OAAA,EAAS;AACZ,MAAA,MAAA,GAAS,MAAA,CAAO,KAAA;AAAA,QACd,IAAA,CAAK,GAAA;AAAA,UACH,gFAAA;AAAA,UACA,CAAC,WAAW,YAAY;AAAA;AAC1B,OACF;AAAA,IACF;AAEA,IAAA,MAAM,MAAA;AAMN,IAAA,IAAI,CAAC,OAAA,EAAS;AACZ,MAAA,MAAM,UAAU,MAAM,IAAA,CAAyB,gBAAgB,CAAA,CAC5D,KAAA,CAAM,aAAa,QAAQ,CAAA,CAC3B,MAAM,MAAA,EAAQ,IAAI,EAClB,MAAA,CAAO,IAAA,CAAK,IAAI,GAAG,CAAC,EACpB,KAAA,EAAM;AACT,MAAA,IAAI,CAAC,OAAA,EAAS;AACZ,QAAA,MAAA,CAAO,KAAA;AAAA,UACL,UAAU,SAAS,CAAA,qCAAA;AAAA,SACrB;AACA,QAAA,YAAA,GAAe,WAAA;AACf,QAAA,OAAO,WAAA;AAAA,MACT;AAAA,IACF;AAEA,IAAA,MAAMC,6BAAA,CAAe,IAAA,EAAM,QAAA,EAAU,aAAa,CAAA;AAElD,IAAA,YAAA,GAAe,WAAA;AACf,IAAA,OAAO,SAAA;AAAA,EACT,SAAS,KAAA,EAAO;AACd,IAAA,MAAM,KAAA;AAAA,EACR,CAAA,SAAE;AACA,IAAA,IAAI,YAAA,EAAc;AAChB,MAAA,MAAMC,uDAAA,CAA4B;AAAA,QAChC,IAAA;AAAA,QACA,SAAA;AAAA,QACA,YAAA;AAAA,QACA,MAAA,EAAQ;AAAA,OACT,CAAA;AAAA,IACH;AAAA,EACF;AACF;;"}
+{"version":3,"file":"performStitching.cjs.js","sources":["../../../../src/database/operations/stitcher/performStitching.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from '@backstage/catalog-client';\nimport {\n  ANNOTATION_EDIT_URL,\n  ANNOTATION_VIEW_URL,\n  Entity,\n  EntityRelation,\n} from '@backstage/catalog-model';\nimport { AlphaEntity, EntityStatusItem } from '@backstage/catalog-model/alpha';\nimport { SerializedError } from '@backstage/errors';\nimport { Knex } from 'knex';\nimport { createHash } from 'node:crypto';\nimport stableStringify from 'fast-json-stable-stringify';\nimport { DbFinalEntitiesRow, DbStitchQueueRow } from '../../tables';\nimport { buildEntitySearch } from './buildEntitySearch';\nimport { markDeferredStitchCompleted } from './markDeferredStitchCompleted';\nimport { syncSearchRows } from './syncSearchRows';\nimport { StitchLeaseExpiresAt } from './getDeferredStitchableEntities';\nimport { LoggerService } from '@backstage/backend-plugin-api';\nimport { retryOnDeadlock } from '../../util';\n\nclass StitchPublicationSupersededError extends Error {}\n\nfunction generateStableHash(entity: Entity) {\n  return createHash('sha1')\n    .update(stableStringify({ ...entity }))\n    .digest('hex');\n}\n\n// See https://github.com/facebook/react/blob/f0cf832e1d0c8544c36aa8b310960885a11a847c/packages/react-dom-bindings/src/shared/sanitizeURL.js\nconst scriptProtocolPattern =\n  // eslint-disable-next-line no-control-regex\n  /^[\\u0000-\\u001F ]*j[\\r\\n\\t]*a[\\r\\n\\t]*v[\\r\\n\\t]*a[\\r\\n\\t]*s[\\r\\n\\t]*c[\\r\\n\\t]*r[\\r\\n\\t]*i[\\r\\n\\t]*p[\\r\\n\\t]*t[\\r\\n\\t]*\\:/i;\n\n/**\n * Performs the act of stitching - to take all of the various outputs from the\n * ingestion process, and stitching them together into the final entity JSON\n * shape.\n */\nexport async function performStitching(options: {\n  knex: Knex | Knex.Transaction;\n  logger: LoggerService;\n  entityRef: string;\n  stitchTicket: string;\n  stitchLeaseExpiresAt: StitchLeaseExpiresAt;\n}): Promise<'changed' | 'unchanged' | 'abandoned'> {\n  const { knex, logger, entityRef, stitchTicket, stitchLeaseExpiresAt } =\n    options;\n\n  // Settle the claim on any completion, without disturbing a successor's\n  // lease. A new request during this lease becomes eligible immediately.\n  // Exceptions skip cleanup so the entity gets retried at a later time.\n  let stitchResult: 'succeeded' | 'abandoned' | undefined;\n\n  try {\n    // Selecting from refresh_state (with an optional left join to\n    // final_entities for the previous hash) should yield exactly one row,\n    // except in abnormal cases where the entity was deleted between the\n    // stitch request and now.\n    const [processedResult, relationsResult] = await Promise.all([\n      knex\n        .with('incoming_references', function incomingReferences(builder) {\n          return builder\n            .from('refresh_state_references')\n            .where({ target_entity_ref: entityRef })\n            .count({ count: '*' });\n        })\n        .select({\n          entityId: 'refresh_state.entity_id',\n          processedEntity: 'refresh_state.processed_entity',\n          errors: 'refresh_state.errors',\n          incomingReferenceCount: 'incoming_references.count',\n          previousHash: 'final_entities.hash',\n        })\n        .from('refresh_state')\n        .where({ 'refresh_state.entity_ref': entityRef })\n        .crossJoin(knex.raw('incoming_references'))\n        .leftOuterJoin('final_entities', {\n          'final_entities.entity_id': 'refresh_state.entity_id',\n        }),\n      knex\n        .distinct({\n          relationType: 'type',\n          relationTarget: 'target_entity_ref',\n        })\n        .from('relations')\n        .where({ source_entity_ref: entityRef })\n        .orderBy('relationType', 'asc')\n        .orderBy('relationTarget', 'asc'),\n    ]);\n\n    // If there were no rows returned, it would mean that there was no\n    // matching row even in the refresh_state. This can happen for example\n    // if we emit a relation to something that hasn't been ingested yet.\n    // It's safe to ignore this stitch attempt in that case.\n    if (!processedResult.length) {\n      logger.debug(\n        `Unable to stitch ${entityRef}, item does not exist in refresh state table`,\n      );\n      stitchResult = 'abandoned';\n      return 'abandoned';\n    }\n\n    const {\n      entityId,\n      processedEntity,\n      errors,\n      incomingReferenceCount,\n      previousHash,\n    } = processedResult[0];\n\n    // If there was no processed entity in place, the target hasn't been\n    // through the processing steps yet. It's safe to ignore this stitch\n    // attempt in that case, since another stitch will be triggered when\n    // that processing has finished.\n    if (!processedEntity) {\n      logger.debug(\n        `Unable to stitch ${entityRef}, the entity has not yet been processed`,\n      );\n      stitchResult = 'abandoned';\n      return 'abandoned';\n    }\n\n    // Grab the processed entity and stitch all of the relevant data into\n    // it\n    const entity = JSON.parse(processedEntity) as AlphaEntity;\n    const isOrphan = Number(incomingReferenceCount) === 0;\n    let statusItems: EntityStatusItem[] = [];\n\n    if (isOrphan) {\n      logger.debug(`${entityRef} is an orphan`);\n      entity.metadata.annotations = {\n        ...entity.metadata.annotations,\n        ['backstage.io/orphan']: 'true',\n      };\n    }\n    if (errors) {\n      const parsedErrors = JSON.parse(errors) as SerializedError[];\n      if (Array.isArray(parsedErrors) && parsedErrors.length) {\n        statusItems = parsedErrors.map(e => ({\n          type: ENTITY_STATUS_CATALOG_PROCESSING_TYPE,\n          level: 'error',\n          message: `${e.name}: ${e.message}`,\n          error: e,\n        }));\n      }\n    }\n    // We opt to do this check here as we otherwise can't guarantee that it will be run after all processors\n    for (const annotation of [ANNOTATION_VIEW_URL, ANNOTATION_EDIT_URL]) {\n      const value = entity.metadata.annotations?.[annotation];\n      if (typeof value === 'string' && scriptProtocolPattern.test(value)) {\n        entity.metadata.annotations![annotation] =\n          'https://backstage.io/annotation-rejected-for-security-reasons';\n      }\n    }\n\n    // TODO: entityRef is lower case and should be uppercase in the final\n    // result\n    entity.relations = relationsResult\n      .filter(row => row.relationType /* exclude null row, if relevant */)\n      .map<EntityRelation>(row => ({\n        type: row.relationType!,\n        targetRef: row.relationTarget!,\n      }));\n    if (statusItems.length) {\n      entity.status = {\n        ...entity.status,\n        items: [...(entity.status?.items ?? []), ...statusItems],\n      };\n    }\n\n    // If the output entity was actually not changed, just abort\n    const hash = generateStableHash(entity);\n    if (hash === previousHash) {\n      logger.debug(`Skipped stitching of ${entityRef}, no changes`);\n      stitchResult = 'succeeded';\n      return 'unchanged';\n    }\n\n    entity.metadata.uid = entityId;\n    if (!entity.metadata.etag) {\n      // If the original data source did not have its own etag handling,\n      // use the hash as a good-quality etag\n      entity.metadata.etag = hash;\n    }\n\n    // This may throw if the entity is invalid, so we call it before\n    // the final_entities write, even though we may end up not needing\n    // to write the search index.\n    const searchEntries = buildEntitySearch(entityId, entity);\n\n    // Guard against concurrent stitchers: if our stitch_ticket no longer\n    // matches stitch_queue, another worker has newer data and we should\n    // not overwrite it. PostgreSQL guards both insert and merge in the\n    // publication statement and rechecks the captured claim after the upsert.\n    // SQLite checks inside its transaction as well\n    // as guarding the merge path. MySQL does not support\n    // ON CONFLICT ... DO UPDATE ... WHERE, so its separate check retains\n    // a best-effort TOCTOU window. Do not turn it into a queue lock here:\n    // that would introduce a new lock-order dependency during publication.\n    const isMySQL = String(knex.client.config.client).includes('mysql');\n    const isPostgres = knex.client.config.client === 'pg';\n\n    // The final_entities row and the search index rows have to land\n    // together. If the search write fails on its own, the next stitch\n    // attempt reads back the hash we just wrote and returns early as\n    // unchanged, so the entity keeps a stale search index for good. It\n    // stays readable by direct lookup and drops out of every filtered or\n    // sorted list query.\n    const writeOutcome = await retryOnDeadlock(\n      () =>\n        knex.transaction(async tx => {\n          // Recheck on every attempt: a request or reclaimed claim may have\n          // superseded this worker while its previous transaction rolled back.\n          // Do not lock the queue here; processing locks refresh state before queue.\n          if (!isPostgres) {\n            const ticketValid = await tx<DbStitchQueueRow>('stitch_queue')\n              .where('entity_ref', entityRef)\n              .where('stitch_ticket', stitchTicket)\n              .first();\n            if (!ticketValid) return 'abandoned' as const;\n          }\n          // Knex's typed insert overload excludes raw INSERT SELECT bodies.\n          let upsert = tx('final_entities')\n            .insert(\n              !isPostgres\n                ? {\n                    entity_id: entityId,\n                    entity_ref: entityRef,\n                    final_entity: JSON.stringify(entity),\n                    hash,\n                    last_updated_at: tx.fn.now(),\n                  }\n                : tx.raw(\n                    '(entity_id, entity_ref, final_entity, hash, last_updated_at) ' +\n                      'select ?, ?, ?, ?, CURRENT_TIMESTAMP ' +\n                      'where exists (select 1 from stitch_queue where entity_ref = ? and stitch_ticket = ?)',\n                    [\n                      entityId,\n                      entityRef,\n                      JSON.stringify(entity),\n                      hash,\n                      entityRef,\n                      stitchTicket,\n                    ],\n                  ),\n            )\n            .onConflict('entity_id')\n            .merge(['final_entity', 'hash', 'last_updated_at']);\n\n          if (!isMySQL) {\n            upsert = upsert.where(\n              tx.raw(\n                'exists (select 1 from stitch_queue where entity_ref = ? and stitch_ticket = ?)',\n                [entityRef, stitchTicket],\n              ),\n            );\n          }\n\n          await upsert;\n\n          // Verify the write took effect. On PostgreSQL, also recheck ownership\n          // using this statement's fresh Read Committed snapshot: the upsert's\n          // snapshot may predate a reclaim that committed while it was running.\n          // A successful upsert holds the final-row lock until commit, so a\n          // successor cannot publish past us after this check. Do not lock the\n          // queue, which would introduce a publication/deletion lock inversion.\n          if (!isMySQL) {\n            const written = await tx<DbFinalEntitiesRow>('final_entities')\n              .where('entity_id', entityId)\n              .where('hash', hash)\n              .modify(qb => {\n                if (isPostgres) {\n                  qb.whereExists(\n                    tx<DbStitchQueueRow>('stitch_queue')\n                      .select(tx.raw('1'))\n                      .where('entity_ref', entityRef)\n                      .where('stitch_ticket', stitchTicket)\n                      .where('next_stitch_at', stitchLeaseExpiresAt),\n                  );\n                }\n              })\n              .select(tx.raw('1'))\n              .first();\n            if (!written) {\n              // Returning normally would commit any stale write already made.\n              throw new StitchPublicationSupersededError();\n            }\n          }\n\n          await syncSearchRows(tx, entityId, searchEntries);\n\n          return 'changed' as const;\n        }),\n      knex,\n    ).catch(error => {\n      // Knex has rolled the publication back before rejecting the transaction.\n      // Supersession is a normal abandonment, not a failure or deadlock retry.\n      if (error instanceof StitchPublicationSupersededError) {\n        return 'abandoned' as const;\n      }\n      throw error;\n    });\n\n    if (writeOutcome === 'abandoned') {\n      logger.debug(`Entity ${entityRef} is already stitched, skipping write.`);\n      stitchResult = 'abandoned';\n      return 'abandoned';\n    }\n\n    stitchResult = 'succeeded';\n    return 'changed';\n  } catch (error) {\n    throw error;\n  } finally {\n    if (stitchResult) {\n      await markDeferredStitchCompleted({\n        knex: knex,\n        entityRef,\n        stitchTicket,\n        stitchLeaseExpiresAt,\n      });\n    }\n  }\n}\n"],"names":["createHash","stableStringify","ENTITY_STATUS_CATALOG_PROCESSING_TYPE","ANNOTATION_VIEW_URL","ANNOTATION_EDIT_URL","buildEntitySearch","retryOnDeadlock","syncSearchRows","markDeferredStitchCompleted"],"mappings":";;;;;;;;;;;;;;;AAoCA,MAAM,yCAAyC,KAAA,CAAM;AAAC;AAEtD,SAAS,mBAAmB,MAAA,EAAgB;AAC1C,EAAA,OAAOA,sBAAA,CAAW,MAAM,CAAA,CACrB,MAAA,CAAOC,gCAAA,CAAgB,EAAE,GAAG,MAAA,EAAQ,CAAC,CAAA,CACrC,MAAA,CAAO,KAAK,CAAA;AACjB;AAGA,MAAM,qBAAA;AAAA;AAAA,EAEJ;AAAA,CAAA;AAOF,eAAsB,iBAAiB,OAAA,EAMY;AACjD,EAAA,MAAM,EAAE,IAAA,EAAM,MAAA,EAAQ,SAAA,EAAW,YAAA,EAAc,sBAAqB,GAClE,OAAA;AAKF,EAAA,IAAI,YAAA;AAEJ,EAAA,IAAI;AAKF,IAAA,MAAM,CAAC,eAAA,EAAiB,eAAe,CAAA,GAAI,MAAM,QAAQ,GAAA,CAAI;AAAA,MAC3D,IAAA,CACG,IAAA,CAAK,qBAAA,EAAuB,SAAS,mBAAmB,OAAA,EAAS;AAChE,QAAA,OAAO,OAAA,CACJ,IAAA,CAAK,0BAA0B,CAAA,CAC/B,MAAM,EAAE,iBAAA,EAAmB,SAAA,EAAW,CAAA,CACtC,KAAA,CAAM,EAAE,KAAA,EAAO,KAAK,CAAA;AAAA,MACzB,CAAC,EACA,MAAA,CAAO;AAAA,QACN,QAAA,EAAU,yBAAA;AAAA,QACV,eAAA,EAAiB,gCAAA;AAAA,QACjB,MAAA,EAAQ,sBAAA;AAAA,QACR,sBAAA,EAAwB,2BAAA;AAAA,QACxB,YAAA,EAAc;AAAA,OACf,CAAA,CACA,IAAA,CAAK,eAAe,CAAA,CACpB,KAAA,CAAM,EAAE,0BAAA,EAA4B,SAAA,EAAW,CAAA,CAC/C,UAAU,IAAA,CAAK,GAAA,CAAI,qBAAqB,CAAC,CAAA,CACzC,cAAc,gBAAA,EAAkB;AAAA,QAC/B,0BAAA,EAA4B;AAAA,OAC7B,CAAA;AAAA,MACH,KACG,QAAA,CAAS;AAAA,QACR,YAAA,EAAc,MAAA;AAAA,QACd,cAAA,EAAgB;AAAA,OACjB,CAAA,CACA,IAAA,CAAK,WAAW,CAAA,CAChB,MAAM,EAAE,iBAAA,EAAmB,SAAA,EAAW,EACtC,OAAA,CAAQ,cAAA,EAAgB,KAAK,CAAA,CAC7B,OAAA,CAAQ,kBAAkB,KAAK;AAAA,KACnC,CAAA;AAMD,IAAA,IAAI,CAAC,gBAAgB,MAAA,EAAQ;AAC3B,MAAA,MAAA,CAAO,KAAA;AAAA,QACL,oBAAoB,SAAS,CAAA,4CAAA;AAAA,OAC/B;AACA,MAAA,YAAA,GAAe,WAAA;AACf,MAAA,OAAO,WAAA;AAAA,IACT;AAEA,IAAA,MAAM;AAAA,MACJ,QAAA;AAAA,MACA,eAAA;AAAA,MACA,MAAA;AAAA,MACA,sBAAA;AAAA,MACA;AAAA,KACF,GAAI,gBAAgB,CAAC,CAAA;AAMrB,IAAA,IAAI,CAAC,eAAA,EAAiB;AACpB,MAAA,MAAA,CAAO,KAAA;AAAA,QACL,oBAAoB,SAAS,CAAA,uCAAA;AAAA,OAC/B;AACA,MAAA,YAAA,GAAe,WAAA;AACf,MAAA,OAAO,WAAA;AAAA,IACT;AAIA,IAAA,MAAM,MAAA,GAAS,IAAA,CAAK,KAAA,CAAM,eAAe,CAAA;AACzC,IAAA,MAAM,QAAA,GAAW,MAAA,CAAO,sBAAsB,CAAA,KAAM,CAAA;AACpD,IAAA,IAAI,cAAkC,EAAC;AAEvC,IAAA,IAAI,QAAA,EAAU;AACZ,MAAA,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,SAAS,CAAA,aAAA,CAAe,CAAA;AACxC,MAAA,MAAA,CAAO,SAAS,WAAA,GAAc;AAAA,QAC5B,GAAG,OAAO,QAAA,CAAS,WAAA;AAAA,QACnB,CAAC,qBAAqB,GAAG;AAAA,OAC3B;AAAA,IACF;AACA,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,MAAM,YAAA,GAAe,IAAA,CAAK,KAAA,CAAM,MAAM,CAAA;AACtC,MAAA,IAAI,KAAA,CAAM,OAAA,CAAQ,YAAY,CAAA,IAAK,aAAa,MAAA,EAAQ;AACtD,QAAA,WAAA,GAAc,YAAA,CAAa,IAAI,CAAA,CAAA,MAAM;AAAA,UACnC,IAAA,EAAMC,mDAAA;AAAA,UACN,KAAA,EAAO,OAAA;AAAA,UACP,SAAS,CAAA,EAAG,CAAA,CAAE,IAAI,CAAA,EAAA,EAAK,EAAE,OAAO,CAAA,CAAA;AAAA,UAChC,KAAA,EAAO;AAAA,SACT,CAAE,CAAA;AAAA,MACJ;AAAA,IACF;AAEA,IAAA,KAAA,MAAW,UAAA,IAAc,CAACC,gCAAA,EAAqBC,gCAAmB,CAAA,EAAG;AACnE,MAAA,MAAM,KAAA,GAAQ,MAAA,CAAO,QAAA,CAAS,WAAA,GAAc,UAAU,CAAA;AACtD,MAAA,IAAI,OAAO,KAAA,KAAU,QAAA,IAAY,qBAAA,CAAsB,IAAA,CAAK,KAAK,CAAA,EAAG;AAClE,QAAA,MAAA,CAAO,QAAA,CAAS,WAAA,CAAa,UAAU,CAAA,GACrC,+DAAA;AAAA,MACJ;AAAA,IACF;AAIA,IAAA,MAAA,CAAO,YAAY,eAAA,CAChB,MAAA;AAAA,MAAO,SAAO,GAAA,CAAI;AAAA;AAAA,KAAgD,CAClE,IAAoB,CAAA,GAAA,MAAQ;AAAA,MAC3B,MAAM,GAAA,CAAI,YAAA;AAAA,MACV,WAAW,GAAA,CAAI;AAAA,KACjB,CAAE,CAAA;AACJ,IAAA,IAAI,YAAY,MAAA,EAAQ;AACtB,MAAA,MAAA,CAAO,MAAA,GAAS;AAAA,QACd,GAAG,MAAA,CAAO,MAAA;AAAA,QACV,KAAA,EAAO,CAAC,GAAI,MAAA,CAAO,QAAQ,KAAA,IAAS,EAAC,EAAI,GAAG,WAAW;AAAA,OACzD;AAAA,IACF;AAGA,IAAA,MAAM,IAAA,GAAO,mBAAmB,MAAM,CAAA;AACtC,IAAA,IAAI,SAAS,YAAA,EAAc;AACzB,MAAA,MAAA,CAAO,KAAA,CAAM,CAAA,qBAAA,EAAwB,SAAS,CAAA,YAAA,CAAc,CAAA;AAC5D,MAAA,YAAA,GAAe,WAAA;AACf,MAAA,OAAO,WAAA;AAAA,IACT;AAEA,IAAA,MAAA,CAAO,SAAS,GAAA,GAAM,QAAA;AACtB,IAAA,IAAI,CAAC,MAAA,CAAO,QAAA,CAAS,IAAA,EAAM;AAGzB,MAAA,MAAA,CAAO,SAAS,IAAA,GAAO,IAAA;AAAA,IACzB;AAKA,IAAA,MAAM,aAAA,GAAgBC,mCAAA,CAAkB,QAAA,EAAU,MAAM,CAAA;AAWxD,IAAA,MAAM,OAAA,GAAU,OAAO,IAAA,CAAK,MAAA,CAAO,OAAO,MAAM,CAAA,CAAE,SAAS,OAAO,CAAA;AAClE,IAAA,MAAM,UAAA,GAAa,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,MAAA,KAAW,IAAA;AAQjD,IAAA,MAAM,eAAe,MAAMC,oBAAA;AAAA,MACzB,MACE,IAAA,CAAK,WAAA,CAAY,OAAM,EAAA,KAAM;AAI3B,QAAA,IAAI,CAAC,UAAA,EAAY;AACf,UAAA,MAAM,WAAA,GAAc,MAAM,EAAA,CAAqB,cAAc,CAAA,CAC1D,KAAA,CAAM,YAAA,EAAc,SAAS,CAAA,CAC7B,KAAA,CAAM,eAAA,EAAiB,YAAY,EACnC,KAAA,EAAM;AACT,UAAA,IAAI,CAAC,aAAa,OAAO,WAAA;AAAA,QAC3B;AAEA,QAAA,IAAI,MAAA,GAAS,EAAA,CAAG,gBAAgB,CAAA,CAC7B,MAAA;AAAA,UACC,CAAC,UAAA,GACG;AAAA,YACE,SAAA,EAAW,QAAA;AAAA,YACX,UAAA,EAAY,SAAA;AAAA,YACZ,YAAA,EAAc,IAAA,CAAK,SAAA,CAAU,MAAM,CAAA;AAAA,YACnC,IAAA;AAAA,YACA,eAAA,EAAiB,EAAA,CAAG,EAAA,CAAG,GAAA;AAAI,cAE7B,EAAA,CAAG,GAAA;AAAA,YACD,wLAAA;AAAA,YAGA;AAAA,cACE,QAAA;AAAA,cACA,SAAA;AAAA,cACA,IAAA,CAAK,UAAU,MAAM,CAAA;AAAA,cACrB,IAAA;AAAA,cACA,SAAA;AAAA,cACA;AAAA;AACF;AACF,SACN,CACC,WAAW,WAAW,CAAA,CACtB,MAAM,CAAC,cAAA,EAAgB,MAAA,EAAQ,iBAAiB,CAAC,CAAA;AAEpD,QAAA,IAAI,CAAC,OAAA,EAAS;AACZ,UAAA,MAAA,GAAS,MAAA,CAAO,KAAA;AAAA,YACd,EAAA,CAAG,GAAA;AAAA,cACD,gFAAA;AAAA,cACA,CAAC,WAAW,YAAY;AAAA;AAC1B,WACF;AAAA,QACF;AAEA,QAAA,MAAM,MAAA;AAQN,QAAA,IAAI,CAAC,OAAA,EAAS;AACZ,UAAA,MAAM,OAAA,GAAU,MAAM,EAAA,CAAuB,gBAAgB,EAC1D,KAAA,CAAM,WAAA,EAAa,QAAQ,CAAA,CAC3B,KAAA,CAAM,MAAA,EAAQ,IAAI,CAAA,CAClB,OAAO,CAAA,EAAA,KAAM;AACZ,YAAA,IAAI,UAAA,EAAY;AACd,cAAA,EAAA,CAAG,WAAA;AAAA,gBACD,GAAqB,cAAc,CAAA,CAChC,OAAO,EAAA,CAAG,GAAA,CAAI,GAAG,CAAC,CAAA,CAClB,MAAM,YAAA,EAAc,SAAS,EAC7B,KAAA,CAAM,eAAA,EAAiB,YAAY,CAAA,CACnC,KAAA,CAAM,kBAAkB,oBAAoB;AAAA,eACjD;AAAA,YACF;AAAA,UACF,CAAC,EACA,MAAA,CAAO,EAAA,CAAG,IAAI,GAAG,CAAC,EAClB,KAAA,EAAM;AACT,UAAA,IAAI,CAAC,OAAA,EAAS;AAEZ,YAAA,MAAM,IAAI,gCAAA,EAAiC;AAAA,UAC7C;AAAA,QACF;AAEA,QAAA,MAAMC,6BAAA,CAAe,EAAA,EAAI,QAAA,EAAU,aAAa,CAAA;AAEhD,QAAA,OAAO,SAAA;AAAA,MACT,CAAC,CAAA;AAAA,MACH;AAAA,KACF,CAAE,MAAM,CAAA,KAAA,KAAS;AAGf,MAAA,IAAI,iBAAiB,gCAAA,EAAkC;AACrD,QAAA,OAAO,WAAA;AAAA,MACT;AACA,MAAA,MAAM,KAAA;AAAA,IACR,CAAC,CAAA;AAED,IAAA,IAAI,iBAAiB,WAAA,EAAa;AAChC,MAAA,MAAA,CAAO,KAAA,CAAM,CAAA,OAAA,EAAU,SAAS,CAAA,qCAAA,CAAuC,CAAA;AACvE,MAAA,YAAA,GAAe,WAAA;AACf,MAAA,OAAO,WAAA;AAAA,IACT;AAEA,IAAA,YAAA,GAAe,WAAA;AACf,IAAA,OAAO,SAAA;AAAA,EACT,SAAS,KAAA,EAAO;AACd,IAAA,MAAM,KAAA;AAAA,EACR,CAAA,SAAE;AACA,IAAA,IAAI,YAAA,EAAc;AAChB,MAAA,MAAMC,uDAAA,CAA4B;AAAA,QAChC,IAAA;AAAA,QACA,SAAA;AAAA,QACA,YAAA;AAAA,QACA;AAAA,OACD,CAAA;AAAA,IACH;AAAA,EACF;AACF;;"}

package/dist/database/operations/stitcher/syncSearchRows.cjs.js

@@ -72,49 +72,34 @@ async function syncPostgres(knex, entityId, searchEntries) {
     ]
   );
 }
-const MYSQL_DEADLOCK_MAX_RETRIES = 3;
 async function syncMysql(knex, entityId, searchEntries) {
-  for (let attempt = 1; ; attempt++) {
-    try {
-      await knex.transaction(async (trx) => {
-        await trx.raw(
-          "CREATE TEMPORARY TABLE IF NOT EXISTS `_desired_search` (`key` VARCHAR(255) NOT NULL, `value` VARCHAR(255) NULL, `original_value` VARCHAR(255) NULL)"
-        );
-        await trx.raw("DELETE FROM `_desired_search`");
-        if (searchEntries.length > 0) {
-          await trx.batchInsert(
-            "_desired_search",
-            searchEntries.map((r) => ({
-              key: r.key,
-              value: r.value,
-              original_value: r.original_value
-            })),
-            util.BATCH_SIZE
-          );
-        }
-        await trx.raw(
-          "DELETE s FROM `search` s WHERE s.entity_id = ? AND NOT EXISTS (  SELECT 1 FROM `_desired_search` d  WHERE d.`key` = s.`key`    AND d.`value` <=> s.`value`    AND BINARY d.`original_value` <=> BINARY s.`original_value`)",
-          [entityId]
-        );
-        await trx.raw(
-          "INSERT INTO `search` (entity_id, `key`, `value`, `original_value`) SELECT ?, d.`key`, d.`value`, d.`original_value` FROM `_desired_search` d WHERE NOT EXISTS (  SELECT 1 FROM `search` s  WHERE s.entity_id = ?    AND s.`key` = d.`key`    AND s.`value` <=> d.`value`    AND BINARY s.`original_value` <=> BINARY d.`original_value`)",
-          [entityId, entityId]
-        );
-      });
-      return;
-    } catch (error) {
-      if (error?.errno === 1213 && attempt < MYSQL_DEADLOCK_MAX_RETRIES) {
-        continue;
-      }
-      throw error;
-    }
+  await knex.raw(
+    "CREATE TEMPORARY TABLE IF NOT EXISTS `_desired_search` (`key` VARCHAR(255) NOT NULL, `value` VARCHAR(255) NULL, `original_value` VARCHAR(255) NULL)"
+  );
+  await knex.raw("DELETE FROM `_desired_search`");
+  if (searchEntries.length > 0) {
+    await knex.batchInsert(
+      "_desired_search",
+      searchEntries.map((r) => ({
+        key: r.key,
+        value: r.value,
+        original_value: r.original_value
+      })),
+      util.BATCH_SIZE
+    ).transacting(knex);
   }
+  await knex.raw(
+    "DELETE s FROM `search` s WHERE s.entity_id = ? AND NOT EXISTS (  SELECT 1 FROM `_desired_search` d  WHERE d.`key` = s.`key`    AND d.`value` <=> s.`value`    AND BINARY d.`original_value` <=> BINARY s.`original_value`)",
+    [entityId]
+  );
+  await knex.raw(
+    "INSERT INTO `search` (entity_id, `key`, `value`, `original_value`) SELECT ?, d.`key`, d.`value`, d.`original_value` FROM `_desired_search` d WHERE NOT EXISTS (  SELECT 1 FROM `search` s  WHERE s.entity_id = ?    AND s.`key` = d.`key`    AND s.`value` <=> d.`value`    AND BINARY s.`original_value` <=> BINARY d.`original_value`)",
+    [entityId, entityId]
+  );
 }
 async function syncBulkReplace(knex, entityId, searchEntries) {
-  await knex.transaction(async (trx) => {
-    await trx("search").where({ entity_id: entityId }).delete();
-    await trx.batchInsert("search", searchEntries, util.BATCH_SIZE);
-  });
+  await knex("search").where({ entity_id: entityId }).delete();
+  await knex.batchInsert("search", searchEntries, util.BATCH_SIZE).transacting(knex);
 }
 
 exports.syncSearchRows = syncSearchRows;

package/dist/database/operations/stitcher/syncSearchRows.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"syncSearchRows.cjs.js","sources":["../../../../src/database/operations/stitcher/syncSearchRows.ts"],"sourcesContent":["/*\n * Copyright 2026 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport { DbSearchRow } from '../../tables';\nimport { BATCH_SIZE, NULL_SENTINEL } from './util';\n\nfunction filterSentinelValues(entries: DbSearchRow[]): DbSearchRow[] {\n  return entries.filter(\n    r => r.value !== NULL_SENTINEL && r.original_value !== NULL_SENTINEL,\n  );\n}\n\n/**\n * Synchronizes the search table rows for a given entity, applying only the\n * minimal set of changes needed. Rows that already exist with the correct\n * values are left untouched, new rows are inserted, and stale rows are\n * deleted — minimizing write churn, dead tuples, and WAL traffic.\n *\n * Uses database-specific strategies:\n * - Postgres: Single writable CTE with unnest (one round-trip, no DDL)\n * - MySQL: Temporary table merge (two queries in a transaction)\n * - SQLite: Simple bulk replace (sufficient for dev/test)\n */\nexport async function syncSearchRows(\n  knex: Knex | Knex.Transaction,\n  entityId: string,\n  searchEntries: DbSearchRow[],\n): Promise<void> {\n  // Dedup by (key, value) — the UNIQUE constraint on (entity_id, key, value)\n  // rejects duplicates, and the same lowercased value with different original\n  // casing is semantically a single entry. Keep the first occurrence, which\n  // matches the first-wins semantics of buildEntitySearch so that both layers\n  // consistently pick the same original_value for a given input order.\n  const dedupMap = new Map<string, DbSearchRow>();\n  for (const entry of searchEntries) {\n    const k = `${entry.key}\\0${\n      entry.value === null ? NULL_SENTINEL : entry.value\n    }`;\n    if (!dedupMap.has(k)) {\n      dedupMap.set(k, entry);\n    }\n  }\n  const deduped = [...dedupMap.values()];\n\n  const client = knex.client.config.client;\n\n  if (client === 'pg') {\n    await syncPostgres(knex, entityId, deduped);\n  } else if (client.includes('mysql')) {\n    await syncMysql(knex, entityId, deduped);\n  } else {\n    await syncBulkReplace(knex, entityId, deduped);\n  }\n}\n\n// ---------------------------------------------------------------------------\n// Postgres: writable CTE + unnest\n//\n// All CTE branches see the same pre-modification snapshot, so the DELETE\n// and INSERT do not interfere with each other. This is a single atomic\n// statement — no explicit transaction wrapper needed.\n//\n// Nullable columns use COALESCE(x, chr(1)) instead of IS NOT DISTINCT FROM\n// so that Postgres can include all three columns in the Hash Cond of the\n// anti-join, rather than pushing nullable comparisons into a Join Filter\n// that degrades to O(n*m) when many rows share the same key. chr(1) (SOH\n// control character) is used as the NULL sentinel — it cannot appear in\n// real entity values since they are human-readable strings.\n// ---------------------------------------------------------------------------\nasync function syncPostgres(\n  knex: Knex | Knex.Transaction,\n  entityId: string,\n  searchEntries: DbSearchRow[],\n): Promise<void> {\n  const filtered = filterSentinelValues(searchEntries);\n  const keys = filtered.map(r => r.key);\n  const values = filtered.map(r => r.value);\n  const originalValues = filtered.map(r => r.original_value);\n\n  await knex.raw(\n    `\n    WITH desired AS (\n      SELECT *\n      FROM unnest(?::text[], ?::text[], ?::text[])\n        AS d(key, value, original_value)\n    ),\n    deleted AS (\n      DELETE FROM \"search\" s\n      WHERE s.entity_id = ?\n        AND NOT EXISTS (\n          SELECT 1 FROM desired d\n          WHERE d.key = s.key\n            AND COALESCE(d.value, chr(1)) = COALESCE(s.value, chr(1))\n            AND COALESCE(d.original_value, chr(1)) = COALESCE(s.original_value, chr(1))\n        )\n    )\n    INSERT INTO \"search\" (entity_id, key, value, original_value)\n    SELECT ?, d.key, d.value, d.original_value\n    FROM desired d\n    WHERE NOT EXISTS (\n      SELECT 1 FROM \"search\" s\n      WHERE s.entity_id = ?\n        AND s.key = d.key\n        AND COALESCE(s.value, chr(1)) = COALESCE(d.value, chr(1))\n        AND COALESCE(s.original_value, chr(1)) = COALESCE(d.original_value, chr(1))\n    )\n    ON CONFLICT (entity_id, key, value)\n    DO UPDATE SET original_value = EXCLUDED.original_value\n    `,\n    [\n      keys,\n      // Knex's types don't include nullable PostgreSQL array parameters,\n      // even though they are supported by the pg driver.\n      values as Knex.RawBinding,\n      originalValues as Knex.RawBinding,\n      entityId,\n      entityId,\n      entityId,\n    ],\n  );\n}\n\n// ---------------------------------------------------------------------------\n// MySQL: temporary table merge with deadlock retry\n//\n// MySQL does not support data-modifying CTEs, so we materialize the desired\n// state into a session-scoped temporary table and then merge it into the\n// real table with two queries. The temp table is created inside the\n// transaction to guarantee it exists on the same pooled connection.\n// CREATE/DROP TEMPORARY TABLE does not cause an implicit commit in MySQL\n// (unlike regular DDL), so this is transaction-safe.\n//\n// InnoDB's next-key (gap) locking can cause deadlocks between concurrent\n// transactions operating on different entity_ids when their gap locks\n// overlap on shared index pages. We retry on deadlock (error 1213) since\n// the operation is idempotent.\n// ---------------------------------------------------------------------------\nconst MYSQL_DEADLOCK_MAX_RETRIES = 3;\n\nasync function syncMysql(\n  knex: Knex | Knex.Transaction,\n  entityId: string,\n  searchEntries: DbSearchRow[],\n): Promise<void> {\n  for (let attempt = 1; ; attempt++) {\n    try {\n      await knex.transaction(async trx => {\n        // Create the temp table inside the transaction so it's guaranteed\n        // to be on the same pooled connection as the merge queries.\n        // CREATE TEMPORARY TABLE does not cause an implicit commit in\n        // MySQL (unlike regular CREATE TABLE), so this is safe.\n        await trx.raw(\n          'CREATE TEMPORARY TABLE IF NOT EXISTS `_desired_search` (' +\n            '`key` VARCHAR(255) NOT NULL, ' +\n            '`value` VARCHAR(255) NULL, ' +\n            '`original_value` VARCHAR(255) NULL' +\n            ')',\n        );\n        // Clear stale data from any previous call on this connection.\n        // Uses DELETE (DML) instead of TRUNCATE (DDL) to avoid an\n        // implicit commit that would break transaction atomicity.\n        await trx.raw('DELETE FROM `_desired_search`');\n\n        if (searchEntries.length > 0) {\n          await trx.batchInsert(\n            '_desired_search',\n            searchEntries.map(r => ({\n              key: r.key,\n              value: r.value,\n              original_value: r.original_value,\n            })),\n            BATCH_SIZE,\n          );\n        }\n\n        // Delete rows that are no longer in the desired set\n        await trx.raw(\n          'DELETE s FROM `search` s ' +\n            'WHERE s.entity_id = ? ' +\n            'AND NOT EXISTS (' +\n            '  SELECT 1 FROM `_desired_search` d' +\n            '  WHERE d.`key` = s.`key`' +\n            '    AND d.`value` <=> s.`value`' +\n            '    AND BINARY d.`original_value` <=> BINARY s.`original_value`' +\n            ')',\n          [entityId],\n        );\n\n        // Insert rows that are new in the desired set. The original_value\n        // column preserves the original casing and must be compared with\n        // BINARY to avoid MySQL's default case-insensitive collation\n        // treating e.g. \"Team-A\" and \"team-a\" as equal.\n        await trx.raw(\n          'INSERT INTO `search` (entity_id, `key`, `value`, `original_value`) ' +\n            'SELECT ?, d.`key`, d.`value`, d.`original_value` ' +\n            'FROM `_desired_search` d ' +\n            'WHERE NOT EXISTS (' +\n            '  SELECT 1 FROM `search` s' +\n            '  WHERE s.entity_id = ?' +\n            '    AND s.`key` = d.`key`' +\n            '    AND s.`value` <=> d.`value`' +\n            '    AND BINARY s.`original_value` <=> BINARY d.`original_value`' +\n            ')',\n          [entityId, entityId],\n        );\n      });\n      return;\n    } catch (error) {\n      // MySQL error 1213: ER_LOCK_DEADLOCK\n      if (\n        (error as any)?.errno === 1213 &&\n        attempt < MYSQL_DEADLOCK_MAX_RETRIES\n      ) {\n        continue;\n      }\n      throw error;\n    }\n  }\n}\n\n// ---------------------------------------------------------------------------\n// SQLite (and fallback): bulk replace\n// ---------------------------------------------------------------------------\nasync function syncBulkReplace(\n  knex: Knex | Knex.Transaction,\n  entityId: string,\n  searchEntries: DbSearchRow[],\n): Promise<void> {\n  await knex.transaction(async trx => {\n    await trx<DbSearchRow>('search').where({ entity_id: entityId }).delete();\n    await trx.batchInsert('search', searchEntries, BATCH_SIZE);\n  });\n}\n"],"names":["NULL_SENTINEL","BATCH_SIZE"],"mappings":";;;;AAoBA,SAAS,qBAAqB,OAAA,EAAuC;AACnE,EAAA,OAAO,OAAA,CAAQ,MAAA;AAAA,IACb,CAAA,CAAA,KAAK,CAAA,CAAE,KAAA,KAAUA,kBAAA,IAAiB,EAAE,cAAA,KAAmBA;AAAA,GACzD;AACF;AAaA,eAAsB,cAAA,CACpB,IAAA,EACA,QAAA,EACA,aAAA,EACe;AAMf,EAAA,MAAM,QAAA,uBAAe,GAAA,EAAyB;AAC9C,EAAA,KAAA,MAAW,SAAS,aAAA,EAAe;AACjC,IAAA,MAAM,CAAA,GAAI,CAAA,EAAG,KAAA,CAAM,GAAG,CAAA,EAAA,EACpB,MAAM,KAAA,KAAU,IAAA,GAAOA,kBAAA,GAAgB,KAAA,CAAM,KAC/C,CAAA,CAAA;AACA,IAAA,IAAI,CAAC,QAAA,CAAS,GAAA,CAAI,CAAC,CAAA,EAAG;AACpB,MAAA,QAAA,CAAS,GAAA,CAAI,GAAG,KAAK,CAAA;AAAA,IACvB;AAAA,EACF;AACA,EAAA,MAAM,OAAA,GAAU,CAAC,GAAG,QAAA,CAAS,QAAQ,CAAA;AAErC,EAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,MAAA;AAElC,EAAA,IAAI,WAAW,IAAA,EAAM;AACnB,IAAA,MAAM,YAAA,CAAa,IAAA,EAAM,QAAA,EAAU,OAAO,CAAA;AAAA,EAC5C,CAAA,MAAA,IAAW,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACnC,IAAA,MAAM,SAAA,CAAU,IAAA,EAAM,QAAA,EAAU,OAAO,CAAA;AAAA,EACzC,CAAA,MAAO;AACL,IAAA,MAAM,eAAA,CAAgB,IAAA,EAAM,QAAA,EAAU,OAAO,CAAA;AAAA,EAC/C;AACF;AAgBA,eAAe,YAAA,CACb,IAAA,EACA,QAAA,EACA,aAAA,EACe;AACf,EAAA,MAAM,QAAA,GAAW,qBAAqB,aAAa,CAAA;AACnD,EAAA,MAAM,IAAA,GAAO,QAAA,CAAS,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,GAAG,CAAA;AACpC,EAAA,MAAM,MAAA,GAAS,QAAA,CAAS,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,KAAK,CAAA;AACxC,EAAA,MAAM,cAAA,GAAiB,QAAA,CAAS,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,cAAc,CAAA;AAEzD,EAAA,MAAM,IAAA,CAAK,GAAA;AAAA,IACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAAA,CAAA;AAAA,IA6BA;AAAA,MACE,IAAA;AAAA;AAAA;AAAA,MAGA,MAAA;AAAA,MACA,cAAA;AAAA,MACA,QAAA;AAAA,MACA,QAAA;AAAA,MACA;AAAA;AACF,GACF;AACF;AAiBA,MAAM,0BAAA,GAA6B,CAAA;AAEnC,eAAe,SAAA,CACb,IAAA,EACA,QAAA,EACA,aAAA,EACe;AACf,EAAA,KAAA,IAAS,OAAA,GAAU,KAAK,OAAA,EAAA,EAAW;AACjC,IAAA,IAAI;AACF,MAAA,MAAM,IAAA,CAAK,WAAA,CAAY,OAAM,GAAA,KAAO;AAKlC,QAAA,MAAM,GAAA,CAAI,GAAA;AAAA,UACR;AAAA,SAKF;AAIA,QAAA,MAAM,GAAA,CAAI,IAAI,+BAA+B,CAAA;AAE7C,QAAA,IAAI,aAAA,CAAc,SAAS,CAAA,EAAG;AAC5B,UAAA,MAAM,GAAA,CAAI,WAAA;AAAA,YACR,iBAAA;AAAA,YACA,aAAA,CAAc,IAAI,CAAA,CAAA,MAAM;AAAA,cACtB,KAAK,CAAA,CAAE,GAAA;AAAA,cACP,OAAO,CAAA,CAAE,KAAA;AAAA,cACT,gBAAgB,CAAA,CAAE;AAAA,aACpB,CAAE,CAAA;AAAA,YACFC;AAAA,WACF;AAAA,QACF;AAGA,QAAA,MAAM,GAAA,CAAI,GAAA;AAAA,UACR,4NAAA;AAAA,UAQA,CAAC,QAAQ;AAAA,SACX;AAMA,QAAA,MAAM,GAAA,CAAI,GAAA;AAAA,UACR,0UAAA;AAAA,UAUA,CAAC,UAAU,QAAQ;AAAA,SACrB;AAAA,MACF,CAAC,CAAA;AACD,MAAA;AAAA,IACF,SAAS,KAAA,EAAO;AAEd,MAAA,IACG,KAAA,EAAe,KAAA,KAAU,IAAA,IAC1B,OAAA,GAAU,0BAAA,EACV;AACA,QAAA;AAAA,MACF;AACA,MAAA,MAAM,KAAA;AAAA,IACR;AAAA,EACF;AACF;AAKA,eAAe,eAAA,CACb,IAAA,EACA,QAAA,EACA,aAAA,EACe;AACf,EAAA,MAAM,IAAA,CAAK,WAAA,CAAY,OAAM,GAAA,KAAO;AAClC,IAAA,MAAM,GAAA,CAAiB,QAAQ,CAAA,CAAE,KAAA,CAAM,EAAE,SAAA,EAAW,QAAA,EAAU,CAAA,CAAE,MAAA,EAAO;AACvE,IAAA,MAAM,GAAA,CAAI,WAAA,CAAY,QAAA,EAAU,aAAA,EAAeA,eAAU,CAAA;AAAA,EAC3D,CAAC,CAAA;AACH;;"}
+{"version":3,"file":"syncSearchRows.cjs.js","sources":["../../../../src/database/operations/stitcher/syncSearchRows.ts"],"sourcesContent":["/*\n * Copyright 2026 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport { DbSearchRow } from '../../tables';\nimport { BATCH_SIZE, NULL_SENTINEL } from './util';\n\nfunction filterSentinelValues(entries: DbSearchRow[]): DbSearchRow[] {\n  return entries.filter(\n    r => r.value !== NULL_SENTINEL && r.original_value !== NULL_SENTINEL,\n  );\n}\n\n/**\n * Synchronizes the search table rows for a given entity, applying only the\n * minimal set of changes needed. Rows that already exist with the correct\n * values are left untouched, new rows are inserted, and stale rows are\n * deleted — minimizing write churn, dead tuples, and WAL traffic.\n *\n * Runs within the caller's publication transaction. The caller owns commit,\n * rollback, and whole-transaction retries; errors propagate unchanged.\n *\n * Uses database-specific strategies:\n * - Postgres: Single writable CTE with unnest (one round-trip, no DDL)\n * - MySQL: Session-scoped temporary table and delete/insert merge\n * - SQLite: Simple bulk replace (sufficient for dev/test)\n */\nexport async function syncSearchRows(\n  knex: Knex.Transaction,\n  entityId: string,\n  searchEntries: DbSearchRow[],\n): Promise<void> {\n  // Dedup by (key, value) — the UNIQUE constraint on (entity_id, key, value)\n  // rejects duplicates, and the same lowercased value with different original\n  // casing is semantically a single entry. Keep the first occurrence, which\n  // matches the first-wins semantics of buildEntitySearch so that both layers\n  // consistently pick the same original_value for a given input order.\n  const dedupMap = new Map<string, DbSearchRow>();\n  for (const entry of searchEntries) {\n    const k = `${entry.key}\\0${\n      entry.value === null ? NULL_SENTINEL : entry.value\n    }`;\n    if (!dedupMap.has(k)) {\n      dedupMap.set(k, entry);\n    }\n  }\n  const deduped = [...dedupMap.values()];\n\n  const client = knex.client.config.client;\n\n  if (client === 'pg') {\n    await syncPostgres(knex, entityId, deduped);\n  } else if (client.includes('mysql')) {\n    await syncMysql(knex, entityId, deduped);\n  } else {\n    await syncBulkReplace(knex, entityId, deduped);\n  }\n}\n\n// ---------------------------------------------------------------------------\n// Postgres: writable CTE + unnest\n//\n// All CTE branches see the same pre-modification snapshot, so the DELETE\n// and INSERT do not interfere with each other. This is a single atomic\n// statement — no explicit transaction wrapper needed.\n//\n// Nullable columns use COALESCE(x, chr(1)) instead of IS NOT DISTINCT FROM\n// so that Postgres can include all three columns in the Hash Cond of the\n// anti-join, rather than pushing nullable comparisons into a Join Filter\n// that degrades to O(n*m) when many rows share the same key. chr(1) (SOH\n// control character) is used as the NULL sentinel — it cannot appear in\n// real entity values since they are human-readable strings.\n// ---------------------------------------------------------------------------\nasync function syncPostgres(\n  knex: Knex.Transaction,\n  entityId: string,\n  searchEntries: DbSearchRow[],\n): Promise<void> {\n  const filtered = filterSentinelValues(searchEntries);\n  const keys = filtered.map(r => r.key);\n  const values = filtered.map(r => r.value);\n  const originalValues = filtered.map(r => r.original_value);\n\n  await knex.raw(\n    `\n    WITH desired AS (\n      SELECT *\n      FROM unnest(?::text[], ?::text[], ?::text[])\n        AS d(key, value, original_value)\n    ),\n    deleted AS (\n      DELETE FROM \"search\" s\n      WHERE s.entity_id = ?\n        AND NOT EXISTS (\n          SELECT 1 FROM desired d\n          WHERE d.key = s.key\n            AND COALESCE(d.value, chr(1)) = COALESCE(s.value, chr(1))\n            AND COALESCE(d.original_value, chr(1)) = COALESCE(s.original_value, chr(1))\n        )\n    )\n    INSERT INTO \"search\" (entity_id, key, value, original_value)\n    SELECT ?, d.key, d.value, d.original_value\n    FROM desired d\n    WHERE NOT EXISTS (\n      SELECT 1 FROM \"search\" s\n      WHERE s.entity_id = ?\n        AND s.key = d.key\n        AND COALESCE(s.value, chr(1)) = COALESCE(d.value, chr(1))\n        AND COALESCE(s.original_value, chr(1)) = COALESCE(d.original_value, chr(1))\n    )\n    ON CONFLICT (entity_id, key, value)\n    DO UPDATE SET original_value = EXCLUDED.original_value\n    `,\n    [\n      keys,\n      // Knex's types don't include nullable PostgreSQL array parameters,\n      // even though they are supported by the pg driver.\n      values as Knex.RawBinding,\n      originalValues as Knex.RawBinding,\n      entityId,\n      entityId,\n      entityId,\n    ],\n  );\n}\n\n// ---------------------------------------------------------------------------\n// MySQL: temporary table merge\n//\n// MySQL does not support data-modifying CTEs, so we materialize the desired\n// state into a session-scoped temporary table and then merge it into the\n// real table with two queries. The temp table is created inside the\n// transaction to guarantee it exists on the same pooled connection.\n// CREATE/DROP TEMPORARY TABLE does not cause an implicit commit in MySQL\n// (unlike regular DDL), so this is transaction-safe.\n//\n// InnoDB's next-key (gap) locking can cause deadlocks between concurrent\n// transactions operating on different entity_ids when their gap locks\n// overlap on shared index pages. Deadlocks propagate to the caller, which must\n// retry the entire publication transaction rather than only these search writes.\n// ---------------------------------------------------------------------------\nasync function syncMysql(\n  knex: Knex.Transaction,\n  entityId: string,\n  searchEntries: DbSearchRow[],\n): Promise<void> {\n  // Create the temp table inside the transaction so it's guaranteed\n  // to be on the same pooled connection as the merge queries.\n  // CREATE TEMPORARY TABLE does not cause an implicit commit in\n  // MySQL (unlike regular CREATE TABLE), so this is safe.\n  await knex.raw(\n    'CREATE TEMPORARY TABLE IF NOT EXISTS `_desired_search` (' +\n      '`key` VARCHAR(255) NOT NULL, ' +\n      '`value` VARCHAR(255) NULL, ' +\n      '`original_value` VARCHAR(255) NULL' +\n      ')',\n  );\n  // Clear stale data from any previous call on this connection.\n  // Uses DELETE (DML) instead of TRUNCATE (DDL) to avoid an\n  // implicit commit that would break transaction atomicity.\n  await knex.raw('DELETE FROM `_desired_search`');\n\n  if (searchEntries.length > 0) {\n    await knex\n      .batchInsert(\n        '_desired_search',\n        searchEntries.map(r => ({\n          key: r.key,\n          value: r.value,\n          original_value: r.original_value,\n        })),\n        BATCH_SIZE,\n      )\n      .transacting(knex);\n  }\n\n  // Delete rows that are no longer in the desired set\n  await knex.raw(\n    'DELETE s FROM `search` s ' +\n      'WHERE s.entity_id = ? ' +\n      'AND NOT EXISTS (' +\n      '  SELECT 1 FROM `_desired_search` d' +\n      '  WHERE d.`key` = s.`key`' +\n      '    AND d.`value` <=> s.`value`' +\n      '    AND BINARY d.`original_value` <=> BINARY s.`original_value`' +\n      ')',\n    [entityId],\n  );\n\n  // Insert rows that are new in the desired set. The original_value\n  // column preserves the original casing and must be compared with\n  // BINARY to avoid MySQL's default case-insensitive collation\n  // treating e.g. \"Team-A\" and \"team-a\" as equal.\n  await knex.raw(\n    'INSERT INTO `search` (entity_id, `key`, `value`, `original_value`) ' +\n      'SELECT ?, d.`key`, d.`value`, d.`original_value` ' +\n      'FROM `_desired_search` d ' +\n      'WHERE NOT EXISTS (' +\n      '  SELECT 1 FROM `search` s' +\n      '  WHERE s.entity_id = ?' +\n      '    AND s.`key` = d.`key`' +\n      '    AND s.`value` <=> d.`value`' +\n      '    AND BINARY s.`original_value` <=> BINARY d.`original_value`' +\n      ')',\n    [entityId, entityId],\n  );\n}\n\n// ---------------------------------------------------------------------------\n// SQLite (and fallback): bulk replace\n// ---------------------------------------------------------------------------\nasync function syncBulkReplace(\n  knex: Knex.Transaction,\n  entityId: string,\n  searchEntries: DbSearchRow[],\n): Promise<void> {\n  await knex<DbSearchRow>('search').where({ entity_id: entityId }).delete();\n  await knex.batchInsert('search', searchEntries, BATCH_SIZE).transacting(knex);\n}\n"],"names":["NULL_SENTINEL","BATCH_SIZE"],"mappings":";;;;AAoBA,SAAS,qBAAqB,OAAA,EAAuC;AACnE,EAAA,OAAO,OAAA,CAAQ,MAAA;AAAA,IACb,CAAA,CAAA,KAAK,CAAA,CAAE,KAAA,KAAUA,kBAAA,IAAiB,EAAE,cAAA,KAAmBA;AAAA,GACzD;AACF;AAgBA,eAAsB,cAAA,CACpB,IAAA,EACA,QAAA,EACA,aAAA,EACe;AAMf,EAAA,MAAM,QAAA,uBAAe,GAAA,EAAyB;AAC9C,EAAA,KAAA,MAAW,SAAS,aAAA,EAAe;AACjC,IAAA,MAAM,CAAA,GAAI,CAAA,EAAG,KAAA,CAAM,GAAG,CAAA,EAAA,EACpB,MAAM,KAAA,KAAU,IAAA,GAAOA,kBAAA,GAAgB,KAAA,CAAM,KAC/C,CAAA,CAAA;AACA,IAAA,IAAI,CAAC,QAAA,CAAS,GAAA,CAAI,CAAC,CAAA,EAAG;AACpB,MAAA,QAAA,CAAS,GAAA,CAAI,GAAG,KAAK,CAAA;AAAA,IACvB;AAAA,EACF;AACA,EAAA,MAAM,OAAA,GAAU,CAAC,GAAG,QAAA,CAAS,QAAQ,CAAA;AAErC,EAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,MAAA;AAElC,EAAA,IAAI,WAAW,IAAA,EAAM;AACnB,IAAA,MAAM,YAAA,CAAa,IAAA,EAAM,QAAA,EAAU,OAAO,CAAA;AAAA,EAC5C,CAAA,MAAA,IAAW,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACnC,IAAA,MAAM,SAAA,CAAU,IAAA,EAAM,QAAA,EAAU,OAAO,CAAA;AAAA,EACzC,CAAA,MAAO;AACL,IAAA,MAAM,eAAA,CAAgB,IAAA,EAAM,QAAA,EAAU,OAAO,CAAA;AAAA,EAC/C;AACF;AAgBA,eAAe,YAAA,CACb,IAAA,EACA,QAAA,EACA,aAAA,EACe;AACf,EAAA,MAAM,QAAA,GAAW,qBAAqB,aAAa,CAAA;AACnD,EAAA,MAAM,IAAA,GAAO,QAAA,CAAS,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,GAAG,CAAA;AACpC,EAAA,MAAM,MAAA,GAAS,QAAA,CAAS,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,KAAK,CAAA;AACxC,EAAA,MAAM,cAAA,GAAiB,QAAA,CAAS,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,cAAc,CAAA;AAEzD,EAAA,MAAM,IAAA,CAAK,GAAA;AAAA,IACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAAA,CAAA;AAAA,IA6BA;AAAA,MACE,IAAA;AAAA;AAAA;AAAA,MAGA,MAAA;AAAA,MACA,cAAA;AAAA,MACA,QAAA;AAAA,MACA,QAAA;AAAA,MACA;AAAA;AACF,GACF;AACF;AAiBA,eAAe,SAAA,CACb,IAAA,EACA,QAAA,EACA,aAAA,EACe;AAKf,EAAA,MAAM,IAAA,CAAK,GAAA;AAAA,IACT;AAAA,GAKF;AAIA,EAAA,MAAM,IAAA,CAAK,IAAI,+BAA+B,CAAA;AAE9C,EAAA,IAAI,aAAA,CAAc,SAAS,CAAA,EAAG;AAC5B,IAAA,MAAM,IAAA,CACH,WAAA;AAAA,MACC,iBAAA;AAAA,MACA,aAAA,CAAc,IAAI,CAAA,CAAA,MAAM;AAAA,QACtB,KAAK,CAAA,CAAE,GAAA;AAAA,QACP,OAAO,CAAA,CAAE,KAAA;AAAA,QACT,gBAAgB,CAAA,CAAE;AAAA,OACpB,CAAE,CAAA;AAAA,MACFC;AAAA,KACF,CACC,YAAY,IAAI,CAAA;AAAA,EACrB;AAGA,EAAA,MAAM,IAAA,CAAK,GAAA;AAAA,IACT,4NAAA;AAAA,IAQA,CAAC,QAAQ;AAAA,GACX;AAMA,EAAA,MAAM,IAAA,CAAK,GAAA;AAAA,IACT,0UAAA;AAAA,IAUA,CAAC,UAAU,QAAQ;AAAA,GACrB;AACF;AAKA,eAAe,eAAA,CACb,IAAA,EACA,QAAA,EACA,aAAA,EACe;AACf,EAAA,MAAM,IAAA,CAAkB,QAAQ,CAAA,CAAE,KAAA,CAAM,EAAE,SAAA,EAAW,QAAA,EAAU,CAAA,CAAE,MAAA,EAAO;AACxE,EAAA,MAAM,KAAK,WAAA,CAAY,QAAA,EAAU,eAAeA,eAAU,CAAA,CAAE,YAAY,IAAI,CAAA;AAC9E;;"}

package/dist/database/util.cjs.js

@@ -25,6 +25,9 @@ function whereInArray(column, values) {
   };
 }
 async function retryOnDeadlock(fn, knex, retries = 3, baseMs = 25) {
+  if (knex.isTransaction) {
+    return fn();
+  }
   let attempt = 0;
   for (; ; ) {
     try {

package/dist/database/util.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.cjs.js","sources":["../../src/database/util.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity } from '@backstage/catalog-model';\nimport { ErrorLike, isError } from '@backstage/errors';\nimport stableStringify from 'fast-json-stable-stringify';\nimport { Knex } from 'knex';\nimport { createHash } from 'node:crypto';\nimport { setTimeout as sleep } from 'node:timers/promises';\n\nexport function generateStableHash(entity: Entity) {\n  return createHash('sha1')\n    .update(stableStringify({ ...entity }))\n    .digest('hex');\n}\n\nexport function generateTargetKey(target: string) {\n  return target.length > 255\n    ? `${target.slice(0, 180)}#sha256:${createHash('sha256')\n        .update(target)\n        .digest('hex')}`\n    : target;\n}\n\n/** Applies an array-backed WHERE clause with a stable query shape on PostgreSQL. */\nexport function whereInArray(column: string, values: string[]) {\n  return (qb: Knex.QueryBuilder) => {\n    if (qb.client.config.client === 'pg') {\n      qb.whereRaw('?? = ANY(?::text[])', [column, values]);\n    } else {\n      qb.whereIn(column, values);\n    }\n  };\n}\n\n/**\n * Retries an operation on database deadlock errors.\n */\nexport async function retryOnDeadlock<T>(\n  fn: () => Promise<T>,\n  knex: Knex | Knex.Transaction,\n  retries = 3,\n  baseMs = 25,\n): Promise<T> {\n  let attempt = 0;\n  for (;;) {\n    try {\n      return await fn();\n    } catch (e: unknown) {\n      if (isDeadlockError(knex, e) && attempt < retries) {\n        await sleep(baseMs * Math.pow(2, attempt));\n        attempt++;\n        continue;\n      }\n      throw e;\n    }\n  }\n}\n\n/**\n * Checks if the given error is a deadlock error for the database engine in use.\n */\nexport function isDeadlockError(\n  knex: Knex | Knex.Transaction,\n  e: unknown,\n): e is ErrorLike {\n  if (knex.client.config.client.includes('pg')) {\n    // PostgreSQL deadlock detection via error code\n    return isError(e) && e.code === '40P01';\n  }\n\n  if (knex.client.config.client.includes('mysql')) {\n    return isError(e) && (e as any).errno === 1213;\n  }\n\n  return false;\n}\n"],"names":["createHash","stableStringify","sleep","isError"],"mappings":";;;;;;;;;;;AAuBO,SAAS,mBAAmB,MAAA,EAAgB;AACjD,EAAA,OAAOA,sBAAA,CAAW,MAAM,CAAA,CACrB,MAAA,CAAOC,gCAAA,CAAgB,EAAE,GAAG,MAAA,EAAQ,CAAC,CAAA,CACrC,MAAA,CAAO,KAAK,CAAA;AACjB;AAEO,SAAS,kBAAkB,MAAA,EAAgB;AAChD,EAAA,OAAO,OAAO,MAAA,GAAS,GAAA,GACnB,GAAG,MAAA,CAAO,KAAA,CAAM,GAAG,GAAG,CAAC,WAAWD,sBAAA,CAAW,QAAQ,EAClD,MAAA,CAAO,MAAM,EACb,MAAA,CAAO,KAAK,CAAC,CAAA,CAAA,GAChB,MAAA;AACN;AAGO,SAAS,YAAA,CAAa,QAAgB,MAAA,EAAkB;AAC7D,EAAA,OAAO,CAAC,EAAA,KAA0B;AAChC,IAAA,IAAI,EAAA,CAAG,MAAA,CAAO,MAAA,CAAO,MAAA,KAAW,IAAA,EAAM;AACpC,MAAA,EAAA,CAAG,QAAA,CAAS,qBAAA,EAAuB,CAAC,MAAA,EAAQ,MAAM,CAAC,CAAA;AAAA,IACrD,CAAA,MAAO;AACL,MAAA,EAAA,CAAG,OAAA,CAAQ,QAAQ,MAAM,CAAA;AAAA,IAC3B;AAAA,EACF,CAAA;AACF;AAKA,eAAsB,gBACpB,EAAA,EACA,IAAA,EACA,OAAA,GAAU,CAAA,EACV,SAAS,EAAA,EACG;AACZ,EAAA,IAAI,OAAA,GAAU,CAAA;AACd,EAAA,WAAS;AACP,IAAA,IAAI;AACF,MAAA,OAAO,MAAM,EAAA,EAAG;AAAA,IAClB,SAAS,CAAA,EAAY;AACnB,MAAA,IAAI,eAAA,CAAgB,IAAA,EAAM,CAAC,CAAA,IAAK,UAAU,OAAA,EAAS;AACjD,QAAA,MAAME,oBAAM,MAAA,GAAS,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,OAAO,CAAC,CAAA;AACzC,QAAA,OAAA,EAAA;AACA,QAAA;AAAA,MACF;AACA,MAAA,MAAM,CAAA;AAAA,IACR;AAAA,EACF;AACF;AAKO,SAAS,eAAA,CACd,MACA,CAAA,EACgB;AAChB,EAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,IAAI,CAAA,EAAG;AAE5C,IAAA,OAAOC,cAAA,CAAQ,CAAC,CAAA,IAAK,CAAA,CAAE,IAAA,KAAS,OAAA;AAAA,EAClC;AAEA,EAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC/C,IAAA,OAAOA,cAAA,CAAQ,CAAC,CAAA,IAAM,CAAA,CAAU,KAAA,KAAU,IAAA;AAAA,EAC5C;AAEA,EAAA,OAAO,KAAA;AACT;;;;;;"}
+{"version":3,"file":"util.cjs.js","sources":["../../src/database/util.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity } from '@backstage/catalog-model';\nimport { ErrorLike, isError } from '@backstage/errors';\nimport stableStringify from 'fast-json-stable-stringify';\nimport { Knex } from 'knex';\nimport { createHash } from 'node:crypto';\nimport { setTimeout as sleep } from 'node:timers/promises';\n\nexport function generateStableHash(entity: Entity) {\n  return createHash('sha1')\n    .update(stableStringify({ ...entity }))\n    .digest('hex');\n}\n\nexport function generateTargetKey(target: string) {\n  return target.length > 255\n    ? `${target.slice(0, 180)}#sha256:${createHash('sha256')\n        .update(target)\n        .digest('hex')}`\n    : target;\n}\n\n/** Applies an array-backed WHERE clause with a stable query shape on PostgreSQL. */\nexport function whereInArray(column: string, values: string[]) {\n  return (qb: Knex.QueryBuilder) => {\n    if (qb.client.config.client === 'pg') {\n      qb.whereRaw('?? = ANY(?::text[])', [column, values]);\n    } else {\n      qb.whereIn(column, values);\n    }\n  };\n}\n\n/**\n * Retries an operation on database deadlock errors.\n *\n * @remarks\n * With a root Knex instance, the callback may run more than once. When wrapping\n * a transaction, recreate the entire transaction on each attempt. Keep\n * non-database side effects outside the callback, since they cannot be rolled\n * back.\n *\n * With an existing Knex transaction, the callback runs exactly once and any\n * deadlock is propagated to the owner, which must retry the whole transaction.\n * Retrying only a statement or savepoint inside an aborted transaction is unsafe.\n */\nexport async function retryOnDeadlock<T>(\n  fn: () => Promise<T>,\n  knex: Knex | Knex.Transaction,\n  retries = 3,\n  baseMs = 25,\n): Promise<T> {\n  // Retrying a statement inside an existing transaction is unsafe. In\n  // PostgreSQL a deadlock aborts the entire transaction, while in MySQL it\n  // rolls the transaction back. Let the caller retry the whole transaction\n  // instead.\n  if (knex.isTransaction) {\n    return fn();\n  }\n\n  let attempt = 0;\n  for (;;) {\n    try {\n      return await fn();\n    } catch (e: unknown) {\n      if (isDeadlockError(knex, e) && attempt < retries) {\n        await sleep(baseMs * Math.pow(2, attempt));\n        attempt++;\n        continue;\n      }\n      throw e;\n    }\n  }\n}\n\n/**\n * Checks if the given error is a deadlock error for the database engine in use.\n */\nexport function isDeadlockError(\n  knex: Knex | Knex.Transaction,\n  e: unknown,\n): e is ErrorLike {\n  if (knex.client.config.client.includes('pg')) {\n    // PostgreSQL deadlock detection via error code\n    return isError(e) && e.code === '40P01';\n  }\n\n  if (knex.client.config.client.includes('mysql')) {\n    return isError(e) && (e as any).errno === 1213;\n  }\n\n  return false;\n}\n"],"names":["createHash","stableStringify","sleep","isError"],"mappings":";;;;;;;;;;;AAuBO,SAAS,mBAAmB,MAAA,EAAgB;AACjD,EAAA,OAAOA,sBAAA,CAAW,MAAM,CAAA,CACrB,MAAA,CAAOC,gCAAA,CAAgB,EAAE,GAAG,MAAA,EAAQ,CAAC,CAAA,CACrC,MAAA,CAAO,KAAK,CAAA;AACjB;AAEO,SAAS,kBAAkB,MAAA,EAAgB;AAChD,EAAA,OAAO,OAAO,MAAA,GAAS,GAAA,GACnB,GAAG,MAAA,CAAO,KAAA,CAAM,GAAG,GAAG,CAAC,WAAWD,sBAAA,CAAW,QAAQ,EAClD,MAAA,CAAO,MAAM,EACb,MAAA,CAAO,KAAK,CAAC,CAAA,CAAA,GAChB,MAAA;AACN;AAGO,SAAS,YAAA,CAAa,QAAgB,MAAA,EAAkB;AAC7D,EAAA,OAAO,CAAC,EAAA,KAA0B;AAChC,IAAA,IAAI,EAAA,CAAG,MAAA,CAAO,MAAA,CAAO,MAAA,KAAW,IAAA,EAAM;AACpC,MAAA,EAAA,CAAG,QAAA,CAAS,qBAAA,EAAuB,CAAC,MAAA,EAAQ,MAAM,CAAC,CAAA;AAAA,IACrD,CAAA,MAAO;AACL,MAAA,EAAA,CAAG,OAAA,CAAQ,QAAQ,MAAM,CAAA;AAAA,IAC3B;AAAA,EACF,CAAA;AACF;AAeA,eAAsB,gBACpB,EAAA,EACA,IAAA,EACA,OAAA,GAAU,CAAA,EACV,SAAS,EAAA,EACG;AAKZ,EAAA,IAAI,KAAK,aAAA,EAAe;AACtB,IAAA,OAAO,EAAA,EAAG;AAAA,EACZ;AAEA,EAAA,IAAI,OAAA,GAAU,CAAA;AACd,EAAA,WAAS;AACP,IAAA,IAAI;AACF,MAAA,OAAO,MAAM,EAAA,EAAG;AAAA,IAClB,SAAS,CAAA,EAAY;AACnB,MAAA,IAAI,eAAA,CAAgB,IAAA,EAAM,CAAC,CAAA,IAAK,UAAU,OAAA,EAAS;AACjD,QAAA,MAAME,oBAAM,MAAA,GAAS,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,OAAO,CAAC,CAAA;AACzC,QAAA,OAAA,EAAA;AACA,QAAA;AAAA,MACF;AACA,MAAA,MAAM,CAAA;AAAA,IACR;AAAA,EACF;AACF;AAKO,SAAS,eAAA,CACd,MACA,CAAA,EACgB;AAChB,EAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,IAAI,CAAA,EAAG;AAE5C,IAAA,OAAOC,cAAA,CAAQ,CAAC,CAAA,IAAK,CAAA,CAAE,IAAA,KAAS,OAAA;AAAA,EAClC;AAEA,EAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC/C,IAAA,OAAOA,cAAA,CAAQ,CAAC,CAAA,IAAM,CAAA,CAAU,KAAA,KAAU,IAAA;AAAA,EAC5C;AAEA,EAAA,OAAO,KAAA;AACT;;;;;;"}

package/dist/processing/DefaultCatalogProcessingEngine.cjs.js

@@ -5,10 +5,10 @@ var errors = require('@backstage/errors');
 var stableStringify = require('fast-json-stable-stringify');
 var api = require('@opentelemetry/api');
 var metrics = require('../util/metrics.cjs.js');
-var markForStitching = require('../database/operations/stitcher/markForStitching.cjs.js');
 var TaskPipeline = require('./TaskPipeline.cjs.js');
 var opentelemetry = require('../util/opentelemetry.cjs.js');
 var deleteOrphanedEntities = require('../database/operations/util/deleteOrphanedEntities.cjs.js');
+var markForStitching = require('../database/operations/stitcher/markForStitching.cjs.js');
 var constants = require('../constants.cjs.js');
 var util = require('../database/util.cjs.js');
 
@@ -175,24 +175,30 @@ class DefaultCatalogProcessingEngine {
                   )}`
                 );
               });
-              await this.processingDatabase.transaction(async (tx) => {
-                await this.processingDatabase.updateProcessedEntityErrors(tx, {
-                  id,
-                  errors: errorsString,
-                  resultHash
-                });
-              });
-              await markForStitching.markForStitching({
-                knex: this.knex,
-                entityRefs: [catalogModel.stringifyEntityRef(unprocessedEntity)]
-              });
+              await util.retryOnDeadlock(
+                () => this.processingDatabase.transaction(async (tx) => {
+                  await this.processingDatabase.updateProcessedEntityErrors(
+                    tx,
+                    {
+                      id,
+                      errors: errorsString,
+                      resultHash
+                    }
+                  );
+                  await markForStitching.markForStitching({
+                    knex: tx,
+                    entityRefs: [catalogModel.stringifyEntityRef(unprocessedEntity)]
+                  });
+                }),
+                this.knex
+              );
               track.markSuccessfulWithErrors();
               return;
             }
             result.completedEntity.metadata.uid = id;
-            const { relationsChange } = await util.retryOnDeadlock(
-              () => this.processingDatabase.transaction(
-                async (tx) => this.processingDatabase.updateProcessedEntity(tx, {
+            await util.retryOnDeadlock(
+              () => this.processingDatabase.transaction(async (tx) => {
+                const { relationsChange } = await this.processingDatabase.updateProcessedEntity(tx, {
                   id,
                   processedEntity: result.completedEntity,
                   resultHash,
@@ -201,23 +207,23 @@ class DefaultCatalogProcessingEngine {
                   deferredEntities: result.deferredEntities,
                   locationKey,
                   refreshKeys: result.refreshKeys
-                })
-              ),
+                });
+                const setOfThingsToStitch = /* @__PURE__ */ new Set([
+                  catalogModel.stringifyEntityRef(result.completedEntity)
+                ]);
+                for (const r of relationsChange.deleted) {
+                  setOfThingsToStitch.add(r.source_entity_ref);
+                }
+                for (const r of relationsChange.inserted) {
+                  setOfThingsToStitch.add(r.source_entity_ref);
+                }
+                await markForStitching.markForStitching({
+                  knex: tx,
+                  entityRefs: setOfThingsToStitch
+                });
+              }),
               this.knex
             );
-            const setOfThingsToStitch = /* @__PURE__ */ new Set([
-              catalogModel.stringifyEntityRef(result.completedEntity)
-            ]);
-            for (const r of relationsChange.deleted) {
-              setOfThingsToStitch.add(r.source_entity_ref);
-            }
-            for (const r of relationsChange.inserted) {
-              setOfThingsToStitch.add(r.source_entity_ref);
-            }
-            await markForStitching.markForStitching({
-              knex: this.knex,
-              entityRefs: setOfThingsToStitch
-            });
             track.markSuccessfulWithChanges();
           } catch (error) {
             track.markFailed(errors.toError(error));

package/dist/processing/DefaultCatalogProcessingEngine.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultCatalogProcessingEngine.cjs.js","sources":["../../src/processing/DefaultCatalogProcessingEngine.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  ANNOTATION_LOCATION,\n  Entity,\n  stringifyEntityRef,\n} from '@backstage/catalog-model';\nimport { serializeError, stringifyError, toError } from '@backstage/errors';\nimport { Hash } from 'node:crypto';\nimport stableStringify from 'fast-json-stable-stringify';\nimport { Knex } from 'knex';\nimport { trace } from '@opentelemetry/api';\nimport { ProcessingDatabase, RefreshStateItem } from '../database/types';\nimport { createCounterMetric, createSummaryMetric } from '../util/metrics';\nimport { CatalogProcessingOrchestrator, EntityProcessingResult } from './types';\nimport { markForStitching } from '../database/operations/stitcher/markForStitching';\nimport { startTaskPipeline } from './TaskPipeline';\nimport { Config } from '@backstage/config';\nimport {\n  addEntityAttributes,\n  TRACER_ID,\n  withActiveSpan,\n} from '../util/opentelemetry';\nimport { deleteOrphanedEntities } from '../database/operations/util/deleteOrphanedEntities';\nimport { EventsService } from '@backstage/plugin-events-node';\nimport { CATALOG_ERRORS_TOPIC } from '../constants';\nimport { retryOnDeadlock } from '../database/util';\nimport { LoggerService, SchedulerService } from '@backstage/backend-plugin-api';\nimport { MetricsService } from '@backstage/backend-plugin-api/alpha';\n\nconst CACHE_TTL = 5;\n\nconst tracer = trace.getTracer(TRACER_ID);\n\nexport type ProgressTracker = ReturnType<typeof progressTracker>;\n\nconst stableStringifyArray = (arr: any[]) => {\n  const sorted = arr.map(stableStringify).sort();\n  return `[${sorted.join(',')}]`;\n};\n\n// NOTE(freben): Perhaps surprisingly, this class does not implement the\n// CatalogProcessingEngine type. That type is externally visible and its name is\n// the way it is for historic reasons. This class has no particular reason to\n// implement that precise interface; nowadays there are several different\n// engines \"hiding\" behind the CatalogProcessingEngine interface, of which this\n// is just one.\nexport class DefaultCatalogProcessingEngine {\n  private readonly config: Config;\n  private readonly scheduler: SchedulerService;\n  private readonly logger: LoggerService;\n  private readonly knex: Knex;\n  private readonly processingDatabase: ProcessingDatabase;\n  private readonly orchestrator: CatalogProcessingOrchestrator;\n  private readonly createHash: () => Hash;\n  private readonly pollingIntervalMs: number;\n  private readonly orphanCleanupIntervalMs: number;\n  private readonly onProcessingError?: (event: {\n    unprocessedEntity: Entity;\n    errors: Error[];\n  }) => Promise<void> | void;\n  private readonly tracker: ProgressTracker;\n  private readonly events: EventsService;\n\n  private stopFunc?: () => void;\n\n  constructor(options: {\n    config: Config;\n    scheduler: SchedulerService;\n    logger: LoggerService;\n    knex: Knex;\n    processingDatabase: ProcessingDatabase;\n    orchestrator: CatalogProcessingOrchestrator;\n    createHash: () => Hash;\n    pollingIntervalMs?: number;\n    orphanCleanupIntervalMs?: number;\n    onProcessingError?: (event: {\n      unprocessedEntity: Entity;\n      errors: Error[];\n    }) => Promise<void> | void;\n    tracker?: ProgressTracker;\n    events: EventsService;\n    metrics: MetricsService;\n  }) {\n    this.config = options.config;\n    this.scheduler = options.scheduler;\n    this.logger = options.logger;\n    this.knex = options.knex;\n    this.processingDatabase = options.processingDatabase;\n    this.orchestrator = options.orchestrator;\n    this.createHash = options.createHash;\n    this.pollingIntervalMs = options.pollingIntervalMs ?? 1_000;\n    this.orphanCleanupIntervalMs = options.orphanCleanupIntervalMs ?? 30_000;\n    this.onProcessingError = options.onProcessingError;\n    this.tracker = options.tracker ?? progressTracker(options.metrics);\n    this.events = options.events;\n\n    this.stopFunc = undefined;\n  }\n\n  async start() {\n    if (this.stopFunc) {\n      throw new Error('Processing engine is already started');\n    }\n\n    const stopPipeline = this.startPipeline();\n    const stopCleanup = this.startOrphanCleanup();\n\n    this.stopFunc = () => {\n      stopPipeline();\n      stopCleanup();\n    };\n  }\n\n  async stop() {\n    if (this.stopFunc) {\n      this.stopFunc();\n      this.stopFunc = undefined;\n    }\n  }\n\n  private startPipeline(): () => void {\n    return startTaskPipeline<RefreshStateItem>({\n      lowWatermark: 5,\n      highWatermark: 10,\n      pollingIntervalMs: this.pollingIntervalMs,\n      loadTasks: async count => {\n        try {\n          const { items } = await this.processingDatabase.transaction(\n            async tx => {\n              return this.processingDatabase.getProcessableEntities(tx, {\n                processBatchSize: count,\n              });\n            },\n          );\n          return items;\n        } catch (error) {\n          this.logger.warn('Failed to load processing items', error);\n          return [];\n        }\n      },\n      processTask: async item => {\n        await withActiveSpan(tracer, 'ProcessingRun', async span => {\n          const track = this.tracker.processStart(item, this.logger);\n          addEntityAttributes(span, item.unprocessedEntity);\n\n          try {\n            const {\n              id,\n              state,\n              unprocessedEntity,\n              entityRef,\n              locationKey,\n              resultHash: previousResultHash,\n            } = item;\n            const result = await this.orchestrator.process({\n              entity: unprocessedEntity,\n              state,\n            });\n\n            track.markProcessorsCompleted(result);\n\n            if (result.ok) {\n              const { ttl: _, ...stateWithoutTtl } = state ?? {};\n              if (\n                stableStringify(stateWithoutTtl) !==\n                stableStringify(result.state)\n              ) {\n                await this.processingDatabase.transaction(async tx => {\n                  await this.processingDatabase.updateEntityCache(tx, {\n                    id,\n                    state: {\n                      ttl: CACHE_TTL,\n                      ...result.state,\n                    },\n                  });\n                });\n              }\n            } else {\n              const maybeTtl = state?.ttl;\n              const ttl = Number.isInteger(maybeTtl) ? (maybeTtl as number) : 0;\n              await this.processingDatabase.transaction(async tx => {\n                await this.processingDatabase.updateEntityCache(tx, {\n                  id,\n                  state: ttl > 0 ? { ...state, ttl: ttl - 1 } : {},\n                });\n              });\n            }\n\n            const location =\n              unprocessedEntity?.metadata?.annotations?.[ANNOTATION_LOCATION];\n            if (result.errors.length) {\n              this.events.publish({\n                topic: CATALOG_ERRORS_TOPIC,\n                eventPayload: {\n                  entity: entityRef,\n                  location,\n                  errors: result.errors,\n                },\n              });\n            }\n            const errorsString = JSON.stringify(\n              result.errors.map(e => serializeError(e)),\n            );\n\n            let hashBuilder = this.createHash().update(errorsString);\n\n            if (result.ok) {\n              const { entityRefs: parents } =\n                await this.processingDatabase.transaction(tx =>\n                  this.processingDatabase.listParents(tx, {\n                    entityRefs: [\n                      entityRef,\n                      ...result.deferredEntities.map(e =>\n                        stringifyEntityRef(e.entity),\n                      ),\n                    ],\n                  }),\n                );\n\n              hashBuilder = hashBuilder\n                .update(stableStringify({ ...result.completedEntity }))\n                .update(stableStringifyArray([...result.deferredEntities]))\n                .update(stableStringifyArray([...result.relations]))\n                .update(stableStringifyArray([...result.refreshKeys]))\n                .update(stableStringifyArray([...parents]));\n            }\n\n            const resultHash = hashBuilder.digest('hex');\n            if (resultHash === previousResultHash) {\n              // If nothing changed in our produced outputs, we cannot have any\n              // significant effect on our surroundings; therefore, we just abort\n              // without any updates / stitching.\n              track.markSuccessfulWithNoChanges();\n              return;\n            }\n\n            // If the result was marked as not OK, it signals that some part of the\n            // processing pipeline threw an exception. This can happen both as part of\n            // non-catastrophic things such as due to validation errors, as well as if\n            // something fatal happens inside the processing for other reasons. In any\n            // case, this means we can't trust that anything in the output is okay. So\n            // just store the errors and trigger a stitch so that they become visible to\n            // the outside.\n            if (!result.ok) {\n              // notify the error listener if the entity can not be processed.\n              Promise.resolve(undefined)\n                .then(() =>\n                  this.onProcessingError?.({\n                    unprocessedEntity,\n                    errors: result.errors,\n                  }),\n                )\n                .catch(error => {\n                  this.logger.debug(\n                    `Processing error listener threw an exception, ${stringifyError(\n                      error,\n                    )}`,\n                  );\n                });\n\n              await this.processingDatabase.transaction(async tx => {\n                await this.processingDatabase.updateProcessedEntityErrors(tx, {\n                  id,\n                  errors: errorsString,\n                  resultHash,\n                });\n              });\n\n              await markForStitching({\n                knex: this.knex,\n                entityRefs: [stringifyEntityRef(unprocessedEntity)],\n              });\n\n              track.markSuccessfulWithErrors();\n              return;\n            }\n\n            result.completedEntity.metadata.uid = id;\n            const { relationsChange } = await retryOnDeadlock(\n              () =>\n                this.processingDatabase.transaction(async tx =>\n                  this.processingDatabase.updateProcessedEntity(tx, {\n                    id,\n                    processedEntity: result.completedEntity,\n                    resultHash,\n                    errors: errorsString,\n                    relations: result.relations,\n                    deferredEntities: result.deferredEntities,\n                    locationKey,\n                    refreshKeys: result.refreshKeys,\n                  }),\n                ),\n              this.knex,\n            );\n\n            // Only stitch entities whose relations actually changed.\n            // In steady state (no relation changes), this is just the\n            // entity itself — no unnecessary stitching of neighbors.\n            const setOfThingsToStitch = new Set<string>([\n              stringifyEntityRef(result.completedEntity),\n            ]);\n            for (const r of relationsChange.deleted) {\n              setOfThingsToStitch.add(r.source_entity_ref);\n            }\n            for (const r of relationsChange.inserted) {\n              setOfThingsToStitch.add(r.source_entity_ref);\n            }\n\n            await markForStitching({\n              knex: this.knex,\n              entityRefs: setOfThingsToStitch,\n            });\n\n            track.markSuccessfulWithChanges();\n          } catch (error) {\n            track.markFailed(toError(error));\n          }\n        });\n      },\n    });\n  }\n\n  private startOrphanCleanup(): () => void {\n    const orphanStrategy =\n      this.config.getOptionalString('catalog.orphanStrategy') ?? 'delete';\n    if (orphanStrategy !== 'delete') {\n      return () => {};\n    }\n\n    const runOnce = async () => {\n      try {\n        const n = await deleteOrphanedEntities({\n          knex: this.knex,\n        });\n        if (n > 0) {\n          this.logger.info(`Deleted ${n} orphaned entities`);\n        }\n      } catch (error) {\n        this.logger.warn(`Failed to delete orphaned entities`, error);\n      }\n    };\n\n    const abortController = new AbortController();\n    this.scheduler.scheduleTask({\n      id: 'catalog_orphan_cleanup',\n      frequency: { milliseconds: this.orphanCleanupIntervalMs },\n      timeout: { milliseconds: this.orphanCleanupIntervalMs * 0.8 },\n      fn: runOnce,\n      signal: abortController.signal,\n    });\n\n    return () => {\n      abortController.abort();\n    };\n  }\n}\n\n// Helps wrap the timing and logging behaviors\nfunction progressTracker(metrics: MetricsService) {\n  // prom-client metrics are deprecated in favour of OpenTelemetry metrics.\n  const promProcessedEntities = createCounterMetric({\n    name: 'catalog_processed_entities_count',\n    help: 'Amount of entities processed, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingDuration = createSummaryMetric({\n    name: 'catalog_processing_duration_seconds',\n    help: 'Time spent executing the full processing flow, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessorsDuration = createSummaryMetric({\n    name: 'catalog_processors_duration_seconds',\n    help: 'Time spent executing catalog processors, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingQueueDelay = createSummaryMetric({\n    name: 'catalog_processing_queue_delay_seconds',\n    help: 'The amount of delay between being scheduled for processing, and the start of actually being processed, DEPRECATED, use OpenTelemetry metrics instead',\n  });\n\n  const processedEntities = metrics.createCounter(\n    'catalog.processed.entities.count',\n    { description: 'Amount of entities processed' },\n  );\n\n  const processingDuration = metrics.createHistogram(\n    'catalog.processing.duration',\n    {\n      description: 'Time spent executing the full processing flow',\n      unit: 'seconds',\n    },\n  );\n\n  const processorsDuration = metrics.createHistogram(\n    'catalog.processors.duration',\n    {\n      description: 'Time spent executing catalog processors',\n      unit: 'seconds',\n    },\n  );\n\n  const processingQueueDelay = metrics.createHistogram(\n    'catalog.processing.queue.delay',\n    {\n      description:\n        'The amount of delay between being scheduled for processing, and the start of actually being processed',\n      unit: 'seconds',\n    },\n  );\n\n  function processStart(item: RefreshStateItem, logger: LoggerService) {\n    const startTime = process.hrtime();\n    const endOverallTimer = promProcessingDuration.startTimer();\n    const endProcessorsTimer = promProcessorsDuration.startTimer();\n\n    logger.debug(`Processing ${item.entityRef}`);\n\n    if (item.nextUpdateAt) {\n      const seconds = -item.nextUpdateAt.diffNow().as('seconds');\n      promProcessingQueueDelay.observe(seconds);\n      processingQueueDelay.record(seconds);\n    }\n\n    function endTime() {\n      const delta = process.hrtime(startTime);\n      return delta[0] + delta[1] / 1e9;\n    }\n\n    function markProcessorsCompleted(result: EntityProcessingResult) {\n      endProcessorsTimer({ result: result.ok ? 'ok' : 'failed' });\n      processorsDuration.record(endTime(), {\n        result: result.ok ? 'ok' : 'failed',\n      });\n    }\n\n    function markSuccessfulWithNoChanges() {\n      endOverallTimer({ result: 'unchanged' });\n      promProcessedEntities.inc({ result: 'unchanged' }, 1);\n\n      processingDuration.record(endTime(), { result: 'unchanged' });\n      processedEntities.add(1, { result: 'unchanged' });\n    }\n\n    function markSuccessfulWithErrors() {\n      endOverallTimer({ result: 'errors' });\n      promProcessedEntities.inc({ result: 'errors' }, 1);\n\n      processingDuration.record(endTime(), { result: 'errors' });\n      processedEntities.add(1, { result: 'errors' });\n    }\n\n    function markSuccessfulWithChanges() {\n      endOverallTimer({ result: 'changed' });\n      promProcessedEntities.inc({ result: 'changed' }, 1);\n\n      processingDuration.record(endTime(), { result: 'changed' });\n      processedEntities.add(1, { result: 'changed' });\n    }\n\n    function markFailed(error: Error) {\n      promProcessedEntities.inc({ result: 'failed' }, 1);\n      processedEntities.add(1, { result: 'failed' });\n      logger.warn(`Processing of ${item.entityRef} failed`, error);\n    }\n\n    return {\n      markProcessorsCompleted,\n      markSuccessfulWithNoChanges,\n      markSuccessfulWithErrors,\n      markSuccessfulWithChanges,\n      markFailed,\n    };\n  }\n\n  return { processStart };\n}\n"],"names":["trace","TRACER_ID","stableStringify","startTaskPipeline","withActiveSpan","addEntityAttributes","ANNOTATION_LOCATION","CATALOG_ERRORS_TOPIC","serializeError","stringifyEntityRef","stringifyError","markForStitching","retryOnDeadlock","toError","deleteOrphanedEntities","metrics","createCounterMetric","createSummaryMetric"],"mappings":";;;;;;;;;;;;;;;;;;AA4CA,MAAM,SAAA,GAAY,CAAA;AAElB,MAAM,MAAA,GAASA,SAAA,CAAM,SAAA,CAAUC,uBAAS,CAAA;AAIxC,MAAM,oBAAA,GAAuB,CAAC,GAAA,KAAe;AAC3C,EAAA,MAAM,MAAA,GAAS,GAAA,CAAI,GAAA,CAAIC,gCAAe,EAAE,IAAA,EAAK;AAC7C,EAAA,OAAO,CAAA,CAAA,EAAI,MAAA,CAAO,IAAA,CAAK,GAAG,CAAC,CAAA,CAAA,CAAA;AAC7B,CAAA;AAQO,MAAM,8BAAA,CAA+B;AAAA,EACzB,MAAA;AAAA,EACA,SAAA;AAAA,EACA,MAAA;AAAA,EACA,IAAA;AAAA,EACA,kBAAA;AAAA,EACA,YAAA;AAAA,EACA,UAAA;AAAA,EACA,iBAAA;AAAA,EACA,uBAAA;AAAA,EACA,iBAAA;AAAA,EAIA,OAAA;AAAA,EACA,MAAA;AAAA,EAET,QAAA;AAAA,EAER,YAAY,OAAA,EAiBT;AACD,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,YAAY,OAAA,CAAQ,SAAA;AACzB,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,OAAO,OAAA,CAAQ,IAAA;AACpB,IAAA,IAAA,CAAK,qBAAqB,OAAA,CAAQ,kBAAA;AAClC,IAAA,IAAA,CAAK,eAAe,OAAA,CAAQ,YAAA;AAC5B,IAAA,IAAA,CAAK,aAAa,OAAA,CAAQ,UAAA;AAC1B,IAAA,IAAA,CAAK,iBAAA,GAAoB,QAAQ,iBAAA,IAAqB,GAAA;AACtD,IAAA,IAAA,CAAK,uBAAA,GAA0B,QAAQ,uBAAA,IAA2B,GAAA;AAClE,IAAA,IAAA,CAAK,oBAAoB,OAAA,CAAQ,iBAAA;AACjC,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA,CAAQ,OAAA,IAAW,eAAA,CAAgB,QAAQ,OAAO,CAAA;AACjE,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AAEtB,IAAA,IAAA,CAAK,QAAA,GAAW,MAAA;AAAA,EAClB;AAAA,EAEA,MAAM,KAAA,GAAQ;AACZ,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,MAAM,IAAI,MAAM,sCAAsC,CAAA;AAAA,IACxD;AAEA,IAAA,MAAM,YAAA,GAAe,KAAK,aAAA,EAAc;AACxC,IAAA,MAAM,WAAA,GAAc,KAAK,kBAAA,EAAmB;AAE5C,IAAA,IAAA,CAAK,WAAW,MAAM;AACpB,MAAA,YAAA,EAAa;AACb,MAAA,WAAA,EAAY;AAAA,IACd,CAAA;AAAA,EACF;AAAA,EAEA,MAAM,IAAA,GAAO;AACX,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,IAAA,CAAK,QAAA,EAAS;AACd,MAAA,IAAA,CAAK,QAAA,GAAW,MAAA;AAAA,IAClB;AAAA,EACF;AAAA,EAEQ,aAAA,GAA4B;AAClC,IAAA,OAAOC,8BAAA,CAAoC;AAAA,MACzC,YAAA,EAAc,CAAA;AAAA,MACd,aAAA,EAAe,EAAA;AAAA,MACf,mBAAmB,IAAA,CAAK,iBAAA;AAAA,MACxB,SAAA,EAAW,OAAM,KAAA,KAAS;AACxB,QAAA,IAAI;AACF,UAAA,MAAM,EAAE,KAAA,EAAM,GAAI,MAAM,KAAK,kBAAA,CAAmB,WAAA;AAAA,YAC9C,OAAM,EAAA,KAAM;AACV,cAAA,OAAO,IAAA,CAAK,kBAAA,CAAmB,sBAAA,CAAuB,EAAA,EAAI;AAAA,gBACxD,gBAAA,EAAkB;AAAA,eACnB,CAAA;AAAA,YACH;AAAA,WACF;AACA,UAAA,OAAO,KAAA;AAAA,QACT,SAAS,KAAA,EAAO;AACd,UAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,iCAAA,EAAmC,KAAK,CAAA;AACzD,UAAA,OAAO,EAAC;AAAA,QACV;AAAA,MACF,CAAA;AAAA,MACA,WAAA,EAAa,OAAM,IAAA,KAAQ;AACzB,QAAA,MAAMC,4BAAA,CAAe,MAAA,EAAQ,eAAA,EAAiB,OAAM,IAAA,KAAQ;AAC1D,UAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,YAAA,CAAa,IAAA,EAAM,KAAK,MAAM,CAAA;AACzD,UAAAC,iCAAA,CAAoB,IAAA,EAAM,KAAK,iBAAiB,CAAA;AAEhD,UAAA,IAAI;AACF,YAAA,MAAM;AAAA,cACJ,EAAA;AAAA,cACA,KAAA;AAAA,cACA,iBAAA;AAAA,cACA,SAAA;AAAA,cACA,WAAA;AAAA,cACA,UAAA,EAAY;AAAA,aACd,GAAI,IAAA;AACJ,YAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,YAAA,CAAa,OAAA,CAAQ;AAAA,cAC7C,MAAA,EAAQ,iBAAA;AAAA,cACR;AAAA,aACD,CAAA;AAED,YAAA,KAAA,CAAM,wBAAwB,MAAM,CAAA;AAEpC,YAAA,IAAI,OAAO,EAAA,EAAI;AACb,cAAA,MAAM,EAAE,GAAA,EAAK,CAAA,EAAG,GAAG,eAAA,EAAgB,GAAI,SAAS,EAAC;AACjD,cAAA,IACEH,iCAAgB,eAAe,CAAA,KAC/BA,gCAAA,CAAgB,MAAA,CAAO,KAAK,CAAA,EAC5B;AACA,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,kBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,iBAAA,CAAkB,EAAA,EAAI;AAAA,oBAClD,EAAA;AAAA,oBACA,KAAA,EAAO;AAAA,sBACL,GAAA,EAAK,SAAA;AAAA,sBACL,GAAG,MAAA,CAAO;AAAA;AACZ,mBACD,CAAA;AAAA,gBACH,CAAC,CAAA;AAAA,cACH;AAAA,YACF,CAAA,MAAO;AACL,cAAA,MAAM,WAAW,KAAA,EAAO,GAAA;AACxB,cAAA,MAAM,GAAA,GAAM,MAAA,CAAO,SAAA,CAAU,QAAQ,IAAK,QAAA,GAAsB,CAAA;AAChE,cAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,iBAAA,CAAkB,EAAA,EAAI;AAAA,kBAClD,EAAA;AAAA,kBACA,KAAA,EAAO,GAAA,GAAM,CAAA,GAAI,EAAE,GAAG,OAAO,GAAA,EAAK,GAAA,GAAM,CAAA,EAAE,GAAI;AAAC,iBAChD,CAAA;AAAA,cACH,CAAC,CAAA;AAAA,YACH;AAEA,YAAA,MAAM,QAAA,GACJ,iBAAA,EAAmB,QAAA,EAAU,WAAA,GAAcI,gCAAmB,CAAA;AAChE,YAAA,IAAI,MAAA,CAAO,OAAO,MAAA,EAAQ;AACxB,cAAA,IAAA,CAAK,OAAO,OAAA,CAAQ;AAAA,gBAClB,KAAA,EAAOC,8BAAA;AAAA,gBACP,YAAA,EAAc;AAAA,kBACZ,MAAA,EAAQ,SAAA;AAAA,kBACR,QAAA;AAAA,kBACA,QAAQ,MAAA,CAAO;AAAA;AACjB,eACD,CAAA;AAAA,YACH;AACA,YAAA,MAAM,eAAe,IAAA,CAAK,SAAA;AAAA,cACxB,OAAO,MAAA,CAAO,GAAA,CAAI,CAAA,CAAA,KAAKC,qBAAA,CAAe,CAAC,CAAC;AAAA,aAC1C;AAEA,YAAA,IAAI,WAAA,GAAc,IAAA,CAAK,UAAA,EAAW,CAAE,OAAO,YAAY,CAAA;AAEvD,YAAA,IAAI,OAAO,EAAA,EAAI;AACb,cAAA,MAAM,EAAE,UAAA,EAAY,OAAA,EAAQ,GAC1B,MAAM,KAAK,kBAAA,CAAmB,WAAA;AAAA,gBAAY,CAAA,EAAA,KACxC,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,EAAA,EAAI;AAAA,kBACtC,UAAA,EAAY;AAAA,oBACV,SAAA;AAAA,oBACA,GAAG,OAAO,gBAAA,CAAiB,GAAA;AAAA,sBAAI,CAAA,CAAA,KAC7BC,+BAAA,CAAmB,CAAA,CAAE,MAAM;AAAA;AAC7B;AACF,iBACD;AAAA,eACH;AAEF,cAAA,WAAA,GAAc,YACX,MAAA,CAAOP,gCAAA,CAAgB,EAAE,GAAG,MAAA,CAAO,iBAAiB,CAAC,EACrD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,gBAAgB,CAAC,CAAC,EACzD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,SAAS,CAAC,CAAC,EAClD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,WAAW,CAAC,CAAC,EACpD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,OAAO,CAAC,CAAC,CAAA;AAAA,YAC9C;AAEA,YAAA,MAAM,UAAA,GAAa,WAAA,CAAY,MAAA,CAAO,KAAK,CAAA;AAC3C,YAAA,IAAI,eAAe,kBAAA,EAAoB;AAIrC,cAAA,KAAA,CAAM,2BAAA,EAA4B;AAClC,cAAA;AAAA,YACF;AASA,YAAA,IAAI,CAAC,OAAO,EAAA,EAAI;AAEd,cAAA,OAAA,CAAQ,OAAA,CAAQ,MAAS,CAAA,CACtB,IAAA;AAAA,gBAAK,MACJ,KAAK,iBAAA,GAAoB;AAAA,kBACvB,iBAAA;AAAA,kBACA,QAAQ,MAAA,CAAO;AAAA,iBAChB;AAAA,eACH,CACC,MAAM,CAAA,KAAA,KAAS;AACd,gBAAA,IAAA,CAAK,MAAA,CAAO,KAAA;AAAA,kBACV,CAAA,8CAAA,EAAiDQ,qBAAA;AAAA,oBAC/C;AAAA,mBACD,CAAA;AAAA,iBACH;AAAA,cACF,CAAC,CAAA;AAEH,cAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,2BAAA,CAA4B,EAAA,EAAI;AAAA,kBAC5D,EAAA;AAAA,kBACA,MAAA,EAAQ,YAAA;AAAA,kBACR;AAAA,iBACD,CAAA;AAAA,cACH,CAAC,CAAA;AAED,cAAA,MAAMC,iCAAA,CAAiB;AAAA,gBACrB,MAAM,IAAA,CAAK,IAAA;AAAA,gBACX,UAAA,EAAY,CAACF,+BAAA,CAAmB,iBAAiB,CAAC;AAAA,eACnD,CAAA;AAED,cAAA,KAAA,CAAM,wBAAA,EAAyB;AAC/B,cAAA;AAAA,YACF;AAEA,YAAA,MAAA,CAAO,eAAA,CAAgB,SAAS,GAAA,GAAM,EAAA;AACtC,YAAA,MAAM,EAAE,eAAA,EAAgB,GAAI,MAAMG,oBAAA;AAAA,cAChC,MACE,KAAK,kBAAA,CAAmB,WAAA;AAAA,gBAAY,OAAM,EAAA,KACxC,IAAA,CAAK,kBAAA,CAAmB,sBAAsB,EAAA,EAAI;AAAA,kBAChD,EAAA;AAAA,kBACA,iBAAiB,MAAA,CAAO,eAAA;AAAA,kBACxB,UAAA;AAAA,kBACA,MAAA,EAAQ,YAAA;AAAA,kBACR,WAAW,MAAA,CAAO,SAAA;AAAA,kBAClB,kBAAkB,MAAA,CAAO,gBAAA;AAAA,kBACzB,WAAA;AAAA,kBACA,aAAa,MAAA,CAAO;AAAA,iBACrB;AAAA,eACH;AAAA,cACF,IAAA,CAAK;AAAA,aACP;AAKA,YAAA,MAAM,mBAAA,uBAA0B,GAAA,CAAY;AAAA,cAC1CH,+BAAA,CAAmB,OAAO,eAAe;AAAA,aAC1C,CAAA;AACD,YAAA,KAAA,MAAW,CAAA,IAAK,gBAAgB,OAAA,EAAS;AACvC,cAAA,mBAAA,CAAoB,GAAA,CAAI,EAAE,iBAAiB,CAAA;AAAA,YAC7C;AACA,YAAA,KAAA,MAAW,CAAA,IAAK,gBAAgB,QAAA,EAAU;AACxC,cAAA,mBAAA,CAAoB,GAAA,CAAI,EAAE,iBAAiB,CAAA;AAAA,YAC7C;AAEA,YAAA,MAAME,iCAAA,CAAiB;AAAA,cACrB,MAAM,IAAA,CAAK,IAAA;AAAA,cACX,UAAA,EAAY;AAAA,aACb,CAAA;AAED,YAAA,KAAA,CAAM,yBAAA,EAA0B;AAAA,UAClC,SAAS,KAAA,EAAO;AACd,YAAA,KAAA,CAAM,UAAA,CAAWE,cAAA,CAAQ,KAAK,CAAC,CAAA;AAAA,UACjC;AAAA,QACF,CAAC,CAAA;AAAA,MACH;AAAA,KACD,CAAA;AAAA,EACH;AAAA,EAEQ,kBAAA,GAAiC;AACvC,IAAA,MAAM,cAAA,GACJ,IAAA,CAAK,MAAA,CAAO,iBAAA,CAAkB,wBAAwB,CAAA,IAAK,QAAA;AAC7D,IAAA,IAAI,mBAAmB,QAAA,EAAU;AAC/B,MAAA,OAAO,MAAM;AAAA,MAAC,CAAA;AAAA,IAChB;AAEA,IAAA,MAAM,UAAU,YAAY;AAC1B,MAAA,IAAI;AACF,QAAA,MAAM,CAAA,GAAI,MAAMC,6CAAA,CAAuB;AAAA,UACrC,MAAM,IAAA,CAAK;AAAA,SACZ,CAAA;AACD,QAAA,IAAI,IAAI,CAAA,EAAG;AACT,UAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,CAAA,QAAA,EAAW,CAAC,CAAA,kBAAA,CAAoB,CAAA;AAAA,QACnD;AAAA,MACF,SAAS,KAAA,EAAO;AACd,QAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,CAAA,kCAAA,CAAA,EAAsC,KAAK,CAAA;AAAA,MAC9D;AAAA,IACF,CAAA;AAEA,IAAA,MAAM,eAAA,GAAkB,IAAI,eAAA,EAAgB;AAC5C,IAAA,IAAA,CAAK,UAAU,YAAA,CAAa;AAAA,MAC1B,EAAA,EAAI,wBAAA;AAAA,MACJ,SAAA,EAAW,EAAE,YAAA,EAAc,IAAA,CAAK,uBAAA,EAAwB;AAAA,MACxD,OAAA,EAAS,EAAE,YAAA,EAAc,IAAA,CAAK,0BAA0B,GAAA,EAAI;AAAA,MAC5D,EAAA,EAAI,OAAA;AAAA,MACJ,QAAQ,eAAA,CAAgB;AAAA,KACzB,CAAA;AAED,IAAA,OAAO,MAAM;AACX,MAAA,eAAA,CAAgB,KAAA,EAAM;AAAA,IACxB,CAAA;AAAA,EACF;AACF;AAGA,SAAS,gBAAgBC,SAAA,EAAyB;AAEhD,EAAA,MAAM,wBAAwBC,2BAAA,CAAoB;AAAA,IAChD,IAAA,EAAM,kCAAA;AAAA,IACN,IAAA,EAAM,6EAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBC,2BAAA,CAAoB;AAAA,IACjD,IAAA,EAAM,qCAAA;AAAA,IACN,IAAA,EAAM,8FAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBA,2BAAA,CAAoB;AAAA,IACjD,IAAA,EAAM,qCAAA;AAAA,IACN,IAAA,EAAM,wFAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,2BAA2BA,2BAAA,CAAoB;AAAA,IACnD,IAAA,EAAM,wCAAA;AAAA,IACN,IAAA,EAAM;AAAA,GACP,CAAA;AAED,EAAA,MAAM,oBAAoBF,SAAA,CAAQ,aAAA;AAAA,IAChC,kCAAA;AAAA,IACA,EAAE,aAAa,8BAAA;AAA+B,GAChD;AAEA,EAAA,MAAM,qBAAqBA,SAAA,CAAQ,eAAA;AAAA,IACjC,6BAAA;AAAA,IACA;AAAA,MACE,WAAA,EAAa,+CAAA;AAAA,MACb,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,MAAM,qBAAqBA,SAAA,CAAQ,eAAA;AAAA,IACjC,6BAAA;AAAA,IACA;AAAA,MACE,WAAA,EAAa,yCAAA;AAAA,MACb,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,MAAM,uBAAuBA,SAAA,CAAQ,eAAA;AAAA,IACnC,gCAAA;AAAA,IACA;AAAA,MACE,WAAA,EACE,uGAAA;AAAA,MACF,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,SAAS,YAAA,CAAa,MAAwB,MAAA,EAAuB;AACnE,IAAA,MAAM,SAAA,GAAY,QAAQ,MAAA,EAAO;AACjC,IAAA,MAAM,eAAA,GAAkB,uBAAuB,UAAA,EAAW;AAC1D,IAAA,MAAM,kBAAA,GAAqB,uBAAuB,UAAA,EAAW;AAE7D,IAAA,MAAA,CAAO,KAAA,CAAM,CAAA,WAAA,EAAc,IAAA,CAAK,SAAS,CAAA,CAAE,CAAA;AAE3C,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,MAAM,UAAU,CAAC,IAAA,CAAK,aAAa,OAAA,EAAQ,CAAE,GAAG,SAAS,CAAA;AACzD,MAAA,wBAAA,CAAyB,QAAQ,OAAO,CAAA;AACxC,MAAA,oBAAA,CAAqB,OAAO,OAAO,CAAA;AAAA,IACrC;AAEA,IAAA,SAAS,OAAA,GAAU;AACjB,MAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,MAAA,CAAO,SAAS,CAAA;AACtC,MAAA,OAAO,KAAA,CAAM,CAAC,CAAA,GAAI,KAAA,CAAM,CAAC,CAAA,GAAI,GAAA;AAAA,IAC/B;AAEA,IAAA,SAAS,wBAAwB,MAAA,EAAgC;AAC/D,MAAA,kBAAA,CAAmB,EAAE,MAAA,EAAQ,MAAA,CAAO,EAAA,GAAK,IAAA,GAAO,UAAU,CAAA;AAC1D,MAAA,kBAAA,CAAmB,MAAA,CAAO,SAAQ,EAAG;AAAA,QACnC,MAAA,EAAQ,MAAA,CAAO,EAAA,GAAK,IAAA,GAAO;AAAA,OAC5B,CAAA;AAAA,IACH;AAEA,IAAA,SAAS,2BAAA,GAA8B;AACrC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,WAAA,EAAa,CAAA;AACvC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,WAAA,IAAe,CAAC,CAAA;AAEpD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAC5D,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAAA,IAClD;AAEA,IAAA,SAAS,wBAAA,GAA2B;AAClC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,QAAA,EAAU,CAAA;AACpC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,QAAA,IAAY,CAAC,CAAA;AAEjD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AACzD,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAAA,IAC/C;AAEA,IAAA,SAAS,yBAAA,GAA4B;AACnC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,SAAA,EAAW,CAAA;AACrC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,SAAA,IAAa,CAAC,CAAA;AAElD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAC1D,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAAA,IAChD;AAEA,IAAA,SAAS,WAAW,KAAA,EAAc;AAChC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,QAAA,IAAY,CAAC,CAAA;AACjD,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAC7C,MAAA,MAAA,CAAO,IAAA,CAAK,CAAA,cAAA,EAAiB,IAAA,CAAK,SAAS,WAAW,KAAK,CAAA;AAAA,IAC7D;AAEA,IAAA,OAAO;AAAA,MACL,uBAAA;AAAA,MACA,2BAAA;AAAA,MACA,wBAAA;AAAA,MACA,yBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF;AAEA,EAAA,OAAO,EAAE,YAAA,EAAa;AACxB;;"}
+{"version":3,"file":"DefaultCatalogProcessingEngine.cjs.js","sources":["../../src/processing/DefaultCatalogProcessingEngine.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  ANNOTATION_LOCATION,\n  Entity,\n  stringifyEntityRef,\n} from '@backstage/catalog-model';\nimport { serializeError, stringifyError, toError } from '@backstage/errors';\nimport { Hash } from 'node:crypto';\nimport stableStringify from 'fast-json-stable-stringify';\nimport { Knex } from 'knex';\nimport { trace } from '@opentelemetry/api';\nimport { ProcessingDatabase, RefreshStateItem } from '../database/types';\nimport { createCounterMetric, createSummaryMetric } from '../util/metrics';\nimport { CatalogProcessingOrchestrator, EntityProcessingResult } from './types';\nimport { startTaskPipeline } from './TaskPipeline';\nimport { Config } from '@backstage/config';\nimport {\n  addEntityAttributes,\n  TRACER_ID,\n  withActiveSpan,\n} from '../util/opentelemetry';\nimport { deleteOrphanedEntities } from '../database/operations/util/deleteOrphanedEntities';\nimport { markForStitching } from '../database/operations/stitcher/markForStitching';\nimport { EventsService } from '@backstage/plugin-events-node';\nimport { CATALOG_ERRORS_TOPIC } from '../constants';\nimport { retryOnDeadlock } from '../database/util';\nimport { LoggerService, SchedulerService } from '@backstage/backend-plugin-api';\nimport { MetricsService } from '@backstage/backend-plugin-api/alpha';\n\nconst CACHE_TTL = 5;\n\nconst tracer = trace.getTracer(TRACER_ID);\n\nexport type ProgressTracker = ReturnType<typeof progressTracker>;\n\nconst stableStringifyArray = (arr: any[]) => {\n  const sorted = arr.map(stableStringify).sort();\n  return `[${sorted.join(',')}]`;\n};\n\n// NOTE(freben): Perhaps surprisingly, this class does not implement the\n// CatalogProcessingEngine type. That type is externally visible and its name is\n// the way it is for historic reasons. This class has no particular reason to\n// implement that precise interface; nowadays there are several different\n// engines \"hiding\" behind the CatalogProcessingEngine interface, of which this\n// is just one.\nexport class DefaultCatalogProcessingEngine {\n  private readonly config: Config;\n  private readonly scheduler: SchedulerService;\n  private readonly logger: LoggerService;\n  private readonly knex: Knex;\n  private readonly processingDatabase: ProcessingDatabase;\n  private readonly orchestrator: CatalogProcessingOrchestrator;\n  private readonly createHash: () => Hash;\n  private readonly pollingIntervalMs: number;\n  private readonly orphanCleanupIntervalMs: number;\n  private readonly onProcessingError?: (event: {\n    unprocessedEntity: Entity;\n    errors: Error[];\n  }) => Promise<void> | void;\n  private readonly tracker: ProgressTracker;\n  private readonly events: EventsService;\n\n  private stopFunc?: () => void;\n\n  constructor(options: {\n    config: Config;\n    scheduler: SchedulerService;\n    logger: LoggerService;\n    knex: Knex;\n    processingDatabase: ProcessingDatabase;\n    orchestrator: CatalogProcessingOrchestrator;\n    createHash: () => Hash;\n    pollingIntervalMs?: number;\n    orphanCleanupIntervalMs?: number;\n    onProcessingError?: (event: {\n      unprocessedEntity: Entity;\n      errors: Error[];\n    }) => Promise<void> | void;\n    tracker?: ProgressTracker;\n    events: EventsService;\n    metrics: MetricsService;\n  }) {\n    this.config = options.config;\n    this.scheduler = options.scheduler;\n    this.logger = options.logger;\n    this.knex = options.knex;\n    this.processingDatabase = options.processingDatabase;\n    this.orchestrator = options.orchestrator;\n    this.createHash = options.createHash;\n    this.pollingIntervalMs = options.pollingIntervalMs ?? 1_000;\n    this.orphanCleanupIntervalMs = options.orphanCleanupIntervalMs ?? 30_000;\n    this.onProcessingError = options.onProcessingError;\n    this.tracker = options.tracker ?? progressTracker(options.metrics);\n    this.events = options.events;\n\n    this.stopFunc = undefined;\n  }\n\n  async start() {\n    if (this.stopFunc) {\n      throw new Error('Processing engine is already started');\n    }\n\n    const stopPipeline = this.startPipeline();\n    const stopCleanup = this.startOrphanCleanup();\n\n    this.stopFunc = () => {\n      stopPipeline();\n      stopCleanup();\n    };\n  }\n\n  async stop() {\n    if (this.stopFunc) {\n      this.stopFunc();\n      this.stopFunc = undefined;\n    }\n  }\n\n  private startPipeline(): () => void {\n    return startTaskPipeline<RefreshStateItem>({\n      lowWatermark: 5,\n      highWatermark: 10,\n      pollingIntervalMs: this.pollingIntervalMs,\n      loadTasks: async count => {\n        try {\n          const { items } = await this.processingDatabase.transaction(\n            async tx => {\n              return this.processingDatabase.getProcessableEntities(tx, {\n                processBatchSize: count,\n              });\n            },\n          );\n          return items;\n        } catch (error) {\n          this.logger.warn('Failed to load processing items', error);\n          return [];\n        }\n      },\n      processTask: async item => {\n        await withActiveSpan(tracer, 'ProcessingRun', async span => {\n          const track = this.tracker.processStart(item, this.logger);\n          addEntityAttributes(span, item.unprocessedEntity);\n\n          try {\n            const {\n              id,\n              state,\n              unprocessedEntity,\n              entityRef,\n              locationKey,\n              resultHash: previousResultHash,\n            } = item;\n            const result = await this.orchestrator.process({\n              entity: unprocessedEntity,\n              state,\n            });\n\n            track.markProcessorsCompleted(result);\n\n            if (result.ok) {\n              const { ttl: _, ...stateWithoutTtl } = state ?? {};\n              if (\n                stableStringify(stateWithoutTtl) !==\n                stableStringify(result.state)\n              ) {\n                await this.processingDatabase.transaction(async tx => {\n                  await this.processingDatabase.updateEntityCache(tx, {\n                    id,\n                    state: {\n                      ttl: CACHE_TTL,\n                      ...result.state,\n                    },\n                  });\n                });\n              }\n            } else {\n              const maybeTtl = state?.ttl;\n              const ttl = Number.isInteger(maybeTtl) ? (maybeTtl as number) : 0;\n              await this.processingDatabase.transaction(async tx => {\n                await this.processingDatabase.updateEntityCache(tx, {\n                  id,\n                  state: ttl > 0 ? { ...state, ttl: ttl - 1 } : {},\n                });\n              });\n            }\n\n            const location =\n              unprocessedEntity?.metadata?.annotations?.[ANNOTATION_LOCATION];\n            if (result.errors.length) {\n              this.events.publish({\n                topic: CATALOG_ERRORS_TOPIC,\n                eventPayload: {\n                  entity: entityRef,\n                  location,\n                  errors: result.errors,\n                },\n              });\n            }\n            const errorsString = JSON.stringify(\n              result.errors.map(e => serializeError(e)),\n            );\n\n            let hashBuilder = this.createHash().update(errorsString);\n\n            if (result.ok) {\n              const { entityRefs: parents } =\n                await this.processingDatabase.transaction(tx =>\n                  this.processingDatabase.listParents(tx, {\n                    entityRefs: [\n                      entityRef,\n                      ...result.deferredEntities.map(e =>\n                        stringifyEntityRef(e.entity),\n                      ),\n                    ],\n                  }),\n                );\n\n              hashBuilder = hashBuilder\n                .update(stableStringify({ ...result.completedEntity }))\n                .update(stableStringifyArray([...result.deferredEntities]))\n                .update(stableStringifyArray([...result.relations]))\n                .update(stableStringifyArray([...result.refreshKeys]))\n                .update(stableStringifyArray([...parents]));\n            }\n\n            const resultHash = hashBuilder.digest('hex');\n            if (resultHash === previousResultHash) {\n              // If nothing changed in our produced outputs, we cannot have any\n              // significant effect on our surroundings; therefore, we just abort\n              // without any updates / stitching.\n              track.markSuccessfulWithNoChanges();\n              return;\n            }\n\n            // If the result was marked as not OK, it signals that some part of the\n            // processing pipeline threw an exception. This can happen both as part of\n            // non-catastrophic things such as due to validation errors, as well as if\n            // something fatal happens inside the processing for other reasons. In any\n            // case, this means we can't trust that anything in the output is okay. So\n            // just store the errors and trigger a stitch so that they become visible to\n            // the outside.\n            if (!result.ok) {\n              // notify the error listener if the entity can not be processed.\n              Promise.resolve(undefined)\n                .then(() =>\n                  this.onProcessingError?.({\n                    unprocessedEntity,\n                    errors: result.errors,\n                  }),\n                )\n                .catch(error => {\n                  this.logger.debug(\n                    `Processing error listener threw an exception, ${stringifyError(\n                      error,\n                    )}`,\n                  );\n                });\n\n              await retryOnDeadlock(\n                () =>\n                  this.processingDatabase.transaction(async tx => {\n                    await this.processingDatabase.updateProcessedEntityErrors(\n                      tx,\n                      {\n                        id,\n                        errors: errorsString,\n                        resultHash,\n                      },\n                    );\n                    await markForStitching({\n                      knex: tx as Knex.Transaction,\n                      entityRefs: [stringifyEntityRef(unprocessedEntity)],\n                    });\n                  }),\n                this.knex,\n              );\n\n              track.markSuccessfulWithErrors();\n              return;\n            }\n\n            result.completedEntity.metadata.uid = id;\n            await retryOnDeadlock(\n              () =>\n                this.processingDatabase.transaction(async tx => {\n                  const { relationsChange } =\n                    await this.processingDatabase.updateProcessedEntity(tx, {\n                      id,\n                      processedEntity: result.completedEntity,\n                      resultHash,\n                      errors: errorsString,\n                      relations: result.relations,\n                      deferredEntities: result.deferredEntities,\n                      locationKey,\n                      refreshKeys: result.refreshKeys,\n                    });\n\n                  // Only stitch entities whose relations actually changed.\n                  // In steady state (no relation changes), this is just the\n                  // entity itself — no unnecessary stitching of neighbors.\n                  const setOfThingsToStitch = new Set<string>([\n                    stringifyEntityRef(result.completedEntity),\n                  ]);\n                  for (const r of relationsChange.deleted) {\n                    setOfThingsToStitch.add(r.source_entity_ref);\n                  }\n                  for (const r of relationsChange.inserted) {\n                    setOfThingsToStitch.add(r.source_entity_ref);\n                  }\n\n                  await markForStitching({\n                    knex: tx as Knex.Transaction,\n                    entityRefs: setOfThingsToStitch,\n                  });\n                }),\n              this.knex,\n            );\n\n            track.markSuccessfulWithChanges();\n          } catch (error) {\n            track.markFailed(toError(error));\n          }\n        });\n      },\n    });\n  }\n\n  private startOrphanCleanup(): () => void {\n    const orphanStrategy =\n      this.config.getOptionalString('catalog.orphanStrategy') ?? 'delete';\n    if (orphanStrategy !== 'delete') {\n      return () => {};\n    }\n\n    const runOnce = async () => {\n      try {\n        const n = await deleteOrphanedEntities({\n          knex: this.knex,\n        });\n        if (n > 0) {\n          this.logger.info(`Deleted ${n} orphaned entities`);\n        }\n      } catch (error) {\n        this.logger.warn(`Failed to delete orphaned entities`, error);\n      }\n    };\n\n    const abortController = new AbortController();\n    this.scheduler.scheduleTask({\n      id: 'catalog_orphan_cleanup',\n      frequency: { milliseconds: this.orphanCleanupIntervalMs },\n      timeout: { milliseconds: this.orphanCleanupIntervalMs * 0.8 },\n      fn: runOnce,\n      signal: abortController.signal,\n    });\n\n    return () => {\n      abortController.abort();\n    };\n  }\n}\n\n// Helps wrap the timing and logging behaviors\nfunction progressTracker(metrics: MetricsService) {\n  // prom-client metrics are deprecated in favour of OpenTelemetry metrics.\n  const promProcessedEntities = createCounterMetric({\n    name: 'catalog_processed_entities_count',\n    help: 'Amount of entities processed, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingDuration = createSummaryMetric({\n    name: 'catalog_processing_duration_seconds',\n    help: 'Time spent executing the full processing flow, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessorsDuration = createSummaryMetric({\n    name: 'catalog_processors_duration_seconds',\n    help: 'Time spent executing catalog processors, DEPRECATED, use OpenTelemetry metrics instead',\n    labelNames: ['result'],\n  });\n  const promProcessingQueueDelay = createSummaryMetric({\n    name: 'catalog_processing_queue_delay_seconds',\n    help: 'The amount of delay between being scheduled for processing, and the start of actually being processed, DEPRECATED, use OpenTelemetry metrics instead',\n  });\n\n  const processedEntities = metrics.createCounter(\n    'catalog.processed.entities.count',\n    { description: 'Amount of entities processed' },\n  );\n\n  const processingDuration = metrics.createHistogram(\n    'catalog.processing.duration',\n    {\n      description: 'Time spent executing the full processing flow',\n      unit: 'seconds',\n    },\n  );\n\n  const processorsDuration = metrics.createHistogram(\n    'catalog.processors.duration',\n    {\n      description: 'Time spent executing catalog processors',\n      unit: 'seconds',\n    },\n  );\n\n  const processingQueueDelay = metrics.createHistogram(\n    'catalog.processing.queue.delay',\n    {\n      description:\n        'The amount of delay between being scheduled for processing, and the start of actually being processed',\n      unit: 'seconds',\n    },\n  );\n\n  function processStart(item: RefreshStateItem, logger: LoggerService) {\n    const startTime = process.hrtime();\n    const endOverallTimer = promProcessingDuration.startTimer();\n    const endProcessorsTimer = promProcessorsDuration.startTimer();\n\n    logger.debug(`Processing ${item.entityRef}`);\n\n    if (item.nextUpdateAt) {\n      const seconds = -item.nextUpdateAt.diffNow().as('seconds');\n      promProcessingQueueDelay.observe(seconds);\n      processingQueueDelay.record(seconds);\n    }\n\n    function endTime() {\n      const delta = process.hrtime(startTime);\n      return delta[0] + delta[1] / 1e9;\n    }\n\n    function markProcessorsCompleted(result: EntityProcessingResult) {\n      endProcessorsTimer({ result: result.ok ? 'ok' : 'failed' });\n      processorsDuration.record(endTime(), {\n        result: result.ok ? 'ok' : 'failed',\n      });\n    }\n\n    function markSuccessfulWithNoChanges() {\n      endOverallTimer({ result: 'unchanged' });\n      promProcessedEntities.inc({ result: 'unchanged' }, 1);\n\n      processingDuration.record(endTime(), { result: 'unchanged' });\n      processedEntities.add(1, { result: 'unchanged' });\n    }\n\n    function markSuccessfulWithErrors() {\n      endOverallTimer({ result: 'errors' });\n      promProcessedEntities.inc({ result: 'errors' }, 1);\n\n      processingDuration.record(endTime(), { result: 'errors' });\n      processedEntities.add(1, { result: 'errors' });\n    }\n\n    function markSuccessfulWithChanges() {\n      endOverallTimer({ result: 'changed' });\n      promProcessedEntities.inc({ result: 'changed' }, 1);\n\n      processingDuration.record(endTime(), { result: 'changed' });\n      processedEntities.add(1, { result: 'changed' });\n    }\n\n    function markFailed(error: Error) {\n      promProcessedEntities.inc({ result: 'failed' }, 1);\n      processedEntities.add(1, { result: 'failed' });\n      logger.warn(`Processing of ${item.entityRef} failed`, error);\n    }\n\n    return {\n      markProcessorsCompleted,\n      markSuccessfulWithNoChanges,\n      markSuccessfulWithErrors,\n      markSuccessfulWithChanges,\n      markFailed,\n    };\n  }\n\n  return { processStart };\n}\n"],"names":["trace","TRACER_ID","stableStringify","startTaskPipeline","withActiveSpan","addEntityAttributes","ANNOTATION_LOCATION","CATALOG_ERRORS_TOPIC","serializeError","stringifyEntityRef","stringifyError","retryOnDeadlock","markForStitching","toError","deleteOrphanedEntities","metrics","createCounterMetric","createSummaryMetric"],"mappings":";;;;;;;;;;;;;;;;;;AA4CA,MAAM,SAAA,GAAY,CAAA;AAElB,MAAM,MAAA,GAASA,SAAA,CAAM,SAAA,CAAUC,uBAAS,CAAA;AAIxC,MAAM,oBAAA,GAAuB,CAAC,GAAA,KAAe;AAC3C,EAAA,MAAM,MAAA,GAAS,GAAA,CAAI,GAAA,CAAIC,gCAAe,EAAE,IAAA,EAAK;AAC7C,EAAA,OAAO,CAAA,CAAA,EAAI,MAAA,CAAO,IAAA,CAAK,GAAG,CAAC,CAAA,CAAA,CAAA;AAC7B,CAAA;AAQO,MAAM,8BAAA,CAA+B;AAAA,EACzB,MAAA;AAAA,EACA,SAAA;AAAA,EACA,MAAA;AAAA,EACA,IAAA;AAAA,EACA,kBAAA;AAAA,EACA,YAAA;AAAA,EACA,UAAA;AAAA,EACA,iBAAA;AAAA,EACA,uBAAA;AAAA,EACA,iBAAA;AAAA,EAIA,OAAA;AAAA,EACA,MAAA;AAAA,EAET,QAAA;AAAA,EAER,YAAY,OAAA,EAiBT;AACD,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,YAAY,OAAA,CAAQ,SAAA;AACzB,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,OAAO,OAAA,CAAQ,IAAA;AACpB,IAAA,IAAA,CAAK,qBAAqB,OAAA,CAAQ,kBAAA;AAClC,IAAA,IAAA,CAAK,eAAe,OAAA,CAAQ,YAAA;AAC5B,IAAA,IAAA,CAAK,aAAa,OAAA,CAAQ,UAAA;AAC1B,IAAA,IAAA,CAAK,iBAAA,GAAoB,QAAQ,iBAAA,IAAqB,GAAA;AACtD,IAAA,IAAA,CAAK,uBAAA,GAA0B,QAAQ,uBAAA,IAA2B,GAAA;AAClE,IAAA,IAAA,CAAK,oBAAoB,OAAA,CAAQ,iBAAA;AACjC,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA,CAAQ,OAAA,IAAW,eAAA,CAAgB,QAAQ,OAAO,CAAA;AACjE,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AAEtB,IAAA,IAAA,CAAK,QAAA,GAAW,MAAA;AAAA,EAClB;AAAA,EAEA,MAAM,KAAA,GAAQ;AACZ,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,MAAM,IAAI,MAAM,sCAAsC,CAAA;AAAA,IACxD;AAEA,IAAA,MAAM,YAAA,GAAe,KAAK,aAAA,EAAc;AACxC,IAAA,MAAM,WAAA,GAAc,KAAK,kBAAA,EAAmB;AAE5C,IAAA,IAAA,CAAK,WAAW,MAAM;AACpB,MAAA,YAAA,EAAa;AACb,MAAA,WAAA,EAAY;AAAA,IACd,CAAA;AAAA,EACF;AAAA,EAEA,MAAM,IAAA,GAAO;AACX,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,IAAA,CAAK,QAAA,EAAS;AACd,MAAA,IAAA,CAAK,QAAA,GAAW,MAAA;AAAA,IAClB;AAAA,EACF;AAAA,EAEQ,aAAA,GAA4B;AAClC,IAAA,OAAOC,8BAAA,CAAoC;AAAA,MACzC,YAAA,EAAc,CAAA;AAAA,MACd,aAAA,EAAe,EAAA;AAAA,MACf,mBAAmB,IAAA,CAAK,iBAAA;AAAA,MACxB,SAAA,EAAW,OAAM,KAAA,KAAS;AACxB,QAAA,IAAI;AACF,UAAA,MAAM,EAAE,KAAA,EAAM,GAAI,MAAM,KAAK,kBAAA,CAAmB,WAAA;AAAA,YAC9C,OAAM,EAAA,KAAM;AACV,cAAA,OAAO,IAAA,CAAK,kBAAA,CAAmB,sBAAA,CAAuB,EAAA,EAAI;AAAA,gBACxD,gBAAA,EAAkB;AAAA,eACnB,CAAA;AAAA,YACH;AAAA,WACF;AACA,UAAA,OAAO,KAAA;AAAA,QACT,SAAS,KAAA,EAAO;AACd,UAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,iCAAA,EAAmC,KAAK,CAAA;AACzD,UAAA,OAAO,EAAC;AAAA,QACV;AAAA,MACF,CAAA;AAAA,MACA,WAAA,EAAa,OAAM,IAAA,KAAQ;AACzB,QAAA,MAAMC,4BAAA,CAAe,MAAA,EAAQ,eAAA,EAAiB,OAAM,IAAA,KAAQ;AAC1D,UAAA,MAAM,QAAQ,IAAA,CAAK,OAAA,CAAQ,YAAA,CAAa,IAAA,EAAM,KAAK,MAAM,CAAA;AACzD,UAAAC,iCAAA,CAAoB,IAAA,EAAM,KAAK,iBAAiB,CAAA;AAEhD,UAAA,IAAI;AACF,YAAA,MAAM;AAAA,cACJ,EAAA;AAAA,cACA,KAAA;AAAA,cACA,iBAAA;AAAA,cACA,SAAA;AAAA,cACA,WAAA;AAAA,cACA,UAAA,EAAY;AAAA,aACd,GAAI,IAAA;AACJ,YAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,YAAA,CAAa,OAAA,CAAQ;AAAA,cAC7C,MAAA,EAAQ,iBAAA;AAAA,cACR;AAAA,aACD,CAAA;AAED,YAAA,KAAA,CAAM,wBAAwB,MAAM,CAAA;AAEpC,YAAA,IAAI,OAAO,EAAA,EAAI;AACb,cAAA,MAAM,EAAE,GAAA,EAAK,CAAA,EAAG,GAAG,eAAA,EAAgB,GAAI,SAAS,EAAC;AACjD,cAAA,IACEH,iCAAgB,eAAe,CAAA,KAC/BA,gCAAA,CAAgB,MAAA,CAAO,KAAK,CAAA,EAC5B;AACA,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,kBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,iBAAA,CAAkB,EAAA,EAAI;AAAA,oBAClD,EAAA;AAAA,oBACA,KAAA,EAAO;AAAA,sBACL,GAAA,EAAK,SAAA;AAAA,sBACL,GAAG,MAAA,CAAO;AAAA;AACZ,mBACD,CAAA;AAAA,gBACH,CAAC,CAAA;AAAA,cACH;AAAA,YACF,CAAA,MAAO;AACL,cAAA,MAAM,WAAW,KAAA,EAAO,GAAA;AACxB,cAAA,MAAM,GAAA,GAAM,MAAA,CAAO,SAAA,CAAU,QAAQ,IAAK,QAAA,GAAsB,CAAA;AAChE,cAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AACpD,gBAAA,MAAM,IAAA,CAAK,kBAAA,CAAmB,iBAAA,CAAkB,EAAA,EAAI;AAAA,kBAClD,EAAA;AAAA,kBACA,KAAA,EAAO,GAAA,GAAM,CAAA,GAAI,EAAE,GAAG,OAAO,GAAA,EAAK,GAAA,GAAM,CAAA,EAAE,GAAI;AAAC,iBAChD,CAAA;AAAA,cACH,CAAC,CAAA;AAAA,YACH;AAEA,YAAA,MAAM,QAAA,GACJ,iBAAA,EAAmB,QAAA,EAAU,WAAA,GAAcI,gCAAmB,CAAA;AAChE,YAAA,IAAI,MAAA,CAAO,OAAO,MAAA,EAAQ;AACxB,cAAA,IAAA,CAAK,OAAO,OAAA,CAAQ;AAAA,gBAClB,KAAA,EAAOC,8BAAA;AAAA,gBACP,YAAA,EAAc;AAAA,kBACZ,MAAA,EAAQ,SAAA;AAAA,kBACR,QAAA;AAAA,kBACA,QAAQ,MAAA,CAAO;AAAA;AACjB,eACD,CAAA;AAAA,YACH;AACA,YAAA,MAAM,eAAe,IAAA,CAAK,SAAA;AAAA,cACxB,OAAO,MAAA,CAAO,GAAA,CAAI,CAAA,CAAA,KAAKC,qBAAA,CAAe,CAAC,CAAC;AAAA,aAC1C;AAEA,YAAA,IAAI,WAAA,GAAc,IAAA,CAAK,UAAA,EAAW,CAAE,OAAO,YAAY,CAAA;AAEvD,YAAA,IAAI,OAAO,EAAA,EAAI;AACb,cAAA,MAAM,EAAE,UAAA,EAAY,OAAA,EAAQ,GAC1B,MAAM,KAAK,kBAAA,CAAmB,WAAA;AAAA,gBAAY,CAAA,EAAA,KACxC,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,EAAA,EAAI;AAAA,kBACtC,UAAA,EAAY;AAAA,oBACV,SAAA;AAAA,oBACA,GAAG,OAAO,gBAAA,CAAiB,GAAA;AAAA,sBAAI,CAAA,CAAA,KAC7BC,+BAAA,CAAmB,CAAA,CAAE,MAAM;AAAA;AAC7B;AACF,iBACD;AAAA,eACH;AAEF,cAAA,WAAA,GAAc,YACX,MAAA,CAAOP,gCAAA,CAAgB,EAAE,GAAG,MAAA,CAAO,iBAAiB,CAAC,EACrD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,gBAAgB,CAAC,CAAC,EACzD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,SAAS,CAAC,CAAC,EAClD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,MAAA,CAAO,WAAW,CAAC,CAAC,EACpD,MAAA,CAAO,oBAAA,CAAqB,CAAC,GAAG,OAAO,CAAC,CAAC,CAAA;AAAA,YAC9C;AAEA,YAAA,MAAM,UAAA,GAAa,WAAA,CAAY,MAAA,CAAO,KAAK,CAAA;AAC3C,YAAA,IAAI,eAAe,kBAAA,EAAoB;AAIrC,cAAA,KAAA,CAAM,2BAAA,EAA4B;AAClC,cAAA;AAAA,YACF;AASA,YAAA,IAAI,CAAC,OAAO,EAAA,EAAI;AAEd,cAAA,OAAA,CAAQ,OAAA,CAAQ,MAAS,CAAA,CACtB,IAAA;AAAA,gBAAK,MACJ,KAAK,iBAAA,GAAoB;AAAA,kBACvB,iBAAA;AAAA,kBACA,QAAQ,MAAA,CAAO;AAAA,iBAChB;AAAA,eACH,CACC,MAAM,CAAA,KAAA,KAAS;AACd,gBAAA,IAAA,CAAK,MAAA,CAAO,KAAA;AAAA,kBACV,CAAA,8CAAA,EAAiDQ,qBAAA;AAAA,oBAC/C;AAAA,mBACD,CAAA;AAAA,iBACH;AAAA,cACF,CAAC,CAAA;AAEH,cAAA,MAAMC,oBAAA;AAAA,gBACJ,MACE,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AAC9C,kBAAA,MAAM,KAAK,kBAAA,CAAmB,2BAAA;AAAA,oBAC5B,EAAA;AAAA,oBACA;AAAA,sBACE,EAAA;AAAA,sBACA,MAAA,EAAQ,YAAA;AAAA,sBACR;AAAA;AACF,mBACF;AACA,kBAAA,MAAMC,iCAAA,CAAiB;AAAA,oBACrB,IAAA,EAAM,EAAA;AAAA,oBACN,UAAA,EAAY,CAACH,+BAAA,CAAmB,iBAAiB,CAAC;AAAA,mBACnD,CAAA;AAAA,gBACH,CAAC,CAAA;AAAA,gBACH,IAAA,CAAK;AAAA,eACP;AAEA,cAAA,KAAA,CAAM,wBAAA,EAAyB;AAC/B,cAAA;AAAA,YACF;AAEA,YAAA,MAAA,CAAO,eAAA,CAAgB,SAAS,GAAA,GAAM,EAAA;AACtC,YAAA,MAAME,oBAAA;AAAA,cACJ,MACE,IAAA,CAAK,kBAAA,CAAmB,WAAA,CAAY,OAAM,EAAA,KAAM;AAC9C,gBAAA,MAAM,EAAE,eAAA,EAAgB,GACtB,MAAM,IAAA,CAAK,kBAAA,CAAmB,sBAAsB,EAAA,EAAI;AAAA,kBACtD,EAAA;AAAA,kBACA,iBAAiB,MAAA,CAAO,eAAA;AAAA,kBACxB,UAAA;AAAA,kBACA,MAAA,EAAQ,YAAA;AAAA,kBACR,WAAW,MAAA,CAAO,SAAA;AAAA,kBAClB,kBAAkB,MAAA,CAAO,gBAAA;AAAA,kBACzB,WAAA;AAAA,kBACA,aAAa,MAAA,CAAO;AAAA,iBACrB,CAAA;AAKH,gBAAA,MAAM,mBAAA,uBAA0B,GAAA,CAAY;AAAA,kBAC1CF,+BAAA,CAAmB,OAAO,eAAe;AAAA,iBAC1C,CAAA;AACD,gBAAA,KAAA,MAAW,CAAA,IAAK,gBAAgB,OAAA,EAAS;AACvC,kBAAA,mBAAA,CAAoB,GAAA,CAAI,EAAE,iBAAiB,CAAA;AAAA,gBAC7C;AACA,gBAAA,KAAA,MAAW,CAAA,IAAK,gBAAgB,QAAA,EAAU;AACxC,kBAAA,mBAAA,CAAoB,GAAA,CAAI,EAAE,iBAAiB,CAAA;AAAA,gBAC7C;AAEA,gBAAA,MAAMG,iCAAA,CAAiB;AAAA,kBACrB,IAAA,EAAM,EAAA;AAAA,kBACN,UAAA,EAAY;AAAA,iBACb,CAAA;AAAA,cACH,CAAC,CAAA;AAAA,cACH,IAAA,CAAK;AAAA,aACP;AAEA,YAAA,KAAA,CAAM,yBAAA,EAA0B;AAAA,UAClC,SAAS,KAAA,EAAO;AACd,YAAA,KAAA,CAAM,UAAA,CAAWC,cAAA,CAAQ,KAAK,CAAC,CAAA;AAAA,UACjC;AAAA,QACF,CAAC,CAAA;AAAA,MACH;AAAA,KACD,CAAA;AAAA,EACH;AAAA,EAEQ,kBAAA,GAAiC;AACvC,IAAA,MAAM,cAAA,GACJ,IAAA,CAAK,MAAA,CAAO,iBAAA,CAAkB,wBAAwB,CAAA,IAAK,QAAA;AAC7D,IAAA,IAAI,mBAAmB,QAAA,EAAU;AAC/B,MAAA,OAAO,MAAM;AAAA,MAAC,CAAA;AAAA,IAChB;AAEA,IAAA,MAAM,UAAU,YAAY;AAC1B,MAAA,IAAI;AACF,QAAA,MAAM,CAAA,GAAI,MAAMC,6CAAA,CAAuB;AAAA,UACrC,MAAM,IAAA,CAAK;AAAA,SACZ,CAAA;AACD,QAAA,IAAI,IAAI,CAAA,EAAG;AACT,UAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,CAAA,QAAA,EAAW,CAAC,CAAA,kBAAA,CAAoB,CAAA;AAAA,QACnD;AAAA,MACF,SAAS,KAAA,EAAO;AACd,QAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,CAAA,kCAAA,CAAA,EAAsC,KAAK,CAAA;AAAA,MAC9D;AAAA,IACF,CAAA;AAEA,IAAA,MAAM,eAAA,GAAkB,IAAI,eAAA,EAAgB;AAC5C,IAAA,IAAA,CAAK,UAAU,YAAA,CAAa;AAAA,MAC1B,EAAA,EAAI,wBAAA;AAAA,MACJ,SAAA,EAAW,EAAE,YAAA,EAAc,IAAA,CAAK,uBAAA,EAAwB;AAAA,MACxD,OAAA,EAAS,EAAE,YAAA,EAAc,IAAA,CAAK,0BAA0B,GAAA,EAAI;AAAA,MAC5D,EAAA,EAAI,OAAA;AAAA,MACJ,QAAQ,eAAA,CAAgB;AAAA,KACzB,CAAA;AAED,IAAA,OAAO,MAAM;AACX,MAAA,eAAA,CAAgB,KAAA,EAAM;AAAA,IACxB,CAAA;AAAA,EACF;AACF;AAGA,SAAS,gBAAgBC,SAAA,EAAyB;AAEhD,EAAA,MAAM,wBAAwBC,2BAAA,CAAoB;AAAA,IAChD,IAAA,EAAM,kCAAA;AAAA,IACN,IAAA,EAAM,6EAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBC,2BAAA,CAAoB;AAAA,IACjD,IAAA,EAAM,qCAAA;AAAA,IACN,IAAA,EAAM,8FAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,yBAAyBA,2BAAA,CAAoB;AAAA,IACjD,IAAA,EAAM,qCAAA;AAAA,IACN,IAAA,EAAM,wFAAA;AAAA,IACN,UAAA,EAAY,CAAC,QAAQ;AAAA,GACtB,CAAA;AACD,EAAA,MAAM,2BAA2BA,2BAAA,CAAoB;AAAA,IACnD,IAAA,EAAM,wCAAA;AAAA,IACN,IAAA,EAAM;AAAA,GACP,CAAA;AAED,EAAA,MAAM,oBAAoBF,SAAA,CAAQ,aAAA;AAAA,IAChC,kCAAA;AAAA,IACA,EAAE,aAAa,8BAAA;AAA+B,GAChD;AAEA,EAAA,MAAM,qBAAqBA,SAAA,CAAQ,eAAA;AAAA,IACjC,6BAAA;AAAA,IACA;AAAA,MACE,WAAA,EAAa,+CAAA;AAAA,MACb,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,MAAM,qBAAqBA,SAAA,CAAQ,eAAA;AAAA,IACjC,6BAAA;AAAA,IACA;AAAA,MACE,WAAA,EAAa,yCAAA;AAAA,MACb,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,MAAM,uBAAuBA,SAAA,CAAQ,eAAA;AAAA,IACnC,gCAAA;AAAA,IACA;AAAA,MACE,WAAA,EACE,uGAAA;AAAA,MACF,IAAA,EAAM;AAAA;AACR,GACF;AAEA,EAAA,SAAS,YAAA,CAAa,MAAwB,MAAA,EAAuB;AACnE,IAAA,MAAM,SAAA,GAAY,QAAQ,MAAA,EAAO;AACjC,IAAA,MAAM,eAAA,GAAkB,uBAAuB,UAAA,EAAW;AAC1D,IAAA,MAAM,kBAAA,GAAqB,uBAAuB,UAAA,EAAW;AAE7D,IAAA,MAAA,CAAO,KAAA,CAAM,CAAA,WAAA,EAAc,IAAA,CAAK,SAAS,CAAA,CAAE,CAAA;AAE3C,IAAA,IAAI,KAAK,YAAA,EAAc;AACrB,MAAA,MAAM,UAAU,CAAC,IAAA,CAAK,aAAa,OAAA,EAAQ,CAAE,GAAG,SAAS,CAAA;AACzD,MAAA,wBAAA,CAAyB,QAAQ,OAAO,CAAA;AACxC,MAAA,oBAAA,CAAqB,OAAO,OAAO,CAAA;AAAA,IACrC;AAEA,IAAA,SAAS,OAAA,GAAU;AACjB,MAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,MAAA,CAAO,SAAS,CAAA;AACtC,MAAA,OAAO,KAAA,CAAM,CAAC,CAAA,GAAI,KAAA,CAAM,CAAC,CAAA,GAAI,GAAA;AAAA,IAC/B;AAEA,IAAA,SAAS,wBAAwB,MAAA,EAAgC;AAC/D,MAAA,kBAAA,CAAmB,EAAE,MAAA,EAAQ,MAAA,CAAO,EAAA,GAAK,IAAA,GAAO,UAAU,CAAA;AAC1D,MAAA,kBAAA,CAAmB,MAAA,CAAO,SAAQ,EAAG;AAAA,QACnC,MAAA,EAAQ,MAAA,CAAO,EAAA,GAAK,IAAA,GAAO;AAAA,OAC5B,CAAA;AAAA,IACH;AAEA,IAAA,SAAS,2BAAA,GAA8B;AACrC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,WAAA,EAAa,CAAA;AACvC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,WAAA,IAAe,CAAC,CAAA;AAEpD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAC5D,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,aAAa,CAAA;AAAA,IAClD;AAEA,IAAA,SAAS,wBAAA,GAA2B;AAClC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,QAAA,EAAU,CAAA;AACpC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,QAAA,IAAY,CAAC,CAAA;AAEjD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AACzD,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAAA,IAC/C;AAEA,IAAA,SAAS,yBAAA,GAA4B;AACnC,MAAA,eAAA,CAAgB,EAAE,MAAA,EAAQ,SAAA,EAAW,CAAA;AACrC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,SAAA,IAAa,CAAC,CAAA;AAElD,MAAA,kBAAA,CAAmB,OAAO,OAAA,EAAQ,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAC1D,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,WAAW,CAAA;AAAA,IAChD;AAEA,IAAA,SAAS,WAAW,KAAA,EAAc;AAChC,MAAA,qBAAA,CAAsB,GAAA,CAAI,EAAE,MAAA,EAAQ,QAAA,IAAY,CAAC,CAAA;AACjD,MAAA,iBAAA,CAAkB,GAAA,CAAI,CAAA,EAAG,EAAE,MAAA,EAAQ,UAAU,CAAA;AAC7C,MAAA,MAAA,CAAO,IAAA,CAAK,CAAA,cAAA,EAAiB,IAAA,CAAK,SAAS,WAAW,KAAK,CAAA;AAAA,IAC7D;AAEA,IAAA,OAAO;AAAA,MACL,uBAAA;AAAA,MACA,2BAAA;AAAA,MACA,wBAAA;AAAA,MACA,yBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF;AAEA,EAAA,OAAO,EAAE,YAAA,EAAa;AACxB;;"}

package/dist/stitching/DefaultStitcher.cjs.js

@@ -47,7 +47,8 @@ class DefaultStitcher {
         return await this.#stitchOne({
           entityRef: item.entityRef,
           stitchTicket: item.stitchTicket,
-          stitchRequestedAt: item.stitchRequestedAt
+          stitchRequestedAt: item.stitchRequestedAt,
+          stitchLeaseExpiresAt: item.stitchLeaseExpiresAt
         });
       }
     });
@@ -83,7 +84,8 @@ class DefaultStitcher {
         knex: this.knex,
         logger: this.logger,
         entityRef: options.entityRef,
-        stitchTicket: options.stitchTicket
+        stitchTicket: options.stitchTicket,
+        stitchLeaseExpiresAt: options.stitchLeaseExpiresAt
       });
       track.markComplete(result);
     } catch (error) {

package/dist/stitching/DefaultStitcher.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultStitcher.cjs.js","sources":["../../src/stitching/DefaultStitcher.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Config } from '@backstage/config';\nimport { durationToMilliseconds, HumanDuration } from '@backstage/types';\nimport { Knex } from 'knex';\nimport { DateTime } from 'luxon';\nimport { getDeferredStitchableEntities } from '../database/operations/stitcher/getDeferredStitchableEntities';\nimport { performStitching } from '../database/operations/stitcher/performStitching';\nimport { startTaskPipeline } from '../processing/TaskPipeline';\nimport { progressTracker } from './progressTracker';\nimport { StitchingStrategy, stitchingStrategyFromConfig } from './types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\nimport { MetricsService } from '@backstage/backend-plugin-api/alpha';\n\ntype DeferredStitchItem = Awaited<\n  ReturnType<typeof getDeferredStitchableEntities>\n>[0];\n\ntype StitchProgressTracker = ReturnType<typeof progressTracker>;\n\n/**\n * Performs the act of stitching - to take all of the various outputs from the\n * ingestion process, and stitching them together into the final entity JSON\n * shape.\n */\nexport class DefaultStitcher {\n  private readonly knex: Knex;\n  private readonly logger: LoggerService;\n  private readonly strategy: StitchingStrategy;\n  private readonly tracker: StitchProgressTracker;\n  private stopFunc?: () => void;\n\n  static fromConfig(\n    config: Config,\n    options: {\n      knex: Knex;\n      logger: LoggerService;\n      metrics: MetricsService;\n    },\n  ): DefaultStitcher {\n    return new DefaultStitcher({\n      knex: options.knex,\n      logger: options.logger,\n      metrics: options.metrics,\n      strategy: stitchingStrategyFromConfig(config),\n    });\n  }\n\n  constructor(options: {\n    knex: Knex;\n    logger: LoggerService;\n    metrics: MetricsService;\n    strategy: StitchingStrategy;\n  }) {\n    this.knex = options.knex;\n    this.logger = options.logger;\n    this.strategy = options.strategy;\n    this.tracker = progressTracker(\n      options.knex,\n      options.logger,\n      options.metrics,\n    );\n  }\n\n  async start() {\n    if (this.stopFunc) {\n      throw new Error('Stitcher is already started');\n    }\n\n    const { pollingInterval, stitchTimeout } = this.strategy;\n\n    const stopPipeline = startTaskPipeline<DeferredStitchItem>({\n      lowWatermark: 2,\n      highWatermark: 5,\n      pollingIntervalMs: durationToMilliseconds(pollingInterval),\n      loadTasks: async count => {\n        return await this.#getStitchableEntities(count, stitchTimeout);\n      },\n      processTask: async item => {\n        return await this.#stitchOne({\n          entityRef: item.entityRef,\n          stitchTicket: item.stitchTicket,\n          stitchRequestedAt: item.stitchRequestedAt,\n        });\n      },\n    });\n\n    this.stopFunc = () => {\n      stopPipeline();\n    };\n  }\n\n  async stop() {\n    if (this.stopFunc) {\n      this.stopFunc();\n      this.stopFunc = undefined;\n    }\n  }\n\n  async #getStitchableEntities(count: number, stitchTimeout: HumanDuration) {\n    try {\n      return await getDeferredStitchableEntities({\n        knex: this.knex,\n        batchSize: count,\n        stitchTimeout: stitchTimeout,\n      });\n    } catch (error) {\n      this.logger.warn('Failed to load stitchable entities', error);\n      return [];\n    }\n  }\n\n  async #stitchOne(options: {\n    entityRef: string;\n    stitchTicket: string;\n    stitchRequestedAt?: DateTime;\n  }) {\n    const track = this.tracker.stitchStart({\n      entityRef: options.entityRef,\n      stitchRequestedAt: options.stitchRequestedAt,\n    });\n\n    try {\n      const result = await performStitching({\n        knex: this.knex,\n        logger: this.logger,\n        entityRef: options.entityRef,\n        stitchTicket: options.stitchTicket,\n      });\n      track.markComplete(result);\n    } catch (error) {\n      track.markFailed(error);\n    }\n  }\n}\n"],"names":["stitchingStrategyFromConfig","progressTracker","startTaskPipeline","durationToMilliseconds","getDeferredStitchableEntities","performStitching"],"mappings":";;;;;;;;;AAuCO,MAAM,eAAA,CAAgB;AAAA,EACV,IAAA;AAAA,EACA,MAAA;AAAA,EACA,QAAA;AAAA,EACA,OAAA;AAAA,EACT,QAAA;AAAA,EAER,OAAO,UAAA,CACL,MAAA,EACA,OAAA,EAKiB;AACjB,IAAA,OAAO,IAAI,eAAA,CAAgB;AAAA,MACzB,MAAM,OAAA,CAAQ,IAAA;AAAA,MACd,QAAQ,OAAA,CAAQ,MAAA;AAAA,MAChB,SAAS,OAAA,CAAQ,OAAA;AAAA,MACjB,QAAA,EAAUA,kCAA4B,MAAM;AAAA,KAC7C,CAAA;AAAA,EACH;AAAA,EAEA,YAAY,OAAA,EAKT;AACD,IAAA,IAAA,CAAK,OAAO,OAAA,CAAQ,IAAA;AACpB,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,WAAW,OAAA,CAAQ,QAAA;AACxB,IAAA,IAAA,CAAK,OAAA,GAAUC,+BAAA;AAAA,MACb,OAAA,CAAQ,IAAA;AAAA,MACR,OAAA,CAAQ,MAAA;AAAA,MACR,OAAA,CAAQ;AAAA,KACV;AAAA,EACF;AAAA,EAEA,MAAM,KAAA,GAAQ;AACZ,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,MAAM,IAAI,MAAM,6BAA6B,CAAA;AAAA,IAC/C;AAEA,IAAA,MAAM,EAAE,eAAA,EAAiB,aAAA,EAAc,GAAI,IAAA,CAAK,QAAA;AAEhD,IAAA,MAAM,eAAeC,8BAAA,CAAsC;AAAA,MACzD,YAAA,EAAc,CAAA;AAAA,MACd,aAAA,EAAe,CAAA;AAAA,MACf,iBAAA,EAAmBC,+BAAuB,eAAe,CAAA;AAAA,MACzD,SAAA,EAAW,OAAM,KAAA,KAAS;AACxB,QAAA,OAAO,MAAM,IAAA,CAAK,sBAAA,CAAuB,KAAA,EAAO,aAAa,CAAA;AAAA,MAC/D,CAAA;AAAA,MACA,WAAA,EAAa,OAAM,IAAA,KAAQ;AACzB,QAAA,OAAO,MAAM,KAAK,UAAA,CAAW;AAAA,UAC3B,WAAW,IAAA,CAAK,SAAA;AAAA,UAChB,cAAc,IAAA,CAAK,YAAA;AAAA,UACnB,mBAAmB,IAAA,CAAK;AAAA,SACzB,CAAA;AAAA,MACH;AAAA,KACD,CAAA;AAED,IAAA,IAAA,CAAK,WAAW,MAAM;AACpB,MAAA,YAAA,EAAa;AAAA,IACf,CAAA;AAAA,EACF;AAAA,EAEA,MAAM,IAAA,GAAO;AACX,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,IAAA,CAAK,QAAA,EAAS;AACd,MAAA,IAAA,CAAK,QAAA,GAAW,MAAA;AAAA,IAClB;AAAA,EACF;AAAA,EAEA,MAAM,sBAAA,CAAuB,KAAA,EAAe,aAAA,EAA8B;AACxE,IAAA,IAAI;AACF,MAAA,OAAO,MAAMC,2DAAA,CAA8B;AAAA,QACzC,MAAM,IAAA,CAAK,IAAA;AAAA,QACX,SAAA,EAAW,KAAA;AAAA,QACX;AAAA,OACD,CAAA;AAAA,IACH,SAAS,KAAA,EAAO;AACd,MAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,oCAAA,EAAsC,KAAK,CAAA;AAC5D,MAAA,OAAO,EAAC;AAAA,IACV;AAAA,EACF;AAAA,EAEA,MAAM,WAAW,OAAA,EAId;AACD,IAAA,MAAM,KAAA,GAAQ,IAAA,CAAK,OAAA,CAAQ,WAAA,CAAY;AAAA,MACrC,WAAW,OAAA,CAAQ,SAAA;AAAA,MACnB,mBAAmB,OAAA,CAAQ;AAAA,KAC5B,CAAA;AAED,IAAA,IAAI;AACF,MAAA,MAAM,MAAA,GAAS,MAAMC,iCAAA,CAAiB;AAAA,QACpC,MAAM,IAAA,CAAK,IAAA;AAAA,QACX,QAAQ,IAAA,CAAK,MAAA;AAAA,QACb,WAAW,OAAA,CAAQ,SAAA;AAAA,QACnB,cAAc,OAAA,CAAQ;AAAA,OACvB,CAAA;AACD,MAAA,KAAA,CAAM,aAAa,MAAM,CAAA;AAAA,IAC3B,SAAS,KAAA,EAAO;AACd,MAAA,KAAA,CAAM,WAAW,KAAK,CAAA;AAAA,IACxB;AAAA,EACF;AACF;;"}
+{"version":3,"file":"DefaultStitcher.cjs.js","sources":["../../src/stitching/DefaultStitcher.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Config } from '@backstage/config';\nimport { durationToMilliseconds, HumanDuration } from '@backstage/types';\nimport { Knex } from 'knex';\nimport { DateTime } from 'luxon';\nimport { getDeferredStitchableEntities } from '../database/operations/stitcher/getDeferredStitchableEntities';\nimport { performStitching } from '../database/operations/stitcher/performStitching';\nimport { startTaskPipeline } from '../processing/TaskPipeline';\nimport { progressTracker } from './progressTracker';\nimport { StitchingStrategy, stitchingStrategyFromConfig } from './types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\nimport { MetricsService } from '@backstage/backend-plugin-api/alpha';\n\ntype DeferredStitchItem = Awaited<\n  ReturnType<typeof getDeferredStitchableEntities>\n>[0];\n\ntype StitchProgressTracker = ReturnType<typeof progressTracker>;\n\n/**\n * Performs the act of stitching - to take all of the various outputs from the\n * ingestion process, and stitching them together into the final entity JSON\n * shape.\n */\nexport class DefaultStitcher {\n  private readonly knex: Knex;\n  private readonly logger: LoggerService;\n  private readonly strategy: StitchingStrategy;\n  private readonly tracker: StitchProgressTracker;\n  private stopFunc?: () => void;\n\n  static fromConfig(\n    config: Config,\n    options: {\n      knex: Knex;\n      logger: LoggerService;\n      metrics: MetricsService;\n    },\n  ): DefaultStitcher {\n    return new DefaultStitcher({\n      knex: options.knex,\n      logger: options.logger,\n      metrics: options.metrics,\n      strategy: stitchingStrategyFromConfig(config),\n    });\n  }\n\n  constructor(options: {\n    knex: Knex;\n    logger: LoggerService;\n    metrics: MetricsService;\n    strategy: StitchingStrategy;\n  }) {\n    this.knex = options.knex;\n    this.logger = options.logger;\n    this.strategy = options.strategy;\n    this.tracker = progressTracker(\n      options.knex,\n      options.logger,\n      options.metrics,\n    );\n  }\n\n  async start() {\n    if (this.stopFunc) {\n      throw new Error('Stitcher is already started');\n    }\n\n    const { pollingInterval, stitchTimeout } = this.strategy;\n\n    const stopPipeline = startTaskPipeline<DeferredStitchItem>({\n      lowWatermark: 2,\n      highWatermark: 5,\n      pollingIntervalMs: durationToMilliseconds(pollingInterval),\n      loadTasks: async count => {\n        return await this.#getStitchableEntities(count, stitchTimeout);\n      },\n      processTask: async item => {\n        return await this.#stitchOne({\n          entityRef: item.entityRef,\n          stitchTicket: item.stitchTicket,\n          stitchRequestedAt: item.stitchRequestedAt,\n          stitchLeaseExpiresAt: item.stitchLeaseExpiresAt,\n        });\n      },\n    });\n\n    this.stopFunc = () => {\n      stopPipeline();\n    };\n  }\n\n  async stop() {\n    if (this.stopFunc) {\n      this.stopFunc();\n      this.stopFunc = undefined;\n    }\n  }\n\n  async #getStitchableEntities(count: number, stitchTimeout: HumanDuration) {\n    try {\n      return await getDeferredStitchableEntities({\n        knex: this.knex,\n        batchSize: count,\n        stitchTimeout: stitchTimeout,\n      });\n    } catch (error) {\n      this.logger.warn('Failed to load stitchable entities', error);\n      return [];\n    }\n  }\n\n  async #stitchOne(options: {\n    entityRef: string;\n    stitchTicket: string;\n    stitchRequestedAt?: DateTime;\n    stitchLeaseExpiresAt: DeferredStitchItem['stitchLeaseExpiresAt'];\n  }) {\n    const track = this.tracker.stitchStart({\n      entityRef: options.entityRef,\n      stitchRequestedAt: options.stitchRequestedAt,\n    });\n\n    try {\n      const result = await performStitching({\n        knex: this.knex,\n        logger: this.logger,\n        entityRef: options.entityRef,\n        stitchTicket: options.stitchTicket,\n        stitchLeaseExpiresAt: options.stitchLeaseExpiresAt,\n      });\n      track.markComplete(result);\n    } catch (error) {\n      track.markFailed(error);\n    }\n  }\n}\n"],"names":["stitchingStrategyFromConfig","progressTracker","startTaskPipeline","durationToMilliseconds","getDeferredStitchableEntities","performStitching"],"mappings":";;;;;;;;;AAuCO,MAAM,eAAA,CAAgB;AAAA,EACV,IAAA;AAAA,EACA,MAAA;AAAA,EACA,QAAA;AAAA,EACA,OAAA;AAAA,EACT,QAAA;AAAA,EAER,OAAO,UAAA,CACL,MAAA,EACA,OAAA,EAKiB;AACjB,IAAA,OAAO,IAAI,eAAA,CAAgB;AAAA,MACzB,MAAM,OAAA,CAAQ,IAAA;AAAA,MACd,QAAQ,OAAA,CAAQ,MAAA;AAAA,MAChB,SAAS,OAAA,CAAQ,OAAA;AAAA,MACjB,QAAA,EAAUA,kCAA4B,MAAM;AAAA,KAC7C,CAAA;AAAA,EACH;AAAA,EAEA,YAAY,OAAA,EAKT;AACD,IAAA,IAAA,CAAK,OAAO,OAAA,CAAQ,IAAA;AACpB,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,WAAW,OAAA,CAAQ,QAAA;AACxB,IAAA,IAAA,CAAK,OAAA,GAAUC,+BAAA;AAAA,MACb,OAAA,CAAQ,IAAA;AAAA,MACR,OAAA,CAAQ,MAAA;AAAA,MACR,OAAA,CAAQ;AAAA,KACV;AAAA,EACF;AAAA,EAEA,MAAM,KAAA,GAAQ;AACZ,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,MAAM,IAAI,MAAM,6BAA6B,CAAA;AAAA,IAC/C;AAEA,IAAA,MAAM,EAAE,eAAA,EAAiB,aAAA,EAAc,GAAI,IAAA,CAAK,QAAA;AAEhD,IAAA,MAAM,eAAeC,8BAAA,CAAsC;AAAA,MACzD,YAAA,EAAc,CAAA;AAAA,MACd,aAAA,EAAe,CAAA;AAAA,MACf,iBAAA,EAAmBC,+BAAuB,eAAe,CAAA;AAAA,MACzD,SAAA,EAAW,OAAM,KAAA,KAAS;AACxB,QAAA,OAAO,MAAM,IAAA,CAAK,sBAAA,CAAuB,KAAA,EAAO,aAAa,CAAA;AAAA,MAC/D,CAAA;AAAA,MACA,WAAA,EAAa,OAAM,IAAA,KAAQ;AACzB,QAAA,OAAO,MAAM,KAAK,UAAA,CAAW;AAAA,UAC3B,WAAW,IAAA,CAAK,SAAA;AAAA,UAChB,cAAc,IAAA,CAAK,YAAA;AAAA,UACnB,mBAAmB,IAAA,CAAK,iBAAA;AAAA,UACxB,sBAAsB,IAAA,CAAK;AAAA,SAC5B,CAAA;AAAA,MACH;AAAA,KACD,CAAA;AAED,IAAA,IAAA,CAAK,WAAW,MAAM;AACpB,MAAA,YAAA,EAAa;AAAA,IACf,CAAA;AAAA,EACF;AAAA,EAEA,MAAM,IAAA,GAAO;AACX,IAAA,IAAI,KAAK,QAAA,EAAU;AACjB,MAAA,IAAA,CAAK,QAAA,EAAS;AACd,MAAA,IAAA,CAAK,QAAA,GAAW,MAAA;AAAA,IAClB;AAAA,EACF;AAAA,EAEA,MAAM,sBAAA,CAAuB,KAAA,EAAe,aAAA,EAA8B;AACxE,IAAA,IAAI;AACF,MAAA,OAAO,MAAMC,2DAAA,CAA8B;AAAA,QACzC,MAAM,IAAA,CAAK,IAAA;AAAA,QACX,SAAA,EAAW,KAAA;AAAA,QACX;AAAA,OACD,CAAA;AAAA,IACH,SAAS,KAAA,EAAO;AACd,MAAA,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,oCAAA,EAAsC,KAAK,CAAA;AAC5D,MAAA,OAAO,EAAC;AAAA,IACV;AAAA,EACF;AAAA,EAEA,MAAM,WAAW,OAAA,EAKd;AACD,IAAA,MAAM,KAAA,GAAQ,IAAA,CAAK,OAAA,CAAQ,WAAA,CAAY;AAAA,MACrC,WAAW,OAAA,CAAQ,SAAA;AAAA,MACnB,mBAAmB,OAAA,CAAQ;AAAA,KAC5B,CAAA;AAED,IAAA,IAAI;AACF,MAAA,MAAM,MAAA,GAAS,MAAMC,iCAAA,CAAiB;AAAA,QACpC,MAAM,IAAA,CAAK,IAAA;AAAA,QACX,QAAQ,IAAA,CAAK,MAAA;AAAA,QACb,WAAW,OAAA,CAAQ,SAAA;AAAA,QACnB,cAAc,OAAA,CAAQ,YAAA;AAAA,QACtB,sBAAsB,OAAA,CAAQ;AAAA,OAC/B,CAAA;AACD,MAAA,KAAA,CAAM,aAAa,MAAM,CAAA;AAAA,IAC3B,SAAS,KAAA,EAAO;AACd,MAAA,KAAA,CAAM,WAAW,KAAK,CAAA;AAAA,IACxB;AAAA,EACF;AACF;;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend",
-  "version": "4.0.1-next.1",
+  "version": "4.0.1-next.2",
   "description": "The Backstage backend plugin that provides the Backstage catalog",
   "backstage": {
     "role": "backend-plugin",
@@ -75,14 +75,14 @@
     "test": "backstage-cli package test"
   },
   "dependencies": {
-    "@backstage/backend-openapi-utils": "0.7.3-next.1",
+    "@backstage/backend-openapi-utils": "0.7.3-next.2",
     "@backstage/backend-plugin-api": "1.11.0-next.1",
     "@backstage/catalog-client": "1.16.3-next.0",
     "@backstage/catalog-model": "1.10.2-next.0",
     "@backstage/config": "1.3.10-next.0",
     "@backstage/errors": "1.3.2-next.0",
     "@backstage/filter-predicates": "0.1.6-next.0",
-    "@backstage/integration": "2.1.3-next.0",
+    "@backstage/integration": "2.1.3-next.1",
     "@backstage/plugin-catalog-common": "1.2.1-next.0",
     "@backstage/plugin-catalog-node": "2.2.6-next.1",
     "@backstage/plugin-events-node": "0.4.27-next.1",
@@ -110,12 +110,12 @@
     "zod-validation-error": "^5.0.0"
   },
   "devDependencies": {
-    "@backstage/backend-defaults": "0.18.1-next.1",
+    "@backstage/backend-defaults": "0.18.1-next.2",
     "@backstage/backend-test-utils": "1.11.8-next.1",
     "@backstage/cli": "0.36.7-next.0",
     "@backstage/plugin-catalog-backend-module-logs": "0.1.27-next.1",
     "@backstage/plugin-scaffolder-common": "2.3.1-next.0",
-    "@backstage/repo-tools": "0.19.2-next.1",
+    "@backstage/repo-tools": "0.19.2-next.2",
     "@types/core-js": "^2.5.4",
     "@types/express": "^4.17.6",
     "@types/git-url-parse": "^9.0.0",