@backstage/plugin-catalog-backend 4.0.1-next.2 → 4.0.1-next.3

package/CHANGELOG.md

@@ -1,5 +1,16 @@
 # @backstage/plugin-catalog-backend
 
+## 4.0.1-next.3
+
+### Patch Changes
+
+- aa6e7c6: Reduce repeated work for entities that fail stitching by gradually increasing retry delays with jitter. Failed entities remain eligible for recovery, and new stitching requests reset their failure count.
+- d7cc6e5: Reject non-positive stitching polling intervals and lease durations at startup to prevent continuous polling and immediate lease expiry. Configuration errors identify the setting that needs correction.
+- 7c376ff: Improved the performance and planner stability of orphan entity cleanup on large PostgreSQL catalogs. Orphan deletion and updates to related entities are now atomic, so a failed cleanup does not leave related entities waiting indefinitely for updated relations. Cleanup also preserves entities when a new reference is committed while it waits to delete them.
+- Updated dependencies
+  - @backstage/catalog-client@1.16.3-next.1
+  - @backstage/backend-plugin-api@1.11.0-next.2
+
 ## 4.0.1-next.2
 
 ### Patch Changes

package/config.schema.json

@@ -169,7 +169,7 @@
                   "type": "string"
                 }
               ],
-              "description": "Polling interval for tasks in seconds"
+              "description": "Interval between queue polls. Must be greater than zero."
             },
             "stitchTimeout": {
               "anyOf": [
@@ -207,7 +207,7 @@
                   "type": "string"
                 }
               ],
-              "description": "How long to wait for a stitch to complete before giving up in seconds"
+              "description": "Lease duration before unfinished work can be reclaimed. Must be greater than zero. This does not cancel an attempt when its lease expires."
             }
           },
           "description": "The strategy to use when stitching together the final entities."

package/dist/database/operations/stitcher/getDeferredStitchableEntities.cjs.js

@@ -10,7 +10,7 @@ async function getDeferredStitchableEntities(options) {
     knex.client.config.client
   );
   const run = async (tx) => {
-    const items = await tx("stitch_queue").select("entity_ref", "next_stitch_at", "stitch_ticket").where("next_stitch_at", "<=", tx.fn.now()).orderBy("next_stitch_at", "asc").limit(batchSize).modify((qb) => {
+    const items = await tx("stitch_queue").select("entity_ref", "next_stitch_at", "stitch_ticket", "failure_count").where("next_stitch_at", "<=", tx.fn.now()).orderBy("next_stitch_at", "asc").limit(batchSize).modify((qb) => {
       if (useLocking) {
         qb.forUpdate().skipLocked();
       }
@@ -19,11 +19,26 @@ async function getDeferredStitchableEntities(options) {
       return [];
     }
     const stitchTicket = node_crypto.randomUUID();
+    const base = types.durationToMilliseconds(stitchTimeout);
+    const nextStitchAt = (failureCount) => {
+      const delay = Math.max(
+        base,
+        Math.min(base * (failureCount + 1) ** 2, 36e5)
+      );
+      const jitter = failureCount > 0 ? delay * 0.1 * Math.random() : 0;
+      return nowPlus(tx, delay + jitter);
+    };
     const update = tx("stitch_queue").whereIn(
       "entity_ref",
       items.map((i) => i.entity_ref)
     ).update({
-      next_stitch_at: nowPlus(tx, stitchTimeout),
+      next_stitch_at: tx.raw(
+        `case entity_ref ${items.map(() => "when ? then ?").join(" ")} end`,
+        items.flatMap((item) => [
+          item.entity_ref,
+          nextStitchAt(item.failure_count)
+        ])
+      ),
       stitch_ticket: stitchTicket
     });
     let leases;
@@ -58,8 +73,8 @@ async function getDeferredStitchableEntities(options) {
   };
   return knex.isTransaction ? await run(knex) : await knex.transaction(run);
 }
-function nowPlus(knex, duration) {
-  const seconds = types.durationToMilliseconds(duration) / 1e3;
+function nowPlus(knex, milliseconds) {
+  const seconds = milliseconds / 1e3;
   if (knex.client.config.client.includes("sqlite3")) {
     return knex.raw(`datetime('now', ?)`, [`${seconds} seconds`]);
   } else if (knex.client.config.client.includes("mysql")) {

package/dist/database/operations/stitcher/getDeferredStitchableEntities.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"getDeferredStitchableEntities.cjs.js","sources":["../../../../src/database/operations/stitcher/getDeferredStitchableEntities.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { durationToMilliseconds, HumanDuration } from '@backstage/types';\nimport { Knex } from 'knex';\nimport { DateTime } from 'luxon';\nimport { randomUUID } from 'node:crypto';\nimport { timestampToDateTime } from '../../conversion';\nimport { DbStitchQueueRow } from '../../tables';\n\n/**\n * Opaque lease value captured by a deferred stitch claim. Pass it unchanged\n * back to completion using the same database. Do not parse or normalize it:\n * PostgreSQL requires its exact timestamp text to retain microsecond precision;\n * MySQL uses timestamp text to avoid timezone/DST conversion through a Date.\n * SQLite also returns its stored timestamp text.\n */\nexport type StitchLeaseExpiresAt = DbStitchQueueRow['next_stitch_at'] & {\n  readonly __stitchLeaseExpiresAt: unique symbol;\n};\n\n// TODO(freben): There is no retry counter or similar. If items start\n// perpetually crashing during stitching, they'll just get silently retried over\n// and over again, for better or worse. This will be visible in metrics though.\n\n/**\n * Finds entities that are marked for deferred stitching.\n *\n * @remarks\n *\n * This assumes that the stitching strategy is set to deferred.\n *\n * Each claim replaces stitch_ticket, fencing workers whose leases expired even\n * when no new stitch request was made. A ticket may be shared across a batch;\n * it is always checked together with entity_ref.\n *\n * All returned items have their next_stitch_at updated to be moved forward by\n * the given timeout duration. This has the effect that they will be picked up\n * for stitching again in the future, if it hasn't completed by that point for\n * some reason (restarts, crashes, etc).\n */\nexport async function getDeferredStitchableEntities(options: {\n  knex: Knex | Knex.Transaction;\n  batchSize: number;\n  stitchTimeout: HumanDuration;\n}): Promise<\n  Array<{\n    entityRef: string;\n    stitchTicket: string;\n    stitchRequestedAt: DateTime; // the time BEFORE moving it forward by the timeout\n    stitchLeaseExpiresAt: StitchLeaseExpiresAt;\n  }>\n> {\n  const { knex, batchSize, stitchTimeout } = options;\n  const useLocking = ['mysql', 'mysql2', 'pg'].includes(\n    knex.client.config.client,\n  );\n\n  // The SELECT FOR UPDATE SKIP LOCKED + UPDATE must run inside a single\n  // transaction so that the row locks held by FOR UPDATE persist until\n  // next_stitch_at has been bumped. Without the transaction the locks\n  // are released after the SELECT auto-commits, and another worker can\n  // claim the same rows before the UPDATE runs.\n  // SQLite also keeps selection and ticket/lease replacement in one transaction,\n  // without attempting the row-locking clauses that it does not support.\n  const run = async (tx: Knex | Knex.Transaction) => {\n    const items: DbStitchQueueRow[] = await tx('stitch_queue')\n      .select('entity_ref', 'next_stitch_at', 'stitch_ticket')\n      .where('next_stitch_at', '<=', tx.fn.now())\n      .orderBy('next_stitch_at', 'asc')\n      .limit(batchSize)\n      .modify(qb => {\n        if (useLocking) {\n          qb.forUpdate().skipLocked();\n        }\n      });\n\n    if (!items.length) {\n      return [];\n    }\n\n    const stitchTicket = randomUUID();\n    const update = tx<DbStitchQueueRow>('stitch_queue')\n      .whereIn(\n        'entity_ref',\n        items.map(i => i.entity_ref),\n      )\n      .update({\n        next_stitch_at: nowPlus(tx, stitchTimeout),\n        stitch_ticket: stitchTicket,\n      });\n\n    // Preserve PostgreSQL's full timestamp precision for the completion guard;\n    // converting it to a JavaScript Date would truncate microseconds.\n    let leases: Pick<DbStitchQueueRow, 'entity_ref' | 'next_stitch_at'>[];\n    if (tx.client.config.client === 'pg') {\n      leases = await update.returning([\n        'entity_ref',\n        tx.raw('next_stitch_at::text as next_stitch_at'),\n      ]);\n    } else if (String(tx.client.config.client).includes('mysql')) {\n      // MySQL has no UPDATE RETURNING; read the lease on the same locked\n      // connection before committing the claim. Cast to text so the driver\n      // cannot normalize the value through a Date across a local DST gap.\n      await update;\n      leases = await tx<DbStitchQueueRow>('stitch_queue')\n        .select<Pick<DbStitchQueueRow, 'entity_ref' | 'next_stitch_at'>[]>(\n          'entity_ref',\n          tx.raw('CAST(next_stitch_at AS CHAR) as next_stitch_at'),\n        )\n        .whereIn(\n          'entity_ref',\n          items.map(i => i.entity_ref),\n        );\n    } else {\n      leases = await update.returning(['entity_ref', 'next_stitch_at']);\n    }\n    const leaseByRef = new Map(\n      leases.map(i => [i.entity_ref, i.next_stitch_at]),\n    );\n\n    return items.map(i => ({\n      entityRef: i.entity_ref,\n      stitchTicket,\n      stitchRequestedAt: timestampToDateTime(i.next_stitch_at),\n      stitchLeaseExpiresAt: leaseByRef.get(\n        i.entity_ref,\n      )! as StitchLeaseExpiresAt,\n    }));\n  };\n\n  return knex.isTransaction ? await run(knex) : await knex.transaction(run);\n}\n\nfunction nowPlus(knex: Knex, duration: HumanDuration): Knex.Raw {\n  const seconds = durationToMilliseconds(duration) / 1000;\n  if (knex.client.config.client.includes('sqlite3')) {\n    return knex.raw(`datetime('now', ?)`, [`${seconds} seconds`]);\n  } else if (knex.client.config.client.includes('mysql')) {\n    return knex.raw(`now() + interval ${seconds} second`);\n  }\n  return knex.raw(`now() + interval '${seconds} seconds'`);\n}\n"],"names":["randomUUID","timestampToDateTime","durationToMilliseconds"],"mappings":";;;;;;AAsDA,eAAsB,8BAA8B,OAAA,EAWlD;AACA,EAAA,MAAM,EAAE,IAAA,EAAM,SAAA,EAAW,aAAA,EAAc,GAAI,OAAA;AAC3C,EAAA,MAAM,UAAA,GAAa,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,QAAA;AAAA,IAC3C,IAAA,CAAK,OAAO,MAAA,CAAO;AAAA,GACrB;AASA,EAAA,MAAM,GAAA,GAAM,OAAO,EAAA,KAAgC;AACjD,IAAA,MAAM,KAAA,GAA4B,MAAM,EAAA,CAAG,cAAc,CAAA,CACtD,MAAA,CAAO,YAAA,EAAc,gBAAA,EAAkB,eAAe,CAAA,CACtD,KAAA,CAAM,gBAAA,EAAkB,IAAA,EAAM,GAAG,EAAA,CAAG,GAAA,EAAK,CAAA,CACzC,OAAA,CAAQ,gBAAA,EAAkB,KAAK,CAAA,CAC/B,KAAA,CAAM,SAAS,CAAA,CACf,MAAA,CAAO,CAAA,EAAA,KAAM;AACZ,MAAA,IAAI,UAAA,EAAY;AACd,QAAA,EAAA,CAAG,SAAA,GAAY,UAAA,EAAW;AAAA,MAC5B;AAAA,IACF,CAAC,CAAA;AAEH,IAAA,IAAI,CAAC,MAAM,MAAA,EAAQ;AACjB,MAAA,OAAO,EAAC;AAAA,IACV;AAEA,IAAA,MAAM,eAAeA,sBAAA,EAAW;AAChC,IAAA,MAAM,MAAA,GAAS,EAAA,CAAqB,cAAc,CAAA,CAC/C,OAAA;AAAA,MACC,YAAA;AAAA,MACA,KAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,MAE5B,MAAA,CAAO;AAAA,MACN,cAAA,EAAgB,OAAA,CAAQ,EAAA,EAAI,aAAa,CAAA;AAAA,MACzC,aAAA,EAAe;AAAA,KAChB,CAAA;AAIH,IAAA,IAAI,MAAA;AACJ,IAAA,IAAI,EAAA,CAAG,MAAA,CAAO,MAAA,CAAO,MAAA,KAAW,IAAA,EAAM;AACpC,MAAA,MAAA,GAAS,MAAM,OAAO,SAAA,CAAU;AAAA,QAC9B,YAAA;AAAA,QACA,EAAA,CAAG,IAAI,wCAAwC;AAAA,OAChD,CAAA;AAAA,IACH,CAAA,MAAA,IAAW,OAAO,EAAA,CAAG,MAAA,CAAO,OAAO,MAAM,CAAA,CAAE,QAAA,CAAS,OAAO,CAAA,EAAG;AAI5D,MAAA,MAAM,MAAA;AACN,MAAA,MAAA,GAAS,MAAM,EAAA,CAAqB,cAAc,CAAA,CAC/C,MAAA;AAAA,QACC,YAAA;AAAA,QACA,EAAA,CAAG,IAAI,gDAAgD;AAAA,OACzD,CACC,OAAA;AAAA,QACC,YAAA;AAAA,QACA,KAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,OAC7B;AAAA,IACJ,CAAA,MAAO;AACL,MAAA,MAAA,GAAS,MAAM,MAAA,CAAO,SAAA,CAAU,CAAC,YAAA,EAAc,gBAAgB,CAAC,CAAA;AAAA,IAClE;AACA,IAAA,MAAM,aAAa,IAAI,GAAA;AAAA,MACrB,MAAA,CAAO,IAAI,CAAA,CAAA,KAAK,CAAC,EAAE,UAAA,EAAY,CAAA,CAAE,cAAc,CAAC;AAAA,KAClD;AAEA,IAAA,OAAO,KAAA,CAAM,IAAI,CAAA,CAAA,MAAM;AAAA,MACrB,WAAW,CAAA,CAAE,UAAA;AAAA,MACb,YAAA;AAAA,MACA,iBAAA,EAAmBC,8BAAA,CAAoB,CAAA,CAAE,cAAc,CAAA;AAAA,MACvD,sBAAsB,UAAA,CAAW,GAAA;AAAA,QAC/B,CAAA,CAAE;AAAA;AACJ,KACF,CAAE,CAAA;AAAA,EACJ,CAAA;AAEA,EAAA,OAAO,IAAA,CAAK,gBAAgB,MAAM,GAAA,CAAI,IAAI,CAAA,GAAI,MAAM,IAAA,CAAK,WAAA,CAAY,GAAG,CAAA;AAC1E;AAEA,SAAS,OAAA,CAAQ,MAAY,QAAA,EAAmC;AAC9D,EAAA,MAAM,OAAA,GAAUC,4BAAA,CAAuB,QAAQ,CAAA,GAAI,GAAA;AACnD,EAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AACjD,IAAA,OAAO,KAAK,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,OAAO,UAAU,CAAC,CAAA;AAAA,EAC9D,WAAW,IAAA,CAAK,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACtD,IAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,iBAAA,EAAoB,OAAO,CAAA,OAAA,CAAS,CAAA;AAAA,EACtD;AACA,EAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,kBAAA,EAAqB,OAAO,CAAA,SAAA,CAAW,CAAA;AACzD;;"}
+{"version":3,"file":"getDeferredStitchableEntities.cjs.js","sources":["../../../../src/database/operations/stitcher/getDeferredStitchableEntities.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { durationToMilliseconds, HumanDuration } from '@backstage/types';\nimport { Knex } from 'knex';\nimport { DateTime } from 'luxon';\nimport { randomUUID } from 'node:crypto';\nimport { timestampToDateTime } from '../../conversion';\nimport { DbStitchQueueRow } from '../../tables';\n\n/**\n * Opaque lease value captured by a deferred stitch claim. Pass it unchanged\n * back to completion using the same database. Do not parse or normalize it:\n * PostgreSQL requires its exact timestamp text to retain microsecond precision;\n * MySQL uses timestamp text to avoid timezone/DST conversion through a Date.\n * SQLite also returns its stored timestamp text.\n */\nexport type StitchLeaseExpiresAt = DbStitchQueueRow['next_stitch_at'] & {\n  readonly __stitchLeaseExpiresAt: unique symbol;\n};\n\n/**\n * Finds entities that are marked for deferred stitching.\n *\n * @remarks\n *\n * This assumes that the stitching strategy is set to deferred.\n *\n * Each claim replaces stitch_ticket, fencing workers whose leases expired even\n * when no new stitch request was made. A ticket may be shared across a batch;\n * it is always checked together with entity_ref.\n *\n * All returned items have their next_stitch_at updated to be moved forward by\n * the configured timeout, extended by quadratic backoff and jitter after\n * failures. This has the effect that they will be picked up\n * for stitching again in the future, if it hasn't completed by that point for\n * some reason (restarts, crashes, etc).\n */\nexport async function getDeferredStitchableEntities(options: {\n  knex: Knex | Knex.Transaction;\n  batchSize: number;\n  stitchTimeout: HumanDuration;\n}): Promise<\n  Array<{\n    entityRef: string;\n    stitchTicket: string;\n    stitchRequestedAt: DateTime; // the time BEFORE moving it forward by the timeout\n    stitchLeaseExpiresAt: StitchLeaseExpiresAt;\n  }>\n> {\n  const { knex, batchSize, stitchTimeout } = options;\n  const useLocking = ['mysql', 'mysql2', 'pg'].includes(\n    knex.client.config.client,\n  );\n\n  // The SELECT FOR UPDATE SKIP LOCKED + UPDATE must run inside a single\n  // transaction so that the row locks held by FOR UPDATE persist until\n  // next_stitch_at has been bumped. Without the transaction the locks\n  // are released after the SELECT auto-commits, and another worker can\n  // claim the same rows before the UPDATE runs.\n  // SQLite also keeps selection and ticket/lease replacement in one transaction,\n  // without attempting the row-locking clauses that it does not support.\n  const run = async (tx: Knex | Knex.Transaction) => {\n    const items: DbStitchQueueRow[] = await tx('stitch_queue')\n      .select('entity_ref', 'next_stitch_at', 'stitch_ticket', 'failure_count')\n      .where('next_stitch_at', '<=', tx.fn.now())\n      .orderBy('next_stitch_at', 'asc')\n      .limit(batchSize)\n      .modify(qb => {\n        if (useLocking) {\n          qb.forUpdate().skipLocked();\n        }\n      });\n\n    if (!items.length) {\n      return [];\n    }\n\n    const stitchTicket = randomUUID();\n    const base = durationToMilliseconds(stitchTimeout);\n    const nextStitchAt = (failureCount: number) => {\n      const delay = Math.max(\n        base,\n        Math.min(base * (failureCount + 1) ** 2, 3_600_000),\n      );\n      const jitter = failureCount > 0 ? delay * 0.1 * Math.random() : 0;\n      return nowPlus(tx, delay + jitter);\n    };\n    const update = tx<DbStitchQueueRow>('stitch_queue')\n      .whereIn(\n        'entity_ref',\n        items.map(i => i.entity_ref),\n      )\n      .update({\n        next_stitch_at: tx.raw(\n          `case entity_ref ${items.map(() => 'when ? then ?').join(' ')} end`,\n          items.flatMap(item => [\n            item.entity_ref,\n            nextStitchAt(item.failure_count),\n          ]),\n        ),\n        stitch_ticket: stitchTicket,\n      });\n\n    // Preserve PostgreSQL's full timestamp precision for the completion guard;\n    // converting it to a JavaScript Date would truncate microseconds.\n    let leases: Pick<DbStitchQueueRow, 'entity_ref' | 'next_stitch_at'>[];\n    if (tx.client.config.client === 'pg') {\n      leases = await update.returning([\n        'entity_ref',\n        tx.raw('next_stitch_at::text as next_stitch_at'),\n      ]);\n    } else if (String(tx.client.config.client).includes('mysql')) {\n      // MySQL has no UPDATE RETURNING; read the lease on the same locked\n      // connection before committing the claim. Cast to text so the driver\n      // cannot normalize the value through a Date across a local DST gap.\n      await update;\n      leases = await tx<DbStitchQueueRow>('stitch_queue')\n        .select<Pick<DbStitchQueueRow, 'entity_ref' | 'next_stitch_at'>[]>(\n          'entity_ref',\n          tx.raw('CAST(next_stitch_at AS CHAR) as next_stitch_at'),\n        )\n        .whereIn(\n          'entity_ref',\n          items.map(i => i.entity_ref),\n        );\n    } else {\n      leases = await update.returning(['entity_ref', 'next_stitch_at']);\n    }\n    const leaseByRef = new Map(\n      leases.map(i => [i.entity_ref, i.next_stitch_at]),\n    );\n\n    return items.map(i => ({\n      entityRef: i.entity_ref,\n      stitchTicket,\n      stitchRequestedAt: timestampToDateTime(i.next_stitch_at),\n      stitchLeaseExpiresAt: leaseByRef.get(\n        i.entity_ref,\n      )! as StitchLeaseExpiresAt,\n    }));\n  };\n\n  return knex.isTransaction ? await run(knex) : await knex.transaction(run);\n}\n\nfunction nowPlus(knex: Knex, milliseconds: number): Knex.Raw {\n  const seconds = milliseconds / 1000;\n  if (knex.client.config.client.includes('sqlite3')) {\n    return knex.raw(`datetime('now', ?)`, [`${seconds} seconds`]);\n  } else if (knex.client.config.client.includes('mysql')) {\n    return knex.raw(`now() + interval ${seconds} second`);\n  }\n  return knex.raw(`now() + interval '${seconds} seconds'`);\n}\n"],"names":["randomUUID","durationToMilliseconds","timestampToDateTime"],"mappings":";;;;;;AAmDA,eAAsB,8BAA8B,OAAA,EAWlD;AACA,EAAA,MAAM,EAAE,IAAA,EAAM,SAAA,EAAW,aAAA,EAAc,GAAI,OAAA;AAC3C,EAAA,MAAM,UAAA,GAAa,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,QAAA;AAAA,IAC3C,IAAA,CAAK,OAAO,MAAA,CAAO;AAAA,GACrB;AASA,EAAA,MAAM,GAAA,GAAM,OAAO,EAAA,KAAgC;AACjD,IAAA,MAAM,KAAA,GAA4B,MAAM,EAAA,CAAG,cAAc,CAAA,CACtD,MAAA,CAAO,YAAA,EAAc,gBAAA,EAAkB,eAAA,EAAiB,eAAe,CAAA,CACvE,KAAA,CAAM,gBAAA,EAAkB,IAAA,EAAM,EAAA,CAAG,EAAA,CAAG,GAAA,EAAK,CAAA,CACzC,OAAA,CAAQ,gBAAA,EAAkB,KAAK,CAAA,CAC/B,KAAA,CAAM,SAAS,CAAA,CACf,MAAA,CAAO,CAAA,EAAA,KAAM;AACZ,MAAA,IAAI,UAAA,EAAY;AACd,QAAA,EAAA,CAAG,SAAA,GAAY,UAAA,EAAW;AAAA,MAC5B;AAAA,IACF,CAAC,CAAA;AAEH,IAAA,IAAI,CAAC,MAAM,MAAA,EAAQ;AACjB,MAAA,OAAO,EAAC;AAAA,IACV;AAEA,IAAA,MAAM,eAAeA,sBAAA,EAAW;AAChC,IAAA,MAAM,IAAA,GAAOC,6BAAuB,aAAa,CAAA;AACjD,IAAA,MAAM,YAAA,GAAe,CAAC,YAAA,KAAyB;AAC7C,MAAA,MAAM,QAAQ,IAAA,CAAK,GAAA;AAAA,QACjB,IAAA;AAAA,QACA,KAAK,GAAA,CAAI,IAAA,GAAA,CAAQ,YAAA,GAAe,CAAA,KAAM,GAAG,IAAS;AAAA,OACpD;AACA,MAAA,MAAM,SAAS,YAAA,GAAe,CAAA,GAAI,QAAQ,GAAA,GAAM,IAAA,CAAK,QAAO,GAAI,CAAA;AAChE,MAAA,OAAO,OAAA,CAAQ,EAAA,EAAI,KAAA,GAAQ,MAAM,CAAA;AAAA,IACnC,CAAA;AACA,IAAA,MAAM,MAAA,GAAS,EAAA,CAAqB,cAAc,CAAA,CAC/C,OAAA;AAAA,MACC,YAAA;AAAA,MACA,KAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,MAE5B,MAAA,CAAO;AAAA,MACN,gBAAgB,EAAA,CAAG,GAAA;AAAA,QACjB,CAAA,gBAAA,EAAmB,MAAM,GAAA,CAAI,MAAM,eAAe,CAAA,CAAE,IAAA,CAAK,GAAG,CAAC,CAAA,IAAA,CAAA;AAAA,QAC7D,KAAA,CAAM,QAAQ,CAAA,IAAA,KAAQ;AAAA,UACpB,IAAA,CAAK,UAAA;AAAA,UACL,YAAA,CAAa,KAAK,aAAa;AAAA,SAChC;AAAA,OACH;AAAA,MACA,aAAA,EAAe;AAAA,KAChB,CAAA;AAIH,IAAA,IAAI,MAAA;AACJ,IAAA,IAAI,EAAA,CAAG,MAAA,CAAO,MAAA,CAAO,MAAA,KAAW,IAAA,EAAM;AACpC,MAAA,MAAA,GAAS,MAAM,OAAO,SAAA,CAAU;AAAA,QAC9B,YAAA;AAAA,QACA,EAAA,CAAG,IAAI,wCAAwC;AAAA,OAChD,CAAA;AAAA,IACH,CAAA,MAAA,IAAW,OAAO,EAAA,CAAG,MAAA,CAAO,OAAO,MAAM,CAAA,CAAE,QAAA,CAAS,OAAO,CAAA,EAAG;AAI5D,MAAA,MAAM,MAAA;AACN,MAAA,MAAA,GAAS,MAAM,EAAA,CAAqB,cAAc,CAAA,CAC/C,MAAA;AAAA,QACC,YAAA;AAAA,QACA,EAAA,CAAG,IAAI,gDAAgD;AAAA,OACzD,CACC,OAAA;AAAA,QACC,YAAA;AAAA,QACA,KAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,OAC7B;AAAA,IACJ,CAAA,MAAO;AACL,MAAA,MAAA,GAAS,MAAM,MAAA,CAAO,SAAA,CAAU,CAAC,YAAA,EAAc,gBAAgB,CAAC,CAAA;AAAA,IAClE;AACA,IAAA,MAAM,aAAa,IAAI,GAAA;AAAA,MACrB,MAAA,CAAO,IAAI,CAAA,CAAA,KAAK,CAAC,EAAE,UAAA,EAAY,CAAA,CAAE,cAAc,CAAC;AAAA,KAClD;AAEA,IAAA,OAAO,KAAA,CAAM,IAAI,CAAA,CAAA,MAAM;AAAA,MACrB,WAAW,CAAA,CAAE,UAAA;AAAA,MACb,YAAA;AAAA,MACA,iBAAA,EAAmBC,8BAAA,CAAoB,CAAA,CAAE,cAAc,CAAA;AAAA,MACvD,sBAAsB,UAAA,CAAW,GAAA;AAAA,QAC/B,CAAA,CAAE;AAAA;AACJ,KACF,CAAE,CAAA;AAAA,EACJ,CAAA;AAEA,EAAA,OAAO,IAAA,CAAK,gBAAgB,MAAM,GAAA,CAAI,IAAI,CAAA,GAAI,MAAM,IAAA,CAAK,WAAA,CAAY,GAAG,CAAA;AAC1E;AAEA,SAAS,OAAA,CAAQ,MAAY,YAAA,EAAgC;AAC3D,EAAA,MAAM,UAAU,YAAA,GAAe,GAAA;AAC/B,EAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AACjD,IAAA,OAAO,KAAK,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,OAAO,UAAU,CAAC,CAAA;AAAA,EAC9D,WAAW,IAAA,CAAK,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACtD,IAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,iBAAA,EAAoB,OAAO,CAAA,OAAA,CAAS,CAAA;AAAA,EACtD;AACA,EAAA,OAAO,IAAA,CAAK,GAAA,CAAI,CAAA,kBAAA,EAAqB,OAAO,CAAA,SAAA,CAAW,CAAA;AACzD;;"}

package/dist/database/operations/stitcher/markDeferredStitchFailed.cjs.js

@@ -0,0 +1,12 @@
+'use strict';
+
+async function markDeferredStitchFailed(options) {
+  const { knex, entityRef, stitchTicket, stitchLeaseExpiresAt } = options;
+  const updated = await knex("stitch_queue").where("entity_ref", entityRef).where("stitch_ticket", stitchTicket).where("next_stitch_at", stitchLeaseExpiresAt).increment("failure_count", 1);
+  if (!updated) {
+    await knex("stitch_queue").where("entity_ref", entityRef).whereNot("stitch_ticket", stitchTicket).where("next_stitch_at", stitchLeaseExpiresAt).update({ next_stitch_at: knex.fn.now() });
+  }
+}
+
+exports.markDeferredStitchFailed = markDeferredStitchFailed;
+//# sourceMappingURL=markDeferredStitchFailed.cjs.js.map

package/dist/database/operations/stitcher/markDeferredStitchFailed.cjs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markDeferredStitchFailed.cjs.js","sources":["../../../../src/database/operations/stitcher/markDeferredStitchFailed.ts"],"sourcesContent":["/*\n * Copyright 2026 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport { DbStitchQueueRow } from '../../tables';\nimport { StitchLeaseExpiresAt } from './getDeferredStitchableEntities';\n\n/**\n * Records a failed attempt only while its claim still owns the queued request.\n * New requests keep their fresh retry budget and become eligible after this\n * attempt settles. Reclaimed leases must never be changed by a stale failure.\n */\nexport async function markDeferredStitchFailed(options: {\n  knex: Knex | Knex.Transaction;\n  entityRef: string;\n  stitchTicket: string;\n  stitchLeaseExpiresAt: StitchLeaseExpiresAt;\n}): Promise<void> {\n  const { knex, entityRef, stitchTicket, stitchLeaseExpiresAt } = options;\n  const updated = await knex<DbStitchQueueRow>('stitch_queue')\n    .where('entity_ref', entityRef)\n    .where('stitch_ticket', stitchTicket)\n    .where('next_stitch_at', stitchLeaseExpiresAt)\n    .increment('failure_count', 1);\n\n  if (!updated) {\n    await knex<DbStitchQueueRow>('stitch_queue')\n      .where('entity_ref', entityRef)\n      .whereNot('stitch_ticket', stitchTicket)\n      .where('next_stitch_at', stitchLeaseExpiresAt)\n      .update({ next_stitch_at: knex.fn.now() });\n  }\n}\n"],"names":[],"mappings":";;AAyBA,eAAsB,yBAAyB,OAAA,EAK7B;AAChB,EAAA,MAAM,EAAE,IAAA,EAAM,SAAA,EAAW,YAAA,EAAc,sBAAqB,GAAI,OAAA;AAChE,EAAA,MAAM,UAAU,MAAM,IAAA,CAAuB,cAAc,CAAA,CACxD,KAAA,CAAM,cAAc,SAAS,CAAA,CAC7B,MAAM,eAAA,EAAiB,YAAY,EACnC,KAAA,CAAM,gBAAA,EAAkB,oBAAoB,CAAA,CAC5C,SAAA,CAAU,iBAAiB,CAAC,CAAA;AAE/B,EAAA,IAAI,CAAC,OAAA,EAAS;AACZ,IAAA,MAAM,IAAA,CAAuB,cAAc,CAAA,CACxC,KAAA,CAAM,cAAc,SAAS,CAAA,CAC7B,QAAA,CAAS,eAAA,EAAiB,YAAY,CAAA,CACtC,MAAM,gBAAA,EAAkB,oBAAoB,EAC5C,MAAA,CAAO,EAAE,gBAAgB,IAAA,CAAK,EAAA,CAAG,GAAA,EAAI,EAAG,CAAA;AAAA,EAC7C;AACF;;"}

package/dist/database/operations/stitcher/markForStitching.cjs.js

@@ -21,9 +21,10 @@ async function markForStitching(options) {
           chunk.map((ref) => ({
             entity_ref: ref,
             stitch_ticket: ticket,
+            failure_count: 0,
             next_stitch_at: knex.fn.now()
           }))
-        ).onConflict("entity_ref").merge(["stitch_ticket"]);
+        ).onConflict("entity_ref").merge(["stitch_ticket", "failure_count"]);
       }
     }, knex);
   }
@@ -35,9 +36,10 @@ async function markForStitching(options) {
           refreshStateRows.map((row) => ({
             entity_ref: row.entity_ref,
             stitch_ticket: ticket,
+            failure_count: 0,
             next_stitch_at: knex.fn.now()
           }))
-        ).onConflict("entity_ref").merge(["stitch_ticket"]);
+        ).onConflict("entity_ref").merge(["stitch_ticket", "failure_count"]);
       }
     }, knex);
   }

package/dist/database/operations/stitcher/markForStitching.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"markForStitching.cjs.js","sources":["../../../../src/database/operations/stitcher/markForStitching.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport splitToChunks from 'lodash/chunk';\nimport { randomUUID as uuid } from 'node:crypto';\nimport { DbRefreshStateRow, DbStitchQueueRow } from '../../tables';\nimport { retryOnDeadlock } from '../../util';\n\nconst UPDATE_CHUNK_SIZE = 100; // Smaller chunks reduce contention\n\n/**\n * Marks a number of entities for stitching some time in the near\n * future.\n *\n * @remarks\n *\n * If there is no existing stitch_queue entry, a new row is created with\n * next_stitch_at set to now() (immediately eligible). If an entry already\n * exists, only the stitch_ticket is updated — the timestamp is left\n * unchanged. This prevents interrupting an in-progress stitch: the worker\n * that claimed the entry pushed next_stitch_at forward by the timeout\n * duration, and we don't want to yank it back to now() which would allow\n * another worker to claim the same entity and cause overlapping stitches.\n *\n * When the in-progress stitch completes, markDeferredStitchCompleted\n * detects the ticket change and makes the entry immediately eligible for\n * the follow-up stitch.\n */\nexport async function markForStitching(options: {\n  knex: Knex | Knex.Transaction;\n  entityRefs?: Iterable<string>;\n  entityIds?: Iterable<string>;\n}): Promise<void> {\n  const entityRefs = sortSplit(options.entityRefs);\n  const entityIds = sortSplit(options.entityIds);\n  const knex = options.knex;\n\n  // It's OK that this is shared across stitch_queue rows; it just needs to\n  // be uniquely generated for every new stitch request.\n  const ticket = uuid();\n\n  for (const chunk of entityRefs) {\n    await retryOnDeadlock(async () => {\n      if (chunk.length > 0) {\n        await knex<DbStitchQueueRow>('stitch_queue')\n          .insert(\n            chunk.map(ref => ({\n              entity_ref: ref,\n              stitch_ticket: ticket,\n              next_stitch_at: knex.fn.now(),\n            })),\n          )\n          .onConflict('entity_ref')\n          .merge(['stitch_ticket']);\n      }\n    }, knex);\n  }\n\n  for (const chunk of entityIds) {\n    await retryOnDeadlock(async () => {\n      // Look up entity_refs from refresh_state by entity_id\n      const refreshStateRows = await knex<DbRefreshStateRow>('refresh_state')\n        .select('entity_ref')\n        .whereIn('entity_id', chunk);\n\n      if (refreshStateRows.length > 0) {\n        await knex<DbStitchQueueRow>('stitch_queue')\n          .insert(\n            refreshStateRows.map(row => ({\n              entity_ref: row.entity_ref,\n              stitch_ticket: ticket,\n              next_stitch_at: knex.fn.now(),\n            })),\n          )\n          .onConflict('entity_ref')\n          .merge(['stitch_ticket']);\n      }\n    }, knex);\n  }\n}\n\nfunction sortSplit(input: Iterable<string> | undefined): string[][] {\n  if (!input) {\n    return [];\n  }\n  const array = Array.isArray(input) ? input.slice() : [...input];\n  array.sort();\n  return splitToChunks(array, UPDATE_CHUNK_SIZE);\n}\n"],"names":["uuid","retryOnDeadlock","splitToChunks"],"mappings":";;;;;;;;;;AAsBA,MAAM,iBAAA,GAAoB,GAAA;AAoB1B,eAAsB,iBAAiB,OAAA,EAIrB;AAChB,EAAA,MAAM,UAAA,GAAa,SAAA,CAAU,OAAA,CAAQ,UAAU,CAAA;AAC/C,EAAA,MAAM,SAAA,GAAY,SAAA,CAAU,OAAA,CAAQ,SAAS,CAAA;AAC7C,EAAA,MAAM,OAAO,OAAA,CAAQ,IAAA;AAIrB,EAAA,MAAM,SAASA,sBAAA,EAAK;AAEpB,EAAA,KAAA,MAAW,SAAS,UAAA,EAAY;AAC9B,IAAA,MAAMC,qBAAgB,YAAY;AAChC,MAAA,IAAI,KAAA,CAAM,SAAS,CAAA,EAAG;AACpB,QAAA,MAAM,IAAA,CAAuB,cAAc,CAAA,CACxC,MAAA;AAAA,UACC,KAAA,CAAM,IAAI,CAAA,GAAA,MAAQ;AAAA,YAChB,UAAA,EAAY,GAAA;AAAA,YACZ,aAAA,EAAe,MAAA;AAAA,YACf,cAAA,EAAgB,IAAA,CAAK,EAAA,CAAG,GAAA;AAAI,WAC9B,CAAE;AAAA,UAEH,UAAA,CAAW,YAAY,EACvB,KAAA,CAAM,CAAC,eAAe,CAAC,CAAA;AAAA,MAC5B;AAAA,IACF,GAAG,IAAI,CAAA;AAAA,EACT;AAEA,EAAA,KAAA,MAAW,SAAS,SAAA,EAAW;AAC7B,IAAA,MAAMA,qBAAgB,YAAY;AAEhC,MAAA,MAAM,gBAAA,GAAmB,MAAM,IAAA,CAAwB,eAAe,CAAA,CACnE,OAAO,YAAY,CAAA,CACnB,OAAA,CAAQ,WAAA,EAAa,KAAK,CAAA;AAE7B,MAAA,IAAI,gBAAA,CAAiB,SAAS,CAAA,EAAG;AAC/B,QAAA,MAAM,IAAA,CAAuB,cAAc,CAAA,CACxC,MAAA;AAAA,UACC,gBAAA,CAAiB,IAAI,CAAA,GAAA,MAAQ;AAAA,YAC3B,YAAY,GAAA,CAAI,UAAA;AAAA,YAChB,aAAA,EAAe,MAAA;AAAA,YACf,cAAA,EAAgB,IAAA,CAAK,EAAA,CAAG,GAAA;AAAI,WAC9B,CAAE;AAAA,UAEH,UAAA,CAAW,YAAY,EACvB,KAAA,CAAM,CAAC,eAAe,CAAC,CAAA;AAAA,MAC5B;AAAA,IACF,GAAG,IAAI,CAAA;AAAA,EACT;AACF;AAEA,SAAS,UAAU,KAAA,EAAiD;AAClE,EAAA,IAAI,CAAC,KAAA,EAAO;AACV,IAAA,OAAO,EAAC;AAAA,EACV;AACA,EAAA,MAAM,KAAA,GAAQ,KAAA,CAAM,OAAA,CAAQ,KAAK,CAAA,GAAI,MAAM,KAAA,EAAM,GAAI,CAAC,GAAG,KAAK,CAAA;AAC9D,EAAA,KAAA,CAAM,IAAA,EAAK;AACX,EAAA,OAAOC,8BAAA,CAAc,OAAO,iBAAiB,CAAA;AAC/C;;"}
+{"version":3,"file":"markForStitching.cjs.js","sources":["../../../../src/database/operations/stitcher/markForStitching.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport splitToChunks from 'lodash/chunk';\nimport { randomUUID as uuid } from 'node:crypto';\nimport { DbRefreshStateRow, DbStitchQueueRow } from '../../tables';\nimport { retryOnDeadlock } from '../../util';\n\nconst UPDATE_CHUNK_SIZE = 100; // Smaller chunks reduce contention\n\n/**\n * Marks a number of entities for stitching some time in the near\n * future.\n *\n * @remarks\n *\n * If there is no existing stitch_queue entry, a new row is created with\n * next_stitch_at set to now() (immediately eligible). If an entry already\n * exists, only the stitch_ticket is updated — the timestamp is left\n * unchanged. This prevents interrupting an in-progress stitch: the worker\n * that claimed the entry pushed next_stitch_at forward by the timeout\n * duration, and we don't want to yank it back to now() which would allow\n * another worker to claim the same entity and cause overlapping stitches.\n *\n * When the in-progress stitch completes, markDeferredStitchCompleted\n * detects the ticket change and makes the entry immediately eligible for\n * the follow-up stitch.\n */\nexport async function markForStitching(options: {\n  knex: Knex | Knex.Transaction;\n  entityRefs?: Iterable<string>;\n  entityIds?: Iterable<string>;\n}): Promise<void> {\n  const entityRefs = sortSplit(options.entityRefs);\n  const entityIds = sortSplit(options.entityIds);\n  const knex = options.knex;\n\n  // It's OK that this is shared across stitch_queue rows; it just needs to\n  // be uniquely generated for every new stitch request.\n  const ticket = uuid();\n\n  for (const chunk of entityRefs) {\n    await retryOnDeadlock(async () => {\n      if (chunk.length > 0) {\n        await knex<DbStitchQueueRow>('stitch_queue')\n          .insert(\n            chunk.map(ref => ({\n              entity_ref: ref,\n              stitch_ticket: ticket,\n              failure_count: 0,\n              next_stitch_at: knex.fn.now(),\n            })),\n          )\n          .onConflict('entity_ref')\n          .merge(['stitch_ticket', 'failure_count']);\n      }\n    }, knex);\n  }\n\n  for (const chunk of entityIds) {\n    await retryOnDeadlock(async () => {\n      // Look up entity_refs from refresh_state by entity_id\n      const refreshStateRows = await knex<DbRefreshStateRow>('refresh_state')\n        .select('entity_ref')\n        .whereIn('entity_id', chunk);\n\n      if (refreshStateRows.length > 0) {\n        await knex<DbStitchQueueRow>('stitch_queue')\n          .insert(\n            refreshStateRows.map(row => ({\n              entity_ref: row.entity_ref,\n              stitch_ticket: ticket,\n              failure_count: 0,\n              next_stitch_at: knex.fn.now(),\n            })),\n          )\n          .onConflict('entity_ref')\n          .merge(['stitch_ticket', 'failure_count']);\n      }\n    }, knex);\n  }\n}\n\nfunction sortSplit(input: Iterable<string> | undefined): string[][] {\n  if (!input) {\n    return [];\n  }\n  const array = Array.isArray(input) ? input.slice() : [...input];\n  array.sort();\n  return splitToChunks(array, UPDATE_CHUNK_SIZE);\n}\n"],"names":["uuid","retryOnDeadlock","splitToChunks"],"mappings":";;;;;;;;;;AAsBA,MAAM,iBAAA,GAAoB,GAAA;AAoB1B,eAAsB,iBAAiB,OAAA,EAIrB;AAChB,EAAA,MAAM,UAAA,GAAa,SAAA,CAAU,OAAA,CAAQ,UAAU,CAAA;AAC/C,EAAA,MAAM,SAAA,GAAY,SAAA,CAAU,OAAA,CAAQ,SAAS,CAAA;AAC7C,EAAA,MAAM,OAAO,OAAA,CAAQ,IAAA;AAIrB,EAAA,MAAM,SAASA,sBAAA,EAAK;AAEpB,EAAA,KAAA,MAAW,SAAS,UAAA,EAAY;AAC9B,IAAA,MAAMC,qBAAgB,YAAY;AAChC,MAAA,IAAI,KAAA,CAAM,SAAS,CAAA,EAAG;AACpB,QAAA,MAAM,IAAA,CAAuB,cAAc,CAAA,CACxC,MAAA;AAAA,UACC,KAAA,CAAM,IAAI,CAAA,GAAA,MAAQ;AAAA,YAChB,UAAA,EAAY,GAAA;AAAA,YACZ,aAAA,EAAe,MAAA;AAAA,YACf,aAAA,EAAe,CAAA;AAAA,YACf,cAAA,EAAgB,IAAA,CAAK,EAAA,CAAG,GAAA;AAAI,WAC9B,CAAE;AAAA,SACJ,CACC,WAAW,YAAY,CAAA,CACvB,MAAM,CAAC,eAAA,EAAiB,eAAe,CAAC,CAAA;AAAA,MAC7C;AAAA,IACF,GAAG,IAAI,CAAA;AAAA,EACT;AAEA,EAAA,KAAA,MAAW,SAAS,SAAA,EAAW;AAC7B,IAAA,MAAMA,qBAAgB,YAAY;AAEhC,MAAA,MAAM,gBAAA,GAAmB,MAAM,IAAA,CAAwB,eAAe,CAAA,CACnE,OAAO,YAAY,CAAA,CACnB,OAAA,CAAQ,WAAA,EAAa,KAAK,CAAA;AAE7B,MAAA,IAAI,gBAAA,CAAiB,SAAS,CAAA,EAAG;AAC/B,QAAA,MAAM,IAAA,CAAuB,cAAc,CAAA,CACxC,MAAA;AAAA,UACC,gBAAA,CAAiB,IAAI,CAAA,GAAA,MAAQ;AAAA,YAC3B,YAAY,GAAA,CAAI,UAAA;AAAA,YAChB,aAAA,EAAe,MAAA;AAAA,YACf,aAAA,EAAe,CAAA;AAAA,YACf,cAAA,EAAgB,IAAA,CAAK,EAAA,CAAG,GAAA;AAAI,WAC9B,CAAE;AAAA,SACJ,CACC,WAAW,YAAY,CAAA,CACvB,MAAM,CAAC,eAAA,EAAiB,eAAe,CAAC,CAAA;AAAA,MAC7C;AAAA,IACF,GAAG,IAAI,CAAA;AAAA,EACT;AACF;AAEA,SAAS,UAAU,KAAA,EAAiD;AAClE,EAAA,IAAI,CAAC,KAAA,EAAO;AACV,IAAA,OAAO,EAAC;AAAA,EACV;AACA,EAAA,MAAM,KAAA,GAAQ,KAAA,CAAM,OAAA,CAAQ,KAAK,CAAA,GAAI,MAAM,KAAA,EAAM,GAAI,CAAC,GAAG,KAAK,CAAA;AAC9D,EAAA,KAAA,CAAM,IAAA,EAAK;AACX,EAAA,OAAOC,8BAAA,CAAc,OAAO,iBAAiB,CAAA;AAC/C;;"}

package/dist/database/operations/stitcher/performStitching.cjs.js

@@ -6,6 +6,7 @@ var node_crypto = require('node:crypto');
 var stableStringify = require('fast-json-stable-stringify');
 var buildEntitySearch = require('./buildEntitySearch.cjs.js');
 var markDeferredStitchCompleted = require('./markDeferredStitchCompleted.cjs.js');
+var markDeferredStitchFailed = require('./markDeferredStitchFailed.cjs.js');
 var syncSearchRows = require('./syncSearchRows.cjs.js');
 var util = require('../../util.cjs.js');
 
@@ -181,6 +182,19 @@ async function performStitching(options) {
     stitchResult = "succeeded";
     return "changed";
   } catch (error) {
+    try {
+      await markDeferredStitchFailed.markDeferredStitchFailed({
+        knex,
+        entityRef,
+        stitchTicket,
+        stitchLeaseExpiresAt
+      });
+    } catch (recordError) {
+      logger.warn(
+        `Failed to record stitch failure for ${entityRef}`,
+        recordError
+      );
+    }
     throw error;
   } finally {
     if (stitchResult) {

package/dist/database/operations/stitcher/performStitching.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"performStitching.cjs.js","sources":["../../../../src/database/operations/stitcher/performStitching.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from '@backstage/catalog-client';\nimport {\n  ANNOTATION_EDIT_URL,\n  ANNOTATION_VIEW_URL,\n  Entity,\n  EntityRelation,\n} from '@backstage/catalog-model';\nimport { AlphaEntity, EntityStatusItem } from '@backstage/catalog-model/alpha';\nimport { SerializedError } from '@backstage/errors';\nimport { Knex } from 'knex';\nimport { createHash } from 'node:crypto';\nimport stableStringify from 'fast-json-stable-stringify';\nimport { DbFinalEntitiesRow, DbStitchQueueRow } from '../../tables';\nimport { buildEntitySearch } from './buildEntitySearch';\nimport { markDeferredStitchCompleted } from './markDeferredStitchCompleted';\nimport { syncSearchRows } from './syncSearchRows';\nimport { StitchLeaseExpiresAt } from './getDeferredStitchableEntities';\nimport { LoggerService } from '@backstage/backend-plugin-api';\nimport { retryOnDeadlock } from '../../util';\n\nclass StitchPublicationSupersededError extends Error {}\n\nfunction generateStableHash(entity: Entity) {\n  return createHash('sha1')\n    .update(stableStringify({ ...entity }))\n    .digest('hex');\n}\n\n// See https://github.com/facebook/react/blob/f0cf832e1d0c8544c36aa8b310960885a11a847c/packages/react-dom-bindings/src/shared/sanitizeURL.js\nconst scriptProtocolPattern =\n  // eslint-disable-next-line no-control-regex\n  /^[\\u0000-\\u001F ]*j[\\r\\n\\t]*a[\\r\\n\\t]*v[\\r\\n\\t]*a[\\r\\n\\t]*s[\\r\\n\\t]*c[\\r\\n\\t]*r[\\r\\n\\t]*i[\\r\\n\\t]*p[\\r\\n\\t]*t[\\r\\n\\t]*\\:/i;\n\n/**\n * Performs the act of stitching - to take all of the various outputs from the\n * ingestion process, and stitching them together into the final entity JSON\n * shape.\n */\nexport async function performStitching(options: {\n  knex: Knex | Knex.Transaction;\n  logger: LoggerService;\n  entityRef: string;\n  stitchTicket: string;\n  stitchLeaseExpiresAt: StitchLeaseExpiresAt;\n}): Promise<'changed' | 'unchanged' | 'abandoned'> {\n  const { knex, logger, entityRef, stitchTicket, stitchLeaseExpiresAt } =\n    options;\n\n  // Settle the claim on any completion, without disturbing a successor's\n  // lease. A new request during this lease becomes eligible immediately.\n  // Exceptions skip cleanup so the entity gets retried at a later time.\n  let stitchResult: 'succeeded' | 'abandoned' | undefined;\n\n  try {\n    // Selecting from refresh_state (with an optional left join to\n    // final_entities for the previous hash) should yield exactly one row,\n    // except in abnormal cases where the entity was deleted between the\n    // stitch request and now.\n    const [processedResult, relationsResult] = await Promise.all([\n      knex\n        .with('incoming_references', function incomingReferences(builder) {\n          return builder\n            .from('refresh_state_references')\n            .where({ target_entity_ref: entityRef })\n            .count({ count: '*' });\n        })\n        .select({\n          entityId: 'refresh_state.entity_id',\n          processedEntity: 'refresh_state.processed_entity',\n          errors: 'refresh_state.errors',\n          incomingReferenceCount: 'incoming_references.count',\n          previousHash: 'final_entities.hash',\n        })\n        .from('refresh_state')\n        .where({ 'refresh_state.entity_ref': entityRef })\n        .crossJoin(knex.raw('incoming_references'))\n        .leftOuterJoin('final_entities', {\n          'final_entities.entity_id': 'refresh_state.entity_id',\n        }),\n      knex\n        .distinct({\n          relationType: 'type',\n          relationTarget: 'target_entity_ref',\n        })\n        .from('relations')\n        .where({ source_entity_ref: entityRef })\n        .orderBy('relationType', 'asc')\n        .orderBy('relationTarget', 'asc'),\n    ]);\n\n    // If there were no rows returned, it would mean that there was no\n    // matching row even in the refresh_state. This can happen for example\n    // if we emit a relation to something that hasn't been ingested yet.\n    // It's safe to ignore this stitch attempt in that case.\n    if (!processedResult.length) {\n      logger.debug(\n        `Unable to stitch ${entityRef}, item does not exist in refresh state table`,\n      );\n      stitchResult = 'abandoned';\n      return 'abandoned';\n    }\n\n    const {\n      entityId,\n      processedEntity,\n      errors,\n      incomingReferenceCount,\n      previousHash,\n    } = processedResult[0];\n\n    // If there was no processed entity in place, the target hasn't been\n    // through the processing steps yet. It's safe to ignore this stitch\n    // attempt in that case, since another stitch will be triggered when\n    // that processing has finished.\n    if (!processedEntity) {\n      logger.debug(\n        `Unable to stitch ${entityRef}, the entity has not yet been processed`,\n      );\n      stitchResult = 'abandoned';\n      return 'abandoned';\n    }\n\n    // Grab the processed entity and stitch all of the relevant data into\n    // it\n    const entity = JSON.parse(processedEntity) as AlphaEntity;\n    const isOrphan = Number(incomingReferenceCount) === 0;\n    let statusItems: EntityStatusItem[] = [];\n\n    if (isOrphan) {\n      logger.debug(`${entityRef} is an orphan`);\n      entity.metadata.annotations = {\n        ...entity.metadata.annotations,\n        ['backstage.io/orphan']: 'true',\n      };\n    }\n    if (errors) {\n      const parsedErrors = JSON.parse(errors) as SerializedError[];\n      if (Array.isArray(parsedErrors) && parsedErrors.length) {\n        statusItems = parsedErrors.map(e => ({\n          type: ENTITY_STATUS_CATALOG_PROCESSING_TYPE,\n          level: 'error',\n          message: `${e.name}: ${e.message}`,\n          error: e,\n        }));\n      }\n    }\n    // We opt to do this check here as we otherwise can't guarantee that it will be run after all processors\n    for (const annotation of [ANNOTATION_VIEW_URL, ANNOTATION_EDIT_URL]) {\n      const value = entity.metadata.annotations?.[annotation];\n      if (typeof value === 'string' && scriptProtocolPattern.test(value)) {\n        entity.metadata.annotations![annotation] =\n          'https://backstage.io/annotation-rejected-for-security-reasons';\n      }\n    }\n\n    // TODO: entityRef is lower case and should be uppercase in the final\n    // result\n    entity.relations = relationsResult\n      .filter(row => row.relationType /* exclude null row, if relevant */)\n      .map<EntityRelation>(row => ({\n        type: row.relationType!,\n        targetRef: row.relationTarget!,\n      }));\n    if (statusItems.length) {\n      entity.status = {\n        ...entity.status,\n        items: [...(entity.status?.items ?? []), ...statusItems],\n      };\n    }\n\n    // If the output entity was actually not changed, just abort\n    const hash = generateStableHash(entity);\n    if (hash === previousHash) {\n      logger.debug(`Skipped stitching of ${entityRef}, no changes`);\n      stitchResult = 'succeeded';\n      return 'unchanged';\n    }\n\n    entity.metadata.uid = entityId;\n    if (!entity.metadata.etag) {\n      // If the original data source did not have its own etag handling,\n      // use the hash as a good-quality etag\n      entity.metadata.etag = hash;\n    }\n\n    // This may throw if the entity is invalid, so we call it before\n    // the final_entities write, even though we may end up not needing\n    // to write the search index.\n    const searchEntries = buildEntitySearch(entityId, entity);\n\n    // Guard against concurrent stitchers: if our stitch_ticket no longer\n    // matches stitch_queue, another worker has newer data and we should\n    // not overwrite it. PostgreSQL guards both insert and merge in the\n    // publication statement and rechecks the captured claim after the upsert.\n    // SQLite checks inside its transaction as well\n    // as guarding the merge path. MySQL does not support\n    // ON CONFLICT ... DO UPDATE ... WHERE, so its separate check retains\n    // a best-effort TOCTOU window. Do not turn it into a queue lock here:\n    // that would introduce a new lock-order dependency during publication.\n    const isMySQL = String(knex.client.config.client).includes('mysql');\n    const isPostgres = knex.client.config.client === 'pg';\n\n    // The final_entities row and the search index rows have to land\n    // together. If the search write fails on its own, the next stitch\n    // attempt reads back the hash we just wrote and returns early as\n    // unchanged, so the entity keeps a stale search index for good. It\n    // stays readable by direct lookup and drops out of every filtered or\n    // sorted list query.\n    const writeOutcome = await retryOnDeadlock(\n      () =>\n        knex.transaction(async tx => {\n          // Recheck on every attempt: a request or reclaimed claim may have\n          // superseded this worker while its previous transaction rolled back.\n          // Do not lock the queue here; processing locks refresh state before queue.\n          if (!isPostgres) {\n            const ticketValid = await tx<DbStitchQueueRow>('stitch_queue')\n              .where('entity_ref', entityRef)\n              .where('stitch_ticket', stitchTicket)\n              .first();\n            if (!ticketValid) return 'abandoned' as const;\n          }\n          // Knex's typed insert overload excludes raw INSERT SELECT bodies.\n          let upsert = tx('final_entities')\n            .insert(\n              !isPostgres\n                ? {\n                    entity_id: entityId,\n                    entity_ref: entityRef,\n                    final_entity: JSON.stringify(entity),\n                    hash,\n                    last_updated_at: tx.fn.now(),\n                  }\n                : tx.raw(\n                    '(entity_id, entity_ref, final_entity, hash, last_updated_at) ' +\n                      'select ?, ?, ?, ?, CURRENT_TIMESTAMP ' +\n                      'where exists (select 1 from stitch_queue where entity_ref = ? and stitch_ticket = ?)',\n                    [\n                      entityId,\n                      entityRef,\n                      JSON.stringify(entity),\n                      hash,\n                      entityRef,\n                      stitchTicket,\n                    ],\n                  ),\n            )\n            .onConflict('entity_id')\n            .merge(['final_entity', 'hash', 'last_updated_at']);\n\n          if (!isMySQL) {\n            upsert = upsert.where(\n              tx.raw(\n                'exists (select 1 from stitch_queue where entity_ref = ? and stitch_ticket = ?)',\n                [entityRef, stitchTicket],\n              ),\n            );\n          }\n\n          await upsert;\n\n          // Verify the write took effect. On PostgreSQL, also recheck ownership\n          // using this statement's fresh Read Committed snapshot: the upsert's\n          // snapshot may predate a reclaim that committed while it was running.\n          // A successful upsert holds the final-row lock until commit, so a\n          // successor cannot publish past us after this check. Do not lock the\n          // queue, which would introduce a publication/deletion lock inversion.\n          if (!isMySQL) {\n            const written = await tx<DbFinalEntitiesRow>('final_entities')\n              .where('entity_id', entityId)\n              .where('hash', hash)\n              .modify(qb => {\n                if (isPostgres) {\n                  qb.whereExists(\n                    tx<DbStitchQueueRow>('stitch_queue')\n                      .select(tx.raw('1'))\n                      .where('entity_ref', entityRef)\n                      .where('stitch_ticket', stitchTicket)\n                      .where('next_stitch_at', stitchLeaseExpiresAt),\n                  );\n                }\n              })\n              .select(tx.raw('1'))\n              .first();\n            if (!written) {\n              // Returning normally would commit any stale write already made.\n              throw new StitchPublicationSupersededError();\n            }\n          }\n\n          await syncSearchRows(tx, entityId, searchEntries);\n\n          return 'changed' as const;\n        }),\n      knex,\n    ).catch(error => {\n      // Knex has rolled the publication back before rejecting the transaction.\n      // Supersession is a normal abandonment, not a failure or deadlock retry.\n      if (error instanceof StitchPublicationSupersededError) {\n        return 'abandoned' as const;\n      }\n      throw error;\n    });\n\n    if (writeOutcome === 'abandoned') {\n      logger.debug(`Entity ${entityRef} is already stitched, skipping write.`);\n      stitchResult = 'abandoned';\n      return 'abandoned';\n    }\n\n    stitchResult = 'succeeded';\n    return 'changed';\n  } catch (error) {\n    throw error;\n  } finally {\n    if (stitchResult) {\n      await markDeferredStitchCompleted({\n        knex: knex,\n        entityRef,\n        stitchTicket,\n        stitchLeaseExpiresAt,\n      });\n    }\n  }\n}\n"],"names":["createHash","stableStringify","ENTITY_STATUS_CATALOG_PROCESSING_TYPE","ANNOTATION_VIEW_URL","ANNOTATION_EDIT_URL","buildEntitySearch","retryOnDeadlock","syncSearchRows","markDeferredStitchCompleted"],"mappings":";;;;;;;;;;;;;;;AAoCA,MAAM,yCAAyC,KAAA,CAAM;AAAC;AAEtD,SAAS,mBAAmB,MAAA,EAAgB;AAC1C,EAAA,OAAOA,sBAAA,CAAW,MAAM,CAAA,CACrB,MAAA,CAAOC,gCAAA,CAAgB,EAAE,GAAG,MAAA,EAAQ,CAAC,CAAA,CACrC,MAAA,CAAO,KAAK,CAAA;AACjB;AAGA,MAAM,qBAAA;AAAA;AAAA,EAEJ;AAAA,CAAA;AAOF,eAAsB,iBAAiB,OAAA,EAMY;AACjD,EAAA,MAAM,EAAE,IAAA,EAAM,MAAA,EAAQ,SAAA,EAAW,YAAA,EAAc,sBAAqB,GAClE,OAAA;AAKF,EAAA,IAAI,YAAA;AAEJ,EAAA,IAAI;AAKF,IAAA,MAAM,CAAC,eAAA,EAAiB,eAAe,CAAA,GAAI,MAAM,QAAQ,GAAA,CAAI;AAAA,MAC3D,IAAA,CACG,IAAA,CAAK,qBAAA,EAAuB,SAAS,mBAAmB,OAAA,EAAS;AAChE,QAAA,OAAO,OAAA,CACJ,IAAA,CAAK,0BAA0B,CAAA,CAC/B,MAAM,EAAE,iBAAA,EAAmB,SAAA,EAAW,CAAA,CACtC,KAAA,CAAM,EAAE,KAAA,EAAO,KAAK,CAAA;AAAA,MACzB,CAAC,EACA,MAAA,CAAO;AAAA,QACN,QAAA,EAAU,yBAAA;AAAA,QACV,eAAA,EAAiB,gCAAA;AAAA,QACjB,MAAA,EAAQ,sBAAA;AAAA,QACR,sBAAA,EAAwB,2BAAA;AAAA,QACxB,YAAA,EAAc;AAAA,OACf,CAAA,CACA,IAAA,CAAK,eAAe,CAAA,CACpB,KAAA,CAAM,EAAE,0BAAA,EAA4B,SAAA,EAAW,CAAA,CAC/C,UAAU,IAAA,CAAK,GAAA,CAAI,qBAAqB,CAAC,CAAA,CACzC,cAAc,gBAAA,EAAkB;AAAA,QAC/B,0BAAA,EAA4B;AAAA,OAC7B,CAAA;AAAA,MACH,KACG,QAAA,CAAS;AAAA,QACR,YAAA,EAAc,MAAA;AAAA,QACd,cAAA,EAAgB;AAAA,OACjB,CAAA,CACA,IAAA,CAAK,WAAW,CAAA,CAChB,MAAM,EAAE,iBAAA,EAAmB,SAAA,EAAW,EACtC,OAAA,CAAQ,cAAA,EAAgB,KAAK,CAAA,CAC7B,OAAA,CAAQ,kBAAkB,KAAK;AAAA,KACnC,CAAA;AAMD,IAAA,IAAI,CAAC,gBAAgB,MAAA,EAAQ;AAC3B,MAAA,MAAA,CAAO,KAAA;AAAA,QACL,oBAAoB,SAAS,CAAA,4CAAA;AAAA,OAC/B;AACA,MAAA,YAAA,GAAe,WAAA;AACf,MAAA,OAAO,WAAA;AAAA,IACT;AAEA,IAAA,MAAM;AAAA,MACJ,QAAA;AAAA,MACA,eAAA;AAAA,MACA,MAAA;AAAA,MACA,sBAAA;AAAA,MACA;AAAA,KACF,GAAI,gBAAgB,CAAC,CAAA;AAMrB,IAAA,IAAI,CAAC,eAAA,EAAiB;AACpB,MAAA,MAAA,CAAO,KAAA;AAAA,QACL,oBAAoB,SAAS,CAAA,uCAAA;AAAA,OAC/B;AACA,MAAA,YAAA,GAAe,WAAA;AACf,MAAA,OAAO,WAAA;AAAA,IACT;AAIA,IAAA,MAAM,MAAA,GAAS,IAAA,CAAK,KAAA,CAAM,eAAe,CAAA;AACzC,IAAA,MAAM,QAAA,GAAW,MAAA,CAAO,sBAAsB,CAAA,KAAM,CAAA;AACpD,IAAA,IAAI,cAAkC,EAAC;AAEvC,IAAA,IAAI,QAAA,EAAU;AACZ,MAAA,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,SAAS,CAAA,aAAA,CAAe,CAAA;AACxC,MAAA,MAAA,CAAO,SAAS,WAAA,GAAc;AAAA,QAC5B,GAAG,OAAO,QAAA,CAAS,WAAA;AAAA,QACnB,CAAC,qBAAqB,GAAG;AAAA,OAC3B;AAAA,IACF;AACA,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,MAAM,YAAA,GAAe,IAAA,CAAK,KAAA,CAAM,MAAM,CAAA;AACtC,MAAA,IAAI,KAAA,CAAM,OAAA,CAAQ,YAAY,CAAA,IAAK,aAAa,MAAA,EAAQ;AACtD,QAAA,WAAA,GAAc,YAAA,CAAa,IAAI,CAAA,CAAA,MAAM;AAAA,UACnC,IAAA,EAAMC,mDAAA;AAAA,UACN,KAAA,EAAO,OAAA;AAAA,UACP,SAAS,CAAA,EAAG,CAAA,CAAE,IAAI,CAAA,EAAA,EAAK,EAAE,OAAO,CAAA,CAAA;AAAA,UAChC,KAAA,EAAO;AAAA,SACT,CAAE,CAAA;AAAA,MACJ;AAAA,IACF;AAEA,IAAA,KAAA,MAAW,UAAA,IAAc,CAACC,gCAAA,EAAqBC,gCAAmB,CAAA,EAAG;AACnE,MAAA,MAAM,KAAA,GAAQ,MAAA,CAAO,QAAA,CAAS,WAAA,GAAc,UAAU,CAAA;AACtD,MAAA,IAAI,OAAO,KAAA,KAAU,QAAA,IAAY,qBAAA,CAAsB,IAAA,CAAK,KAAK,CAAA,EAAG;AAClE,QAAA,MAAA,CAAO,QAAA,CAAS,WAAA,CAAa,UAAU,CAAA,GACrC,+DAAA;AAAA,MACJ;AAAA,IACF;AAIA,IAAA,MAAA,CAAO,YAAY,eAAA,CAChB,MAAA;AAAA,MAAO,SAAO,GAAA,CAAI;AAAA;AAAA,KAAgD,CAClE,IAAoB,CAAA,GAAA,MAAQ;AAAA,MAC3B,MAAM,GAAA,CAAI,YAAA;AAAA,MACV,WAAW,GAAA,CAAI;AAAA,KACjB,CAAE,CAAA;AACJ,IAAA,IAAI,YAAY,MAAA,EAAQ;AACtB,MAAA,MAAA,CAAO,MAAA,GAAS;AAAA,QACd,GAAG,MAAA,CAAO,MAAA;AAAA,QACV,KAAA,EAAO,CAAC,GAAI,MAAA,CAAO,QAAQ,KAAA,IAAS,EAAC,EAAI,GAAG,WAAW;AAAA,OACzD;AAAA,IACF;AAGA,IAAA,MAAM,IAAA,GAAO,mBAAmB,MAAM,CAAA;AACtC,IAAA,IAAI,SAAS,YAAA,EAAc;AACzB,MAAA,MAAA,CAAO,KAAA,CAAM,CAAA,qBAAA,EAAwB,SAAS,CAAA,YAAA,CAAc,CAAA;AAC5D,MAAA,YAAA,GAAe,WAAA;AACf,MAAA,OAAO,WAAA;AAAA,IACT;AAEA,IAAA,MAAA,CAAO,SAAS,GAAA,GAAM,QAAA;AACtB,IAAA,IAAI,CAAC,MAAA,CAAO,QAAA,CAAS,IAAA,EAAM;AAGzB,MAAA,MAAA,CAAO,SAAS,IAAA,GAAO,IAAA;AAAA,IACzB;AAKA,IAAA,MAAM,aAAA,GAAgBC,mCAAA,CAAkB,QAAA,EAAU,MAAM,CAAA;AAWxD,IAAA,MAAM,OAAA,GAAU,OAAO,IAAA,CAAK,MAAA,CAAO,OAAO,MAAM,CAAA,CAAE,SAAS,OAAO,CAAA;AAClE,IAAA,MAAM,UAAA,GAAa,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,MAAA,KAAW,IAAA;AAQjD,IAAA,MAAM,eAAe,MAAMC,oBAAA;AAAA,MACzB,MACE,IAAA,CAAK,WAAA,CAAY,OAAM,EAAA,KAAM;AAI3B,QAAA,IAAI,CAAC,UAAA,EAAY;AACf,UAAA,MAAM,WAAA,GAAc,MAAM,EAAA,CAAqB,cAAc,CAAA,CAC1D,KAAA,CAAM,YAAA,EAAc,SAAS,CAAA,CAC7B,KAAA,CAAM,eAAA,EAAiB,YAAY,EACnC,KAAA,EAAM;AACT,UAAA,IAAI,CAAC,aAAa,OAAO,WAAA;AAAA,QAC3B;AAEA,QAAA,IAAI,MAAA,GAAS,EAAA,CAAG,gBAAgB,CAAA,CAC7B,MAAA;AAAA,UACC,CAAC,UAAA,GACG;AAAA,YACE,SAAA,EAAW,QAAA;AAAA,YACX,UAAA,EAAY,SAAA;AAAA,YACZ,YAAA,EAAc,IAAA,CAAK,SAAA,CAAU,MAAM,CAAA;AAAA,YACnC,IAAA;AAAA,YACA,eAAA,EAAiB,EAAA,CAAG,EAAA,CAAG,GAAA;AAAI,cAE7B,EAAA,CAAG,GAAA;AAAA,YACD,wLAAA;AAAA,YAGA;AAAA,cACE,QAAA;AAAA,cACA,SAAA;AAAA,cACA,IAAA,CAAK,UAAU,MAAM,CAAA;AAAA,cACrB,IAAA;AAAA,cACA,SAAA;AAAA,cACA;AAAA;AACF;AACF,SACN,CACC,WAAW,WAAW,CAAA,CACtB,MAAM,CAAC,cAAA,EAAgB,MAAA,EAAQ,iBAAiB,CAAC,CAAA;AAEpD,QAAA,IAAI,CAAC,OAAA,EAAS;AACZ,UAAA,MAAA,GAAS,MAAA,CAAO,KAAA;AAAA,YACd,EAAA,CAAG,GAAA;AAAA,cACD,gFAAA;AAAA,cACA,CAAC,WAAW,YAAY;AAAA;AAC1B,WACF;AAAA,QACF;AAEA,QAAA,MAAM,MAAA;AAQN,QAAA,IAAI,CAAC,OAAA,EAAS;AACZ,UAAA,MAAM,OAAA,GAAU,MAAM,EAAA,CAAuB,gBAAgB,EAC1D,KAAA,CAAM,WAAA,EAAa,QAAQ,CAAA,CAC3B,KAAA,CAAM,MAAA,EAAQ,IAAI,CAAA,CAClB,OAAO,CAAA,EAAA,KAAM;AACZ,YAAA,IAAI,UAAA,EAAY;AACd,cAAA,EAAA,CAAG,WAAA;AAAA,gBACD,GAAqB,cAAc,CAAA,CAChC,OAAO,EAAA,CAAG,GAAA,CAAI,GAAG,CAAC,CAAA,CAClB,MAAM,YAAA,EAAc,SAAS,EAC7B,KAAA,CAAM,eAAA,EAAiB,YAAY,CAAA,CACnC,KAAA,CAAM,kBAAkB,oBAAoB;AAAA,eACjD;AAAA,YACF;AAAA,UACF,CAAC,EACA,MAAA,CAAO,EAAA,CAAG,IAAI,GAAG,CAAC,EAClB,KAAA,EAAM;AACT,UAAA,IAAI,CAAC,OAAA,EAAS;AAEZ,YAAA,MAAM,IAAI,gCAAA,EAAiC;AAAA,UAC7C;AAAA,QACF;AAEA,QAAA,MAAMC,6BAAA,CAAe,EAAA,EAAI,QAAA,EAAU,aAAa,CAAA;AAEhD,QAAA,OAAO,SAAA;AAAA,MACT,CAAC,CAAA;AAAA,MACH;AAAA,KACF,CAAE,MAAM,CAAA,KAAA,KAAS;AAGf,MAAA,IAAI,iBAAiB,gCAAA,EAAkC;AACrD,QAAA,OAAO,WAAA;AAAA,MACT;AACA,MAAA,MAAM,KAAA;AAAA,IACR,CAAC,CAAA;AAED,IAAA,IAAI,iBAAiB,WAAA,EAAa;AAChC,MAAA,MAAA,CAAO,KAAA,CAAM,CAAA,OAAA,EAAU,SAAS,CAAA,qCAAA,CAAuC,CAAA;AACvE,MAAA,YAAA,GAAe,WAAA;AACf,MAAA,OAAO,WAAA;AAAA,IACT;AAEA,IAAA,YAAA,GAAe,WAAA;AACf,IAAA,OAAO,SAAA;AAAA,EACT,SAAS,KAAA,EAAO;AACd,IAAA,MAAM,KAAA;AAAA,EACR,CAAA,SAAE;AACA,IAAA,IAAI,YAAA,EAAc;AAChB,MAAA,MAAMC,uDAAA,CAA4B;AAAA,QAChC,IAAA;AAAA,QACA,SAAA;AAAA,QACA,YAAA;AAAA,QACA;AAAA,OACD,CAAA;AAAA,IACH;AAAA,EACF;AACF;;"}
+{"version":3,"file":"performStitching.cjs.js","sources":["../../../../src/database/operations/stitcher/performStitching.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from '@backstage/catalog-client';\nimport {\n  ANNOTATION_EDIT_URL,\n  ANNOTATION_VIEW_URL,\n  Entity,\n  EntityRelation,\n} from '@backstage/catalog-model';\nimport { AlphaEntity, EntityStatusItem } from '@backstage/catalog-model/alpha';\nimport { SerializedError } from '@backstage/errors';\nimport { Knex } from 'knex';\nimport { createHash } from 'node:crypto';\nimport stableStringify from 'fast-json-stable-stringify';\nimport { DbFinalEntitiesRow, DbStitchQueueRow } from '../../tables';\nimport { buildEntitySearch } from './buildEntitySearch';\nimport { markDeferredStitchCompleted } from './markDeferredStitchCompleted';\nimport { markDeferredStitchFailed } from './markDeferredStitchFailed';\nimport { syncSearchRows } from './syncSearchRows';\nimport { StitchLeaseExpiresAt } from './getDeferredStitchableEntities';\nimport { LoggerService } from '@backstage/backend-plugin-api';\nimport { retryOnDeadlock } from '../../util';\n\nclass StitchPublicationSupersededError extends Error {}\n\nfunction generateStableHash(entity: Entity) {\n  return createHash('sha1')\n    .update(stableStringify({ ...entity }))\n    .digest('hex');\n}\n\n// See https://github.com/facebook/react/blob/f0cf832e1d0c8544c36aa8b310960885a11a847c/packages/react-dom-bindings/src/shared/sanitizeURL.js\nconst scriptProtocolPattern =\n  // eslint-disable-next-line no-control-regex\n  /^[\\u0000-\\u001F ]*j[\\r\\n\\t]*a[\\r\\n\\t]*v[\\r\\n\\t]*a[\\r\\n\\t]*s[\\r\\n\\t]*c[\\r\\n\\t]*r[\\r\\n\\t]*i[\\r\\n\\t]*p[\\r\\n\\t]*t[\\r\\n\\t]*\\:/i;\n\n/**\n * Performs the act of stitching - to take all of the various outputs from the\n * ingestion process, and stitching them together into the final entity JSON\n * shape.\n */\nexport async function performStitching(options: {\n  knex: Knex | Knex.Transaction;\n  logger: LoggerService;\n  entityRef: string;\n  stitchTicket: string;\n  stitchLeaseExpiresAt: StitchLeaseExpiresAt;\n}): Promise<'changed' | 'unchanged' | 'abandoned'> {\n  const { knex, logger, entityRef, stitchTicket, stitchLeaseExpiresAt } =\n    options;\n\n  // Settle the claim on any completion, without disturbing a successor's\n  // lease. A new request during this lease becomes eligible immediately.\n  // Exceptions skip cleanup so the entity gets retried at a later time.\n  let stitchResult: 'succeeded' | 'abandoned' | undefined;\n\n  try {\n    // Selecting from refresh_state (with an optional left join to\n    // final_entities for the previous hash) should yield exactly one row,\n    // except in abnormal cases where the entity was deleted between the\n    // stitch request and now.\n    const [processedResult, relationsResult] = await Promise.all([\n      knex\n        .with('incoming_references', function incomingReferences(builder) {\n          return builder\n            .from('refresh_state_references')\n            .where({ target_entity_ref: entityRef })\n            .count({ count: '*' });\n        })\n        .select({\n          entityId: 'refresh_state.entity_id',\n          processedEntity: 'refresh_state.processed_entity',\n          errors: 'refresh_state.errors',\n          incomingReferenceCount: 'incoming_references.count',\n          previousHash: 'final_entities.hash',\n        })\n        .from('refresh_state')\n        .where({ 'refresh_state.entity_ref': entityRef })\n        .crossJoin(knex.raw('incoming_references'))\n        .leftOuterJoin('final_entities', {\n          'final_entities.entity_id': 'refresh_state.entity_id',\n        }),\n      knex\n        .distinct({\n          relationType: 'type',\n          relationTarget: 'target_entity_ref',\n        })\n        .from('relations')\n        .where({ source_entity_ref: entityRef })\n        .orderBy('relationType', 'asc')\n        .orderBy('relationTarget', 'asc'),\n    ]);\n\n    // If there were no rows returned, it would mean that there was no\n    // matching row even in the refresh_state. This can happen for example\n    // if we emit a relation to something that hasn't been ingested yet.\n    // It's safe to ignore this stitch attempt in that case.\n    if (!processedResult.length) {\n      logger.debug(\n        `Unable to stitch ${entityRef}, item does not exist in refresh state table`,\n      );\n      stitchResult = 'abandoned';\n      return 'abandoned';\n    }\n\n    const {\n      entityId,\n      processedEntity,\n      errors,\n      incomingReferenceCount,\n      previousHash,\n    } = processedResult[0];\n\n    // If there was no processed entity in place, the target hasn't been\n    // through the processing steps yet. It's safe to ignore this stitch\n    // attempt in that case, since another stitch will be triggered when\n    // that processing has finished.\n    if (!processedEntity) {\n      logger.debug(\n        `Unable to stitch ${entityRef}, the entity has not yet been processed`,\n      );\n      stitchResult = 'abandoned';\n      return 'abandoned';\n    }\n\n    // Grab the processed entity and stitch all of the relevant data into\n    // it\n    const entity = JSON.parse(processedEntity) as AlphaEntity;\n    const isOrphan = Number(incomingReferenceCount) === 0;\n    let statusItems: EntityStatusItem[] = [];\n\n    if (isOrphan) {\n      logger.debug(`${entityRef} is an orphan`);\n      entity.metadata.annotations = {\n        ...entity.metadata.annotations,\n        ['backstage.io/orphan']: 'true',\n      };\n    }\n    if (errors) {\n      const parsedErrors = JSON.parse(errors) as SerializedError[];\n      if (Array.isArray(parsedErrors) && parsedErrors.length) {\n        statusItems = parsedErrors.map(e => ({\n          type: ENTITY_STATUS_CATALOG_PROCESSING_TYPE,\n          level: 'error',\n          message: `${e.name}: ${e.message}`,\n          error: e,\n        }));\n      }\n    }\n    // We opt to do this check here as we otherwise can't guarantee that it will be run after all processors\n    for (const annotation of [ANNOTATION_VIEW_URL, ANNOTATION_EDIT_URL]) {\n      const value = entity.metadata.annotations?.[annotation];\n      if (typeof value === 'string' && scriptProtocolPattern.test(value)) {\n        entity.metadata.annotations![annotation] =\n          'https://backstage.io/annotation-rejected-for-security-reasons';\n      }\n    }\n\n    // TODO: entityRef is lower case and should be uppercase in the final\n    // result\n    entity.relations = relationsResult\n      .filter(row => row.relationType /* exclude null row, if relevant */)\n      .map<EntityRelation>(row => ({\n        type: row.relationType!,\n        targetRef: row.relationTarget!,\n      }));\n    if (statusItems.length) {\n      entity.status = {\n        ...entity.status,\n        items: [...(entity.status?.items ?? []), ...statusItems],\n      };\n    }\n\n    // If the output entity was actually not changed, just abort\n    const hash = generateStableHash(entity);\n    if (hash === previousHash) {\n      logger.debug(`Skipped stitching of ${entityRef}, no changes`);\n      stitchResult = 'succeeded';\n      return 'unchanged';\n    }\n\n    entity.metadata.uid = entityId;\n    if (!entity.metadata.etag) {\n      // If the original data source did not have its own etag handling,\n      // use the hash as a good-quality etag\n      entity.metadata.etag = hash;\n    }\n\n    // This may throw if the entity is invalid, so we call it before\n    // the final_entities write, even though we may end up not needing\n    // to write the search index.\n    const searchEntries = buildEntitySearch(entityId, entity);\n\n    // Guard against concurrent stitchers: if our stitch_ticket no longer\n    // matches stitch_queue, another worker has newer data and we should\n    // not overwrite it. PostgreSQL guards both insert and merge in the\n    // publication statement and rechecks the captured claim after the upsert.\n    // SQLite checks inside its transaction as well\n    // as guarding the merge path. MySQL does not support\n    // ON CONFLICT ... DO UPDATE ... WHERE, so its separate check retains\n    // a best-effort TOCTOU window. Do not turn it into a queue lock here:\n    // that would introduce a new lock-order dependency during publication.\n    const isMySQL = String(knex.client.config.client).includes('mysql');\n    const isPostgres = knex.client.config.client === 'pg';\n\n    // The final_entities row and the search index rows have to land\n    // together. If the search write fails on its own, the next stitch\n    // attempt reads back the hash we just wrote and returns early as\n    // unchanged, so the entity keeps a stale search index for good. It\n    // stays readable by direct lookup and drops out of every filtered or\n    // sorted list query.\n    const writeOutcome = await retryOnDeadlock(\n      () =>\n        knex.transaction(async tx => {\n          // Recheck on every attempt: a request or reclaimed claim may have\n          // superseded this worker while its previous transaction rolled back.\n          // Do not lock the queue here; processing locks refresh state before queue.\n          if (!isPostgres) {\n            const ticketValid = await tx<DbStitchQueueRow>('stitch_queue')\n              .where('entity_ref', entityRef)\n              .where('stitch_ticket', stitchTicket)\n              .first();\n            if (!ticketValid) return 'abandoned' as const;\n          }\n          // Knex's typed insert overload excludes raw INSERT SELECT bodies.\n          let upsert = tx('final_entities')\n            .insert(\n              !isPostgres\n                ? {\n                    entity_id: entityId,\n                    entity_ref: entityRef,\n                    final_entity: JSON.stringify(entity),\n                    hash,\n                    last_updated_at: tx.fn.now(),\n                  }\n                : tx.raw(\n                    '(entity_id, entity_ref, final_entity, hash, last_updated_at) ' +\n                      'select ?, ?, ?, ?, CURRENT_TIMESTAMP ' +\n                      'where exists (select 1 from stitch_queue where entity_ref = ? and stitch_ticket = ?)',\n                    [\n                      entityId,\n                      entityRef,\n                      JSON.stringify(entity),\n                      hash,\n                      entityRef,\n                      stitchTicket,\n                    ],\n                  ),\n            )\n            .onConflict('entity_id')\n            .merge(['final_entity', 'hash', 'last_updated_at']);\n\n          if (!isMySQL) {\n            upsert = upsert.where(\n              tx.raw(\n                'exists (select 1 from stitch_queue where entity_ref = ? and stitch_ticket = ?)',\n                [entityRef, stitchTicket],\n              ),\n            );\n          }\n\n          await upsert;\n\n          // Verify the write took effect. On PostgreSQL, also recheck ownership\n          // using this statement's fresh Read Committed snapshot: the upsert's\n          // snapshot may predate a reclaim that committed while it was running.\n          // A successful upsert holds the final-row lock until commit, so a\n          // successor cannot publish past us after this check. Do not lock the\n          // queue, which would introduce a publication/deletion lock inversion.\n          if (!isMySQL) {\n            const written = await tx<DbFinalEntitiesRow>('final_entities')\n              .where('entity_id', entityId)\n              .where('hash', hash)\n              .modify(qb => {\n                if (isPostgres) {\n                  qb.whereExists(\n                    tx<DbStitchQueueRow>('stitch_queue')\n                      .select(tx.raw('1'))\n                      .where('entity_ref', entityRef)\n                      .where('stitch_ticket', stitchTicket)\n                      .where('next_stitch_at', stitchLeaseExpiresAt),\n                  );\n                }\n              })\n              .select(tx.raw('1'))\n              .first();\n            if (!written) {\n              // Returning normally would commit any stale write already made.\n              throw new StitchPublicationSupersededError();\n            }\n          }\n\n          await syncSearchRows(tx, entityId, searchEntries);\n\n          return 'changed' as const;\n        }),\n      knex,\n    ).catch(error => {\n      // Knex has rolled the publication back before rejecting the transaction.\n      // Supersession is a normal abandonment, not a failure or deadlock retry.\n      if (error instanceof StitchPublicationSupersededError) {\n        return 'abandoned' as const;\n      }\n      throw error;\n    });\n\n    if (writeOutcome === 'abandoned') {\n      logger.debug(`Entity ${entityRef} is already stitched, skipping write.`);\n      stitchResult = 'abandoned';\n      return 'abandoned';\n    }\n\n    stitchResult = 'succeeded';\n    return 'changed';\n  } catch (error) {\n    try {\n      await markDeferredStitchFailed({\n        knex,\n        entityRef,\n        stitchTicket,\n        stitchLeaseExpiresAt,\n      });\n    } catch (recordError) {\n      // A database outage may also prevent recording the failure. Keep the\n      // queued work and report the original stitching error to the caller.\n      logger.warn(\n        `Failed to record stitch failure for ${entityRef}`,\n        recordError,\n      );\n    }\n    throw error;\n  } finally {\n    if (stitchResult) {\n      await markDeferredStitchCompleted({\n        knex: knex,\n        entityRef,\n        stitchTicket,\n        stitchLeaseExpiresAt,\n      });\n    }\n  }\n}\n"],"names":["createHash","stableStringify","ENTITY_STATUS_CATALOG_PROCESSING_TYPE","ANNOTATION_VIEW_URL","ANNOTATION_EDIT_URL","buildEntitySearch","retryOnDeadlock","syncSearchRows","markDeferredStitchFailed","markDeferredStitchCompleted"],"mappings":";;;;;;;;;;;;;;;;AAqCA,MAAM,yCAAyC,KAAA,CAAM;AAAC;AAEtD,SAAS,mBAAmB,MAAA,EAAgB;AAC1C,EAAA,OAAOA,sBAAA,CAAW,MAAM,CAAA,CACrB,MAAA,CAAOC,gCAAA,CAAgB,EAAE,GAAG,MAAA,EAAQ,CAAC,CAAA,CACrC,MAAA,CAAO,KAAK,CAAA;AACjB;AAGA,MAAM,qBAAA;AAAA;AAAA,EAEJ;AAAA,CAAA;AAOF,eAAsB,iBAAiB,OAAA,EAMY;AACjD,EAAA,MAAM,EAAE,IAAA,EAAM,MAAA,EAAQ,SAAA,EAAW,YAAA,EAAc,sBAAqB,GAClE,OAAA;AAKF,EAAA,IAAI,YAAA;AAEJ,EAAA,IAAI;AAKF,IAAA,MAAM,CAAC,eAAA,EAAiB,eAAe,CAAA,GAAI,MAAM,QAAQ,GAAA,CAAI;AAAA,MAC3D,IAAA,CACG,IAAA,CAAK,qBAAA,EAAuB,SAAS,mBAAmB,OAAA,EAAS;AAChE,QAAA,OAAO,OAAA,CACJ,IAAA,CAAK,0BAA0B,CAAA,CAC/B,MAAM,EAAE,iBAAA,EAAmB,SAAA,EAAW,CAAA,CACtC,KAAA,CAAM,EAAE,KAAA,EAAO,KAAK,CAAA;AAAA,MACzB,CAAC,EACA,MAAA,CAAO;AAAA,QACN,QAAA,EAAU,yBAAA;AAAA,QACV,eAAA,EAAiB,gCAAA;AAAA,QACjB,MAAA,EAAQ,sBAAA;AAAA,QACR,sBAAA,EAAwB,2BAAA;AAAA,QACxB,YAAA,EAAc;AAAA,OACf,CAAA,CACA,IAAA,CAAK,eAAe,CAAA,CACpB,KAAA,CAAM,EAAE,0BAAA,EAA4B,SAAA,EAAW,CAAA,CAC/C,UAAU,IAAA,CAAK,GAAA,CAAI,qBAAqB,CAAC,CAAA,CACzC,cAAc,gBAAA,EAAkB;AAAA,QAC/B,0BAAA,EAA4B;AAAA,OAC7B,CAAA;AAAA,MACH,KACG,QAAA,CAAS;AAAA,QACR,YAAA,EAAc,MAAA;AAAA,QACd,cAAA,EAAgB;AAAA,OACjB,CAAA,CACA,IAAA,CAAK,WAAW,CAAA,CAChB,MAAM,EAAE,iBAAA,EAAmB,SAAA,EAAW,EACtC,OAAA,CAAQ,cAAA,EAAgB,KAAK,CAAA,CAC7B,OAAA,CAAQ,kBAAkB,KAAK;AAAA,KACnC,CAAA;AAMD,IAAA,IAAI,CAAC,gBAAgB,MAAA,EAAQ;AAC3B,MAAA,MAAA,CAAO,KAAA;AAAA,QACL,oBAAoB,SAAS,CAAA,4CAAA;AAAA,OAC/B;AACA,MAAA,YAAA,GAAe,WAAA;AACf,MAAA,OAAO,WAAA;AAAA,IACT;AAEA,IAAA,MAAM;AAAA,MACJ,QAAA;AAAA,MACA,eAAA;AAAA,MACA,MAAA;AAAA,MACA,sBAAA;AAAA,MACA;AAAA,KACF,GAAI,gBAAgB,CAAC,CAAA;AAMrB,IAAA,IAAI,CAAC,eAAA,EAAiB;AACpB,MAAA,MAAA,CAAO,KAAA;AAAA,QACL,oBAAoB,SAAS,CAAA,uCAAA;AAAA,OAC/B;AACA,MAAA,YAAA,GAAe,WAAA;AACf,MAAA,OAAO,WAAA;AAAA,IACT;AAIA,IAAA,MAAM,MAAA,GAAS,IAAA,CAAK,KAAA,CAAM,eAAe,CAAA;AACzC,IAAA,MAAM,QAAA,GAAW,MAAA,CAAO,sBAAsB,CAAA,KAAM,CAAA;AACpD,IAAA,IAAI,cAAkC,EAAC;AAEvC,IAAA,IAAI,QAAA,EAAU;AACZ,MAAA,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,SAAS,CAAA,aAAA,CAAe,CAAA;AACxC,MAAA,MAAA,CAAO,SAAS,WAAA,GAAc;AAAA,QAC5B,GAAG,OAAO,QAAA,CAAS,WAAA;AAAA,QACnB,CAAC,qBAAqB,GAAG;AAAA,OAC3B;AAAA,IACF;AACA,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,MAAM,YAAA,GAAe,IAAA,CAAK,KAAA,CAAM,MAAM,CAAA;AACtC,MAAA,IAAI,KAAA,CAAM,OAAA,CAAQ,YAAY,CAAA,IAAK,aAAa,MAAA,EAAQ;AACtD,QAAA,WAAA,GAAc,YAAA,CAAa,IAAI,CAAA,CAAA,MAAM;AAAA,UACnC,IAAA,EAAMC,mDAAA;AAAA,UACN,KAAA,EAAO,OAAA;AAAA,UACP,SAAS,CAAA,EAAG,CAAA,CAAE,IAAI,CAAA,EAAA,EAAK,EAAE,OAAO,CAAA,CAAA;AAAA,UAChC,KAAA,EAAO;AAAA,SACT,CAAE,CAAA;AAAA,MACJ;AAAA,IACF;AAEA,IAAA,KAAA,MAAW,UAAA,IAAc,CAACC,gCAAA,EAAqBC,gCAAmB,CAAA,EAAG;AACnE,MAAA,MAAM,KAAA,GAAQ,MAAA,CAAO,QAAA,CAAS,WAAA,GAAc,UAAU,CAAA;AACtD,MAAA,IAAI,OAAO,KAAA,KAAU,QAAA,IAAY,qBAAA,CAAsB,IAAA,CAAK,KAAK,CAAA,EAAG;AAClE,QAAA,MAAA,CAAO,QAAA,CAAS,WAAA,CAAa,UAAU,CAAA,GACrC,+DAAA;AAAA,MACJ;AAAA,IACF;AAIA,IAAA,MAAA,CAAO,YAAY,eAAA,CAChB,MAAA;AAAA,MAAO,SAAO,GAAA,CAAI;AAAA;AAAA,KAAgD,CAClE,IAAoB,CAAA,GAAA,MAAQ;AAAA,MAC3B,MAAM,GAAA,CAAI,YAAA;AAAA,MACV,WAAW,GAAA,CAAI;AAAA,KACjB,CAAE,CAAA;AACJ,IAAA,IAAI,YAAY,MAAA,EAAQ;AACtB,MAAA,MAAA,CAAO,MAAA,GAAS;AAAA,QACd,GAAG,MAAA,CAAO,MAAA;AAAA,QACV,KAAA,EAAO,CAAC,GAAI,MAAA,CAAO,QAAQ,KAAA,IAAS,EAAC,EAAI,GAAG,WAAW;AAAA,OACzD;AAAA,IACF;AAGA,IAAA,MAAM,IAAA,GAAO,mBAAmB,MAAM,CAAA;AACtC,IAAA,IAAI,SAAS,YAAA,EAAc;AACzB,MAAA,MAAA,CAAO,KAAA,CAAM,CAAA,qBAAA,EAAwB,SAAS,CAAA,YAAA,CAAc,CAAA;AAC5D,MAAA,YAAA,GAAe,WAAA;AACf,MAAA,OAAO,WAAA;AAAA,IACT;AAEA,IAAA,MAAA,CAAO,SAAS,GAAA,GAAM,QAAA;AACtB,IAAA,IAAI,CAAC,MAAA,CAAO,QAAA,CAAS,IAAA,EAAM;AAGzB,MAAA,MAAA,CAAO,SAAS,IAAA,GAAO,IAAA;AAAA,IACzB;AAKA,IAAA,MAAM,aAAA,GAAgBC,mCAAA,CAAkB,QAAA,EAAU,MAAM,CAAA;AAWxD,IAAA,MAAM,OAAA,GAAU,OAAO,IAAA,CAAK,MAAA,CAAO,OAAO,MAAM,CAAA,CAAE,SAAS,OAAO,CAAA;AAClE,IAAA,MAAM,UAAA,GAAa,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,MAAA,KAAW,IAAA;AAQjD,IAAA,MAAM,eAAe,MAAMC,oBAAA;AAAA,MACzB,MACE,IAAA,CAAK,WAAA,CAAY,OAAM,EAAA,KAAM;AAI3B,QAAA,IAAI,CAAC,UAAA,EAAY;AACf,UAAA,MAAM,WAAA,GAAc,MAAM,EAAA,CAAqB,cAAc,CAAA,CAC1D,KAAA,CAAM,YAAA,EAAc,SAAS,CAAA,CAC7B,KAAA,CAAM,eAAA,EAAiB,YAAY,EACnC,KAAA,EAAM;AACT,UAAA,IAAI,CAAC,aAAa,OAAO,WAAA;AAAA,QAC3B;AAEA,QAAA,IAAI,MAAA,GAAS,EAAA,CAAG,gBAAgB,CAAA,CAC7B,MAAA;AAAA,UACC,CAAC,UAAA,GACG;AAAA,YACE,SAAA,EAAW,QAAA;AAAA,YACX,UAAA,EAAY,SAAA;AAAA,YACZ,YAAA,EAAc,IAAA,CAAK,SAAA,CAAU,MAAM,CAAA;AAAA,YACnC,IAAA;AAAA,YACA,eAAA,EAAiB,EAAA,CAAG,EAAA,CAAG,GAAA;AAAI,cAE7B,EAAA,CAAG,GAAA;AAAA,YACD,wLAAA;AAAA,YAGA;AAAA,cACE,QAAA;AAAA,cACA,SAAA;AAAA,cACA,IAAA,CAAK,UAAU,MAAM,CAAA;AAAA,cACrB,IAAA;AAAA,cACA,SAAA;AAAA,cACA;AAAA;AACF;AACF,SACN,CACC,WAAW,WAAW,CAAA,CACtB,MAAM,CAAC,cAAA,EAAgB,MAAA,EAAQ,iBAAiB,CAAC,CAAA;AAEpD,QAAA,IAAI,CAAC,OAAA,EAAS;AACZ,UAAA,MAAA,GAAS,MAAA,CAAO,KAAA;AAAA,YACd,EAAA,CAAG,GAAA;AAAA,cACD,gFAAA;AAAA,cACA,CAAC,WAAW,YAAY;AAAA;AAC1B,WACF;AAAA,QACF;AAEA,QAAA,MAAM,MAAA;AAQN,QAAA,IAAI,CAAC,OAAA,EAAS;AACZ,UAAA,MAAM,OAAA,GAAU,MAAM,EAAA,CAAuB,gBAAgB,EAC1D,KAAA,CAAM,WAAA,EAAa,QAAQ,CAAA,CAC3B,KAAA,CAAM,MAAA,EAAQ,IAAI,CAAA,CAClB,OAAO,CAAA,EAAA,KAAM;AACZ,YAAA,IAAI,UAAA,EAAY;AACd,cAAA,EAAA,CAAG,WAAA;AAAA,gBACD,GAAqB,cAAc,CAAA,CAChC,OAAO,EAAA,CAAG,GAAA,CAAI,GAAG,CAAC,CAAA,CAClB,MAAM,YAAA,EAAc,SAAS,EAC7B,KAAA,CAAM,eAAA,EAAiB,YAAY,CAAA,CACnC,KAAA,CAAM,kBAAkB,oBAAoB;AAAA,eACjD;AAAA,YACF;AAAA,UACF,CAAC,EACA,MAAA,CAAO,EAAA,CAAG,IAAI,GAAG,CAAC,EAClB,KAAA,EAAM;AACT,UAAA,IAAI,CAAC,OAAA,EAAS;AAEZ,YAAA,MAAM,IAAI,gCAAA,EAAiC;AAAA,UAC7C;AAAA,QACF;AAEA,QAAA,MAAMC,6BAAA,CAAe,EAAA,EAAI,QAAA,EAAU,aAAa,CAAA;AAEhD,QAAA,OAAO,SAAA;AAAA,MACT,CAAC,CAAA;AAAA,MACH;AAAA,KACF,CAAE,MAAM,CAAA,KAAA,KAAS;AAGf,MAAA,IAAI,iBAAiB,gCAAA,EAAkC;AACrD,QAAA,OAAO,WAAA;AAAA,MACT;AACA,MAAA,MAAM,KAAA;AAAA,IACR,CAAC,CAAA;AAED,IAAA,IAAI,iBAAiB,WAAA,EAAa;AAChC,MAAA,MAAA,CAAO,KAAA,CAAM,CAAA,OAAA,EAAU,SAAS,CAAA,qCAAA,CAAuC,CAAA;AACvE,MAAA,YAAA,GAAe,WAAA;AACf,MAAA,OAAO,WAAA;AAAA,IACT;AAEA,IAAA,YAAA,GAAe,WAAA;AACf,IAAA,OAAO,SAAA;AAAA,EACT,SAAS,KAAA,EAAO;AACd,IAAA,IAAI;AACF,MAAA,MAAMC,iDAAA,CAAyB;AAAA,QAC7B,IAAA;AAAA,QACA,SAAA;AAAA,QACA,YAAA;AAAA,QACA;AAAA,OACD,CAAA;AAAA,IACH,SAAS,WAAA,EAAa;AAGpB,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,uCAAuC,SAAS,CAAA,CAAA;AAAA,QAChD;AAAA,OACF;AAAA,IACF;AACA,IAAA,MAAM,KAAA;AAAA,EACR,CAAA,SAAE;AACA,IAAA,IAAI,YAAA,EAAc;AAChB,MAAA,MAAMC,uDAAA,CAA4B;AAAA,QAChC,IAAA;AAAA,QACA,SAAA;AAAA,QACA,YAAA;AAAA,QACA;AAAA,OACD,CAAA;AAAA,IACH;AAAA,EACF;AACF;;"}

package/dist/database/operations/util/deleteOrphanedEntities.cjs.js

@@ -1,6 +1,7 @@
 'use strict';
 
 var uniq = require('lodash/uniq');
+var util = require('../../util.cjs.js');
 var markForStitching = require('../stitcher/markForStitching.cjs.js');
 
 function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e : { default: e }; }
@@ -9,41 +10,67 @@ var uniq__default = /*#__PURE__*/_interopDefaultCompat(uniq);
 
 async function deleteOrphanedEntities(options) {
   const { knex } = options;
-  let total = 0;
-  for (let i = 0; i < 100; ++i) {
-    const candidates = await knex.with(
-      "orphans",
-      ["entity_id", "entity_ref"],
-      (orphans) => orphans.from("refresh_state").select("refresh_state.entity_id", "refresh_state.entity_ref").leftOuterJoin(
-        "refresh_state_references",
-        "refresh_state_references.target_entity_ref",
-        "refresh_state.entity_ref"
-      ).whereNull("refresh_state_references.target_entity_ref")
-    ).select({
-      entityId: "orphans.entity_id",
-      relationSourceId: "refresh_state.entity_id"
-    }).from("orphans").leftOuterJoin(
+  const runIteration = async (tx) => {
+    const findOrphanRefs = (orphanRefs) => orphanRefs.from("refresh_state").select("refresh_state.entity_ref").leftOuterJoin(
+      "refresh_state_references",
+      "refresh_state_references.target_entity_ref",
+      "refresh_state.entity_ref"
+    ).whereNull("refresh_state_references.target_entity_ref");
+    const candidateQuery = tx.client.config.client.includes("pg") ? tx.withMaterialized("orphan_refs", ["entity_ref"], findOrphanRefs) : tx.with("orphan_refs", ["entity_ref"], findOrphanRefs);
+    const candidates = await candidateQuery.select({
+      entityId: "orphan.entity_id",
+      relationSourceId: "relation_source.entity_id"
+    }).from("orphan_refs").join(
+      "refresh_state as orphan",
+      "orphan.entity_ref",
+      "orphan_refs.entity_ref"
+    ).leftOuterJoin(
       "relations",
       "relations.target_entity_ref",
-      "orphans.entity_ref"
+      "orphan_refs.entity_ref"
     ).leftOuterJoin(
-      "refresh_state",
-      "refresh_state.entity_ref",
+      "refresh_state as relation_source",
+      "relation_source.entity_ref",
       "relations.source_entity_ref"
     );
     if (!candidates.length) {
-      break;
+      return { deleted: 0, done: true };
     }
     const orphanIds = uniq__default.default(candidates.map((r) => r.entityId));
+    await tx("refresh_state").select("entity_id").whereIn("entity_id", orphanIds).forUpdate();
+    const deleted = await tx.table("refresh_state").delete().whereIn("entity_id", orphanIds).whereNotExists(
+      (references) => references.select(tx.raw("1")).from("refresh_state_references").whereRaw("?? = ??", [
+        "refresh_state_references.target_entity_ref",
+        "refresh_state.entity_ref"
+      ])
+    );
+    if (deleted === 0) {
+      return { deleted, done: true };
+    }
+    let deletedIds;
+    if (deleted === orphanIds.length) {
+      deletedIds = new Set(orphanIds);
+    } else {
+      const remaining = await tx("refresh_state").select("entity_id").whereIn("entity_id", orphanIds);
+      const remainingIds = new Set(remaining.map((row) => row.entity_id));
+      deletedIds = new Set(orphanIds.filter((id) => !remainingIds.has(id)));
+    }
     const orphanRelationIds = uniq__default.default(
-      candidates.map((r) => r.relationSourceId).filter(Boolean)
+      candidates.filter((candidate) => deletedIds.has(candidate.entityId)).map((candidate) => candidate.relationSourceId).filter(Boolean)
     );
-    total += orphanIds.length;
-    await knex.table("refresh_state").delete().whereIn("entity_id", orphanIds);
     await markForStitching.markForStitching({
-      knex,
+      knex: tx,
       entityIds: orphanRelationIds
     });
+    return { deleted, done: false };
+  };
+  let total = 0;
+  for (let i = 0; i < 100; ++i) {
+    const result = knex.isTransaction ? await runIteration(knex) : await util.retryOnDeadlock(() => knex.transaction(runIteration), knex);
+    total += result.deleted;
+    if (result.done) {
+      break;
+    }
   }
   return total;
 }

package/dist/database/operations/util/deleteOrphanedEntities.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"deleteOrphanedEntities.cjs.js","sources":["../../../../src/database/operations/util/deleteOrphanedEntities.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport uniq from 'lodash/uniq';\nimport { DbRefreshStateRow } from '../../tables';\nimport { markForStitching } from '../stitcher/markForStitching';\n\n/**\n * Finds and deletes all orphaned entities, i.e. entities that do not have any\n * incoming references to them, and also eagerly deletes all of their children\n * that would otherwise become orphaned.\n */\nexport async function deleteOrphanedEntities(options: {\n  knex: Knex.Transaction | Knex;\n}): Promise<number> {\n  const { knex } = options;\n\n  let total = 0;\n\n  // Limit iterations for sanity\n  for (let i = 0; i < 100; ++i) {\n    const candidates = await knex\n      .with('orphans', ['entity_id', 'entity_ref'], orphans =>\n        orphans\n          .from('refresh_state')\n          .select('refresh_state.entity_id', 'refresh_state.entity_ref')\n          .leftOuterJoin(\n            'refresh_state_references',\n            'refresh_state_references.target_entity_ref',\n            'refresh_state.entity_ref',\n          )\n          .whereNull('refresh_state_references.target_entity_ref'),\n      )\n      .select({\n        entityId: 'orphans.entity_id',\n        relationSourceId: 'refresh_state.entity_id',\n      })\n      .from('orphans')\n      .leftOuterJoin(\n        'relations',\n        'relations.target_entity_ref',\n        'orphans.entity_ref',\n      )\n      .leftOuterJoin(\n        'refresh_state',\n        'refresh_state.entity_ref',\n        'relations.source_entity_ref',\n      );\n\n    if (!candidates.length) {\n      break;\n    }\n\n    const orphanIds: string[] = uniq(candidates.map(r => r.entityId));\n    const orphanRelationIds: string[] = uniq(\n      candidates.map(r => r.relationSourceId).filter(Boolean),\n    );\n\n    total += orphanIds.length;\n\n    // Delete the orphans themselves\n    await knex\n      .table<DbRefreshStateRow>('refresh_state')\n      .delete()\n      .whereIn('entity_id', orphanIds);\n\n    // Mark all of the things that the orphans had relations to for stitching\n    await markForStitching({\n      knex,\n      entityIds: orphanRelationIds,\n    });\n  }\n\n  return total;\n}\n"],"names":["uniq","markForStitching"],"mappings":";;;;;;;;;AA0BA,eAAsB,uBAAuB,OAAA,EAEzB;AAClB,EAAA,MAAM,EAAE,MAAK,GAAI,OAAA;AAEjB,EAAA,IAAI,KAAA,GAAQ,CAAA;AAGZ,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,GAAA,EAAK,EAAE,CAAA,EAAG;AAC5B,IAAA,MAAM,UAAA,GAAa,MAAM,IAAA,CACtB,IAAA;AAAA,MAAK,SAAA;AAAA,MAAW,CAAC,aAAa,YAAY,CAAA;AAAA,MAAG,CAAA,OAAA,KAC5C,QACG,IAAA,CAAK,eAAe,EACpB,MAAA,CAAO,yBAAA,EAA2B,0BAA0B,CAAA,CAC5D,aAAA;AAAA,QACC,0BAAA;AAAA,QACA,4CAAA;AAAA,QACA;AAAA,OACF,CACC,UAAU,4CAA4C;AAAA,MAE1D,MAAA,CAAO;AAAA,MACN,QAAA,EAAU,mBAAA;AAAA,MACV,gBAAA,EAAkB;AAAA,KACnB,CAAA,CACA,IAAA,CAAK,SAAS,CAAA,CACd,aAAA;AAAA,MACC,WAAA;AAAA,MACA,6BAAA;AAAA,MACA;AAAA,KACF,CACC,aAAA;AAAA,MACC,eAAA;AAAA,MACA,0BAAA;AAAA,MACA;AAAA,KACF;AAEF,IAAA,IAAI,CAAC,WAAW,MAAA,EAAQ;AACtB,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,YAAsBA,qBAAA,CAAK,UAAA,CAAW,IAAI,CAAA,CAAA,KAAK,CAAA,CAAE,QAAQ,CAAC,CAAA;AAChE,IAAA,MAAM,iBAAA,GAA8BA,qBAAA;AAAA,MAClC,WAAW,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,gBAAgB,CAAA,CAAE,OAAO,OAAO;AAAA,KACxD;AAEA,IAAA,KAAA,IAAS,SAAA,CAAU,MAAA;AAGnB,IAAA,MAAM,IAAA,CACH,MAAyB,eAAe,CAAA,CACxC,QAAO,CACP,OAAA,CAAQ,aAAa,SAAS,CAAA;AAGjC,IAAA,MAAMC,iCAAA,CAAiB;AAAA,MACrB,IAAA;AAAA,MACA,SAAA,EAAW;AAAA,KACZ,CAAA;AAAA,EACH;AAEA,EAAA,OAAO,KAAA;AACT;;"}
+{"version":3,"file":"deleteOrphanedEntities.cjs.js","sources":["../../../../src/database/operations/util/deleteOrphanedEntities.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport uniq from 'lodash/uniq';\nimport { DbRefreshStateRow } from '../../tables';\nimport { retryOnDeadlock } from '../../util';\nimport { markForStitching } from '../stitcher/markForStitching';\n\n/**\n * Finds and deletes all orphaned entities, i.e. entities that do not have any\n * incoming references to them, and also eagerly deletes all of their children\n * that would otherwise become orphaned.\n */\nexport async function deleteOrphanedEntities(options: {\n  knex: Knex.Transaction | Knex;\n}): Promise<number> {\n  const { knex } = options;\n\n  const runIteration = async (tx: Knex.Transaction | Knex) => {\n    // Keep orphan discovery and relation lookup in one statement so that they\n    // observe the same database snapshot.\n    const findOrphanRefs = (orphanRefs: Knex.QueryBuilder) =>\n      orphanRefs\n        .from('refresh_state')\n        .select('refresh_state.entity_ref')\n        .leftOuterJoin(\n          'refresh_state_references',\n          'refresh_state_references.target_entity_ref',\n          'refresh_state.entity_ref',\n        )\n        .whereNull('refresh_state_references.target_entity_ref');\n\n    const candidateQuery = tx.client.config.client.includes('pg')\n      ? tx.withMaterialized('orphan_refs', ['entity_ref'], findOrphanRefs)\n      : tx.with('orphan_refs', ['entity_ref'], findOrphanRefs);\n\n    const candidates = await candidateQuery\n      .select({\n        entityId: 'orphan.entity_id',\n        relationSourceId: 'relation_source.entity_id',\n      })\n      .from('orphan_refs')\n      .join(\n        'refresh_state as orphan',\n        'orphan.entity_ref',\n        'orphan_refs.entity_ref',\n      )\n      .leftOuterJoin(\n        'relations',\n        'relations.target_entity_ref',\n        'orphan_refs.entity_ref',\n      )\n      .leftOuterJoin(\n        'refresh_state as relation_source',\n        'relation_source.entity_ref',\n        'relations.source_entity_ref',\n      );\n\n    if (!candidates.length) {\n      return { deleted: 0, done: true };\n    }\n\n    const orphanIds: string[] = uniq(candidates.map(r => r.entityId));\n\n    // Serialize with concurrent reference inserts, which take a key-share lock\n    // on the target row while checking their foreign key. Rechecking in the\n    // following statement then observes references that committed while this\n    // lock was being acquired.\n    await tx<DbRefreshStateRow>('refresh_state')\n      .select('entity_id')\n      .whereIn('entity_id', orphanIds)\n      .forUpdate();\n\n    // Recheck the orphan status in the deletion statement. An entity may have\n    // gained a reference since the candidate query completed.\n    const deleted = await tx\n      .table<DbRefreshStateRow>('refresh_state')\n      .delete()\n      .whereIn('entity_id', orphanIds)\n      .whereNotExists(references =>\n        references\n          .select(tx.raw('1'))\n          .from('refresh_state_references')\n          .whereRaw('?? = ??', [\n            'refresh_state_references.target_entity_ref',\n            'refresh_state.entity_ref',\n          ]),\n      );\n\n    if (deleted === 0) {\n      return { deleted, done: true };\n    }\n\n    let deletedIds: Set<string>;\n    if (deleted === orphanIds.length) {\n      deletedIds = new Set(orphanIds);\n    } else {\n      const remaining = await tx<DbRefreshStateRow>('refresh_state')\n        .select('entity_id')\n        .whereIn('entity_id', orphanIds);\n      const remainingIds = new Set(remaining.map(row => row.entity_id));\n      deletedIds = new Set(orphanIds.filter(id => !remainingIds.has(id)));\n    }\n\n    const orphanRelationIds: string[] = uniq(\n      candidates\n        .filter(candidate => deletedIds.has(candidate.entityId))\n        .map(candidate => candidate.relationSourceId)\n        .filter(Boolean),\n    );\n\n    // Mark all of the things that the orphans had relations to for stitching\n    await markForStitching({\n      knex: tx,\n      entityIds: orphanRelationIds,\n    });\n\n    return { deleted, done: false };\n  };\n\n  let total = 0;\n\n  // Limit iterations for sanity\n  for (let i = 0; i < 100; ++i) {\n    const result = knex.isTransaction\n      ? await runIteration(knex)\n      : await retryOnDeadlock(() => knex.transaction(runIteration), knex);\n\n    total += result.deleted;\n    if (result.done) {\n      break;\n    }\n  }\n\n  return total;\n}\n"],"names":["uniq","markForStitching","retryOnDeadlock"],"mappings":";;;;;;;;;;AA2BA,eAAsB,uBAAuB,OAAA,EAEzB;AAClB,EAAA,MAAM,EAAE,MAAK,GAAI,OAAA;AAEjB,EAAA,MAAM,YAAA,GAAe,OAAO,EAAA,KAAgC;AAG1D,IAAA,MAAM,cAAA,GAAiB,CAAC,UAAA,KACtB,UAAA,CACG,KAAK,eAAe,CAAA,CACpB,MAAA,CAAO,0BAA0B,CAAA,CACjC,aAAA;AAAA,MACC,0BAAA;AAAA,MACA,4CAAA;AAAA,MACA;AAAA,KACF,CACC,UAAU,4CAA4C,CAAA;AAE3D,IAAA,MAAM,cAAA,GAAiB,GAAG,MAAA,CAAO,MAAA,CAAO,OAAO,QAAA,CAAS,IAAI,CAAA,GACxD,EAAA,CAAG,gBAAA,CAAiB,aAAA,EAAe,CAAC,YAAY,CAAA,EAAG,cAAc,CAAA,GACjE,EAAA,CAAG,KAAK,aAAA,EAAe,CAAC,YAAY,CAAA,EAAG,cAAc,CAAA;AAEzD,IAAA,MAAM,UAAA,GAAa,MAAM,cAAA,CACtB,MAAA,CAAO;AAAA,MACN,QAAA,EAAU,kBAAA;AAAA,MACV,gBAAA,EAAkB;AAAA,KACnB,CAAA,CACA,IAAA,CAAK,aAAa,CAAA,CAClB,IAAA;AAAA,MACC,yBAAA;AAAA,MACA,mBAAA;AAAA,MACA;AAAA,KACF,CACC,aAAA;AAAA,MACC,WAAA;AAAA,MACA,6BAAA;AAAA,MACA;AAAA,KACF,CACC,aAAA;AAAA,MACC,kCAAA;AAAA,MACA,4BAAA;AAAA,MACA;AAAA,KACF;AAEF,IAAA,IAAI,CAAC,WAAW,MAAA,EAAQ;AACtB,MAAA,OAAO,EAAE,OAAA,EAAS,CAAA,EAAG,IAAA,EAAM,IAAA,EAAK;AAAA,IAClC;AAEA,IAAA,MAAM,YAAsBA,qBAAA,CAAK,UAAA,CAAW,IAAI,CAAA,CAAA,KAAK,CAAA,CAAE,QAAQ,CAAC,CAAA;AAMhE,IAAA,MAAM,EAAA,CAAsB,eAAe,CAAA,CACxC,MAAA,CAAO,WAAW,EAClB,OAAA,CAAQ,WAAA,EAAa,SAAS,CAAA,CAC9B,SAAA,EAAU;AAIb,IAAA,MAAM,OAAA,GAAU,MAAM,EAAA,CACnB,KAAA,CAAyB,eAAe,CAAA,CACxC,MAAA,EAAO,CACP,OAAA,CAAQ,WAAA,EAAa,SAAS,CAAA,CAC9B,cAAA;AAAA,MAAe,CAAA,UAAA,KACd,UAAA,CACG,MAAA,CAAO,EAAA,CAAG,GAAA,CAAI,GAAG,CAAC,CAAA,CAClB,IAAA,CAAK,0BAA0B,CAAA,CAC/B,QAAA,CAAS,SAAA,EAAW;AAAA,QACnB,4CAAA;AAAA,QACA;AAAA,OACD;AAAA,KACL;AAEF,IAAA,IAAI,YAAY,CAAA,EAAG;AACjB,MAAA,OAAO,EAAE,OAAA,EAAS,IAAA,EAAM,IAAA,EAAK;AAAA,IAC/B;AAEA,IAAA,IAAI,UAAA;AACJ,IAAA,IAAI,OAAA,KAAY,UAAU,MAAA,EAAQ;AAChC,MAAA,UAAA,GAAa,IAAI,IAAI,SAAS,CAAA;AAAA,IAChC,CAAA,MAAO;AACL,MAAA,MAAM,SAAA,GAAY,MAAM,EAAA,CAAsB,eAAe,CAAA,CAC1D,OAAO,WAAW,CAAA,CAClB,OAAA,CAAQ,WAAA,EAAa,SAAS,CAAA;AACjC,MAAA,MAAM,YAAA,GAAe,IAAI,GAAA,CAAI,SAAA,CAAU,IAAI,CAAA,GAAA,KAAO,GAAA,CAAI,SAAS,CAAC,CAAA;AAChE,MAAA,UAAA,GAAa,IAAI,GAAA,CAAI,SAAA,CAAU,MAAA,CAAO,CAAA,EAAA,KAAM,CAAC,YAAA,CAAa,GAAA,CAAI,EAAE,CAAC,CAAC,CAAA;AAAA,IACpE;AAEA,IAAA,MAAM,iBAAA,GAA8BA,qBAAA;AAAA,MAClC,UAAA,CACG,MAAA,CAAO,CAAA,SAAA,KAAa,UAAA,CAAW,IAAI,SAAA,CAAU,QAAQ,CAAC,CAAA,CACtD,IAAI,CAAA,SAAA,KAAa,SAAA,CAAU,gBAAgB,CAAA,CAC3C,OAAO,OAAO;AAAA,KACnB;AAGA,IAAA,MAAMC,iCAAA,CAAiB;AAAA,MACrB,IAAA,EAAM,EAAA;AAAA,MACN,SAAA,EAAW;AAAA,KACZ,CAAA;AAED,IAAA,OAAO,EAAE,OAAA,EAAS,IAAA,EAAM,KAAA,EAAM;AAAA,EAChC,CAAA;AAEA,EAAA,IAAI,KAAA,GAAQ,CAAA;AAGZ,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,GAAA,EAAK,EAAE,CAAA,EAAG;AAC5B,IAAA,MAAM,MAAA,GAAS,IAAA,CAAK,aAAA,GAChB,MAAM,aAAa,IAAI,CAAA,GACvB,MAAMC,oBAAA,CAAgB,MAAM,IAAA,CAAK,WAAA,CAAY,YAAY,GAAG,IAAI,CAAA;AAEpE,IAAA,KAAA,IAAS,MAAA,CAAO,OAAA;AAChB,IAAA,IAAI,OAAO,IAAA,EAAM;AACf,MAAA;AAAA,IACF;AAAA,EACF;AAEA,EAAA,OAAO,KAAA;AACT;;"}

package/dist/stitching/types.cjs.js

@@ -1,12 +1,25 @@
 'use strict';
 
 var config = require('@backstage/config');
+var types = require('@backstage/types');
+var errors = require('@backstage/errors');
 
 function stitchingStrategyFromConfig(config$1) {
   const pollingIntervalKey = "catalog.stitchingStrategy.pollingInterval";
   const stitchTimeoutKey = "catalog.stitchingStrategy.stitchTimeout";
   const pollingInterval = config$1.has(pollingIntervalKey) ? config.readDurationFromConfig(config$1, { key: pollingIntervalKey }) : { seconds: 1 };
   const stitchTimeout = config$1.has(stitchTimeoutKey) ? config.readDurationFromConfig(config$1, { key: stitchTimeoutKey }) : { seconds: 60 };
+  for (const [key, duration] of [
+    [pollingIntervalKey, pollingInterval],
+    [stitchTimeoutKey, stitchTimeout]
+  ]) {
+    const milliseconds = types.durationToMilliseconds(duration);
+    if (!Number.isFinite(milliseconds) || milliseconds <= 0) {
+      throw new errors.InputError(
+        `Duration in config at '${key}' must be finite and greater than zero`
+      );
+    }
+  }
   return {
     pollingInterval,
     stitchTimeout

package/dist/stitching/types.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.cjs.js","sources":["../../src/stitching/types.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Config, readDurationFromConfig } from '@backstage/config';\nimport { HumanDuration } from '@backstage/types';\n\n/**\n * Configuration for the stitching process, controlling polling and timeout\n * behavior for the deferred stitching worker.\n */\nexport type StitchingStrategy = {\n  pollingInterval: HumanDuration;\n  stitchTimeout: HumanDuration;\n};\n\nexport function stitchingStrategyFromConfig(config: Config): StitchingStrategy {\n  const pollingIntervalKey = 'catalog.stitchingStrategy.pollingInterval';\n  const stitchTimeoutKey = 'catalog.stitchingStrategy.stitchTimeout';\n\n  const pollingInterval = config.has(pollingIntervalKey)\n    ? readDurationFromConfig(config, { key: pollingIntervalKey })\n    : { seconds: 1 };\n  const stitchTimeout = config.has(stitchTimeoutKey)\n    ? readDurationFromConfig(config, { key: stitchTimeoutKey })\n    : { seconds: 60 };\n\n  return {\n    pollingInterval: pollingInterval,\n    stitchTimeout: stitchTimeout,\n  };\n}\n"],"names":["config","readDurationFromConfig"],"mappings":";;;;AA4BO,SAAS,4BAA4BA,QAAA,EAAmC;AAC7E,EAAA,MAAM,kBAAA,GAAqB,2CAAA;AAC3B,EAAA,MAAM,gBAAA,GAAmB,yCAAA;AAEzB,EAAA,MAAM,eAAA,GAAkBA,QAAA,CAAO,GAAA,CAAI,kBAAkB,IACjDC,6BAAA,CAAuBD,QAAA,EAAQ,EAAE,GAAA,EAAK,kBAAA,EAAoB,CAAA,GAC1D,EAAE,SAAS,CAAA,EAAE;AACjB,EAAA,MAAM,aAAA,GAAgBA,QAAA,CAAO,GAAA,CAAI,gBAAgB,IAC7CC,6BAAA,CAAuBD,QAAA,EAAQ,EAAE,GAAA,EAAK,gBAAA,EAAkB,CAAA,GACxD,EAAE,SAAS,EAAA,EAAG;AAElB,EAAA,OAAO;AAAA,IACL,eAAA;AAAA,IACA;AAAA,GACF;AACF;;"}
+{"version":3,"file":"types.cjs.js","sources":["../../src/stitching/types.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Config, readDurationFromConfig } from '@backstage/config';\nimport { durationToMilliseconds, HumanDuration } from '@backstage/types';\nimport { InputError } from '@backstage/errors';\n\n/**\n * Configuration for the stitching process, controlling polling and timeout\n * behavior for the deferred stitching worker.\n */\nexport type StitchingStrategy = {\n  pollingInterval: HumanDuration;\n  stitchTimeout: HumanDuration;\n};\n\nexport function stitchingStrategyFromConfig(config: Config): StitchingStrategy {\n  const pollingIntervalKey = 'catalog.stitchingStrategy.pollingInterval';\n  const stitchTimeoutKey = 'catalog.stitchingStrategy.stitchTimeout';\n\n  const pollingInterval = config.has(pollingIntervalKey)\n    ? readDurationFromConfig(config, { key: pollingIntervalKey })\n    : { seconds: 1 };\n  const stitchTimeout = config.has(stitchTimeoutKey)\n    ? readDurationFromConfig(config, { key: stitchTimeoutKey })\n    : { seconds: 60 };\n\n  for (const [key, duration] of [\n    [pollingIntervalKey, pollingInterval],\n    [stitchTimeoutKey, stitchTimeout],\n  ] as const) {\n    const milliseconds = durationToMilliseconds(duration);\n    if (!Number.isFinite(milliseconds) || milliseconds <= 0) {\n      throw new InputError(\n        `Duration in config at '${key}' must be finite and greater than zero`,\n      );\n    }\n  }\n\n  return {\n    pollingInterval: pollingInterval,\n    stitchTimeout: stitchTimeout,\n  };\n}\n"],"names":["config","readDurationFromConfig","durationToMilliseconds","InputError"],"mappings":";;;;;;AA6BO,SAAS,4BAA4BA,QAAA,EAAmC;AAC7E,EAAA,MAAM,kBAAA,GAAqB,2CAAA;AAC3B,EAAA,MAAM,gBAAA,GAAmB,yCAAA;AAEzB,EAAA,MAAM,eAAA,GAAkBA,QAAA,CAAO,GAAA,CAAI,kBAAkB,IACjDC,6BAAA,CAAuBD,QAAA,EAAQ,EAAE,GAAA,EAAK,kBAAA,EAAoB,CAAA,GAC1D,EAAE,SAAS,CAAA,EAAE;AACjB,EAAA,MAAM,aAAA,GAAgBA,QAAA,CAAO,GAAA,CAAI,gBAAgB,IAC7CC,6BAAA,CAAuBD,QAAA,EAAQ,EAAE,GAAA,EAAK,gBAAA,EAAkB,CAAA,GACxD,EAAE,SAAS,EAAA,EAAG;AAElB,EAAA,KAAA,MAAW,CAAC,GAAA,EAAK,QAAQ,CAAA,IAAK;AAAA,IAC5B,CAAC,oBAAoB,eAAe,CAAA;AAAA,IACpC,CAAC,kBAAkB,aAAa;AAAA,GAClC,EAAY;AACV,IAAA,MAAM,YAAA,GAAeE,6BAAuB,QAAQ,CAAA;AACpD,IAAA,IAAI,CAAC,MAAA,CAAO,QAAA,CAAS,YAAY,CAAA,IAAK,gBAAgB,CAAA,EAAG;AACvD,MAAA,MAAM,IAAIC,iBAAA;AAAA,QACR,0BAA0B,GAAG,CAAA,sCAAA;AAAA,OAC/B;AAAA,IACF;AAAA,EACF;AAEA,EAAA,OAAO;AAAA,IACL,eAAA;AAAA,IACA;AAAA,GACF;AACF;;"}

package/migrations/20261007000000_stitch_failure_count.js

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2026 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+/** @param {import('knex').Knex} knex */
+exports.up = async function up(knex) {
+  await knex.schema.alterTable('stitch_queue', table => {
+    table.integer('failure_count').notNullable().defaultTo(0);
+  });
+};
+
+/** @param {import('knex').Knex} knex */
+exports.down = async function down(knex) {
+  await knex.schema.alterTable('stitch_queue', table => {
+    table.dropColumn('failure_count');
+  });
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend",
-  "version": "4.0.1-next.2",
+  "version": "4.0.1-next.3",
   "description": "The Backstage backend plugin that provides the Backstage catalog",
   "backstage": {
     "role": "backend-plugin",
@@ -76,8 +76,8 @@
   },
   "dependencies": {
     "@backstage/backend-openapi-utils": "0.7.3-next.2",
-    "@backstage/backend-plugin-api": "1.11.0-next.1",
-    "@backstage/catalog-client": "1.16.3-next.0",
+    "@backstage/backend-plugin-api": "1.11.0-next.2",
+    "@backstage/catalog-client": "1.16.3-next.1",
     "@backstage/catalog-model": "1.10.2-next.0",
     "@backstage/config": "1.3.10-next.0",
     "@backstage/errors": "1.3.2-next.0",
@@ -110,12 +110,12 @@
     "zod-validation-error": "^5.0.0"
   },
   "devDependencies": {
-    "@backstage/backend-defaults": "0.18.1-next.2",
-    "@backstage/backend-test-utils": "1.11.8-next.1",
+    "@backstage/backend-defaults": "0.18.1-next.3",
+    "@backstage/backend-test-utils": "1.11.8-next.2",
     "@backstage/cli": "0.36.7-next.0",
     "@backstage/plugin-catalog-backend-module-logs": "0.1.27-next.1",
     "@backstage/plugin-scaffolder-common": "2.3.1-next.0",
-    "@backstage/repo-tools": "0.19.2-next.2",
+    "@backstage/repo-tools": "0.19.2-next.3",
     "@types/core-js": "^2.5.4",
     "@types/express": "^4.17.6",
     "@types/git-url-parse": "^9.0.0",