@backstage/plugin-catalog-backend 3.8.2-next.1 → 3.9.0-next.2

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @backstage/plugin-catalog-backend
 
+## 3.9.0-next.2
+
+### Minor Changes
+
+- c7c0ec3: Added a `refresh-catalog-entity` action so agents and MCP clients can re-queue a single entity for processing after creating or updating it — useful for reading back fresh data immediately after a scaffolder run without waiting for the next scheduled processing loop.
+
+### Patch Changes
+
+- aa318d0: Migrated internal query filter handling from `EntityFilter` to `FilterPredicate`, simplifying the filter parsing and query application pipeline.
+- 10f0713: Replaced the delete-all and reinsert pattern for the `relations` table with a diff-based sync that only touches rows that actually changed. In steady state (the common case), zero writes occur, eliminating write churn, dead tuples, and WAL traffic from the processing path. Stitching is now also skipped for relation neighbors that did not change.
+- eb6dff2: Fixed an issue where PostgreSQL deadlock errors during entity provider mutations were silently swallowed, causing entities to be dropped until the next full refresh. Transactions are now automatically retried on deadlock with exponential back-off.
+- dd562f0: Fixed a potential MySQL deadlock during concurrent entity processing by retrying the `updateProcessedEntity` transaction on deadlock errors.
+- b031a48: Fixed an issue where SCM `location.moved` events would generate new locations in the database for files that were not actively tracked.
+- Updated dependencies
+  - @backstage/backend-plugin-api@1.10.0-next.1
+
 ## 3.8.2-next.1
 
 ### Patch Changes

package/dist/actions/createRefreshCatalogEntityAction.cjs.js

@@ -0,0 +1,67 @@
+'use strict';
+
+var catalogModel = require('@backstage/catalog-model');
+var errors = require('@backstage/errors');
+
+const createRefreshCatalogEntityAction = ({
+  catalog,
+  actionsRegistry
+}) => {
+  actionsRegistry.register({
+    name: "refresh-catalog-entity",
+    title: "Refresh Catalog Entity",
+    attributes: {
+      destructive: false,
+      readOnly: false,
+      idempotent: true
+    },
+    description: `
+This allows you to trigger a refresh of a single entity in the software catalog, requeuing it for processing.
+This is useful immediately after creating or updating an entity (for example via a scaffolder template invoked by an MCP client) so the new data becomes visible before subsequent actions read it.
+Each entity in the software catalog has a unique name, kind, and namespace. If the name alone matches multiple entities, provide the kind (and namespace) to disambiguate.
+    `,
+    schema: {
+      input: (z) => z.object({
+        kind: z.string().min(1).describe(
+          'The kind of the entity to refresh, e.g. "Component", "API", "System". If the kind is unknown it can be omitted.'
+        ).optional(),
+        namespace: z.string().min(1).describe(
+          "The namespace of the entity to refresh. If the namespace is unknown it can be omitted."
+        ).optional(),
+        name: z.string().trim().min(1).describe("The name of the entity to refresh.")
+      }),
+      output: (z) => z.object({
+        entityRef: z.string().describe("The canonical entity reference that was refreshed.")
+      })
+    },
+    action: async ({ input, credentials }) => {
+      const filter = { "metadata.name": input.name };
+      if (input.kind) {
+        filter.kind = input.kind;
+      }
+      if (input.namespace) {
+        filter["metadata.namespace"] = input.namespace;
+      }
+      const { items } = await catalog.queryEntities(
+        { filter },
+        { credentials }
+      );
+      if (items.length === 0) {
+        throw new errors.InputError(`No entity found with name "${input.name}"`);
+      }
+      if (items.length > 1) {
+        throw new errors.ConflictError(
+          `Multiple entities found with name "${input.name}", please provide more specific filters. Entities found: ${items.map((item) => `"${catalogModel.stringifyEntityRef(item)}"`).join(", ")}`
+        );
+      }
+      const entityRef = catalogModel.stringifyEntityRef(items[0]);
+      await catalog.refreshEntity(entityRef, { credentials });
+      return {
+        output: { entityRef }
+      };
+    }
+  });
+};
+
+exports.createRefreshCatalogEntityAction = createRefreshCatalogEntityAction;
+//# sourceMappingURL=createRefreshCatalogEntityAction.cjs.js.map

package/dist/actions/createRefreshCatalogEntityAction.cjs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createRefreshCatalogEntityAction.cjs.js","sources":["../../src/actions/createRefreshCatalogEntityAction.ts"],"sourcesContent":["/*\n * Copyright 2026 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nimport type { ActionsRegistryService } from '@backstage/backend-plugin-api/alpha';\nimport { stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError, InputError } from '@backstage/errors';\nimport type { CatalogService } from '@backstage/plugin-catalog-node';\n\nexport const createRefreshCatalogEntityAction = ({\n  catalog,\n  actionsRegistry,\n}: {\n  catalog: CatalogService;\n  actionsRegistry: ActionsRegistryService;\n}) => {\n  actionsRegistry.register({\n    name: 'refresh-catalog-entity',\n    title: 'Refresh Catalog Entity',\n    attributes: {\n      destructive: false,\n      readOnly: false,\n      idempotent: true,\n    },\n    description: `\nThis allows you to trigger a refresh of a single entity in the software catalog, requeuing it for processing.\nThis is useful immediately after creating or updating an entity (for example via a scaffolder template invoked by an MCP client) so the new data becomes visible before subsequent actions read it.\nEach entity in the software catalog has a unique name, kind, and namespace. If the name alone matches multiple entities, provide the kind (and namespace) to disambiguate.\n    `,\n    schema: {\n      input: z =>\n        z.object({\n          kind: z\n            .string()\n            .min(1)\n            .describe(\n              'The kind of the entity to refresh, e.g. \"Component\", \"API\", \"System\". If the kind is unknown it can be omitted.',\n            )\n            .optional(),\n          namespace: z\n            .string()\n            .min(1)\n            .describe(\n              'The namespace of the entity to refresh. If the namespace is unknown it can be omitted.',\n            )\n            .optional(),\n          name: z\n            .string()\n            .trim()\n            .min(1)\n            .describe('The name of the entity to refresh.'),\n        }),\n      output: z =>\n        z.object({\n          entityRef: z\n            .string()\n            .describe('The canonical entity reference that was refreshed.'),\n        }),\n    },\n    action: async ({ input, credentials }) => {\n      const filter: Record<string, string> = { 'metadata.name': input.name };\n\n      if (input.kind) {\n        filter.kind = input.kind;\n      }\n\n      if (input.namespace) {\n        filter['metadata.namespace'] = input.namespace;\n      }\n\n      const { items } = await catalog.queryEntities(\n        { filter },\n        { credentials },\n      );\n\n      if (items.length === 0) {\n        throw new InputError(`No entity found with name \"${input.name}\"`);\n      }\n\n      if (items.length > 1) {\n        throw new ConflictError(\n          `Multiple entities found with name \"${\n            input.name\n          }\", please provide more specific filters. Entities found: ${items\n            .map(item => `\"${stringifyEntityRef(item)}\"`)\n            .join(', ')}`,\n        );\n      }\n\n      const entityRef = stringifyEntityRef(items[0]);\n\n      await catalog.refreshEntity(entityRef, { credentials });\n\n      return {\n        output: { entityRef },\n      };\n    },\n  });\n};\n"],"names":["InputError","ConflictError","stringifyEntityRef"],"mappings":";;;;;AAoBO,MAAM,mCAAmC,CAAC;AAAA,EAC/C,OAAA;AAAA,EACA;AACF,CAAA,KAGM;AACJ,EAAA,eAAA,CAAgB,QAAA,CAAS;AAAA,IACvB,IAAA,EAAM,wBAAA;AAAA,IACN,KAAA,EAAO,wBAAA;AAAA,IACP,UAAA,EAAY;AAAA,MACV,WAAA,EAAa,KAAA;AAAA,MACb,QAAA,EAAU,KAAA;AAAA,MACV,UAAA,EAAY;AAAA,KACd;AAAA,IACA,WAAA,EAAa;AAAA;AAAA;AAAA;AAAA,IAAA,CAAA;AAAA,IAKb,MAAA,EAAQ;AAAA,MACN,KAAA,EAAO,CAAA,CAAA,KACL,CAAA,CAAE,MAAA,CAAO;AAAA,QACP,MAAM,CAAA,CACH,MAAA,EAAO,CACP,GAAA,CAAI,CAAC,CAAA,CACL,QAAA;AAAA,UACC;AAAA,UAED,QAAA,EAAS;AAAA,QACZ,WAAW,CAAA,CACR,MAAA,EAAO,CACP,GAAA,CAAI,CAAC,CAAA,CACL,QAAA;AAAA,UACC;AAAA,UAED,QAAA,EAAS;AAAA,QACZ,IAAA,EAAM,CAAA,CACH,MAAA,EAAO,CACP,IAAA,GACA,GAAA,CAAI,CAAC,CAAA,CACL,QAAA,CAAS,oCAAoC;AAAA,OACjD,CAAA;AAAA,MACH,MAAA,EAAQ,CAAA,CAAA,KACN,CAAA,CAAE,MAAA,CAAO;AAAA,QACP,SAAA,EAAW,CAAA,CACR,MAAA,EAAO,CACP,SAAS,oDAAoD;AAAA,OACjE;AAAA,KACL;AAAA,IACA,MAAA,EAAQ,OAAO,EAAE,KAAA,EAAO,aAAY,KAAM;AACxC,MAAA,MAAM,MAAA,GAAiC,EAAE,eAAA,EAAiB,KAAA,CAAM,IAAA,EAAK;AAErE,MAAA,IAAI,MAAM,IAAA,EAAM;AACd,QAAA,MAAA,CAAO,OAAO,KAAA,CAAM,IAAA;AAAA,MACtB;AAEA,MAAA,IAAI,MAAM,SAAA,EAAW;AACnB,QAAA,MAAA,CAAO,oBAAoB,IAAI,KAAA,CAAM,SAAA;AAAA,MACvC;AAEA,MAAA,MAAM,EAAE,KAAA,EAAM,GAAI,MAAM,OAAA,CAAQ,aAAA;AAAA,QAC9B,EAAE,MAAA,EAAO;AAAA,QACT,EAAE,WAAA;AAAY,OAChB;AAEA,MAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AACtB,QAAA,MAAM,IAAIA,iBAAA,CAAW,CAAA,2BAAA,EAA8B,KAAA,CAAM,IAAI,CAAA,CAAA,CAAG,CAAA;AAAA,MAClE;AAEA,MAAA,IAAI,KAAA,CAAM,SAAS,CAAA,EAAG;AACpB,QAAA,MAAM,IAAIC,oBAAA;AAAA,UACR,CAAA,mCAAA,EACE,KAAA,CAAM,IACR,CAAA,yDAAA,EAA4D,MACzD,GAAA,CAAI,CAAA,IAAA,KAAQ,CAAA,CAAA,EAAIC,+BAAA,CAAmB,IAAI,CAAC,CAAA,CAAA,CAAG,CAAA,CAC3C,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,SACf;AAAA,MACF;AAEA,MAAA,MAAM,SAAA,GAAYA,+BAAA,CAAmB,KAAA,CAAM,CAAC,CAAC,CAAA;AAE7C,MAAA,MAAM,OAAA,CAAQ,aAAA,CAAc,SAAA,EAAW,EAAE,aAAa,CAAA;AAEtD,MAAA,OAAO;AAAA,QACL,MAAA,EAAQ,EAAE,SAAA;AAAU,OACtB;AAAA,IACF;AAAA,GACD,CAAA;AACH;;;;"}

package/dist/actions/index.cjs.js

@@ -6,6 +6,7 @@ var createValidateEntityAction = require('./createValidateEntityAction.cjs.js');
 var createRegisterCatalogEntitiesAction = require('./createRegisterCatalogEntitiesAction.cjs.js');
 var createUnregisterCatalogEntitiesAction = require('./createUnregisterCatalogEntitiesAction.cjs.js');
 var createQueryCatalogEntitiesAction = require('./createQueryCatalogEntitiesAction.cjs.js');
+var createRefreshCatalogEntityAction = require('./createRefreshCatalogEntityAction.cjs.js');
 
 const createCatalogActions = (options) => {
   createGetCatalogModelDescriptionAction.createGetCatalogModelDescriptionAction(options);
@@ -14,6 +15,7 @@ const createCatalogActions = (options) => {
   createRegisterCatalogEntitiesAction.createRegisterCatalogEntitiesAction(options);
   createUnregisterCatalogEntitiesAction.createUnregisterCatalogEntitiesAction(options);
   createQueryCatalogEntitiesAction.createQueryCatalogEntitiesAction(options);
+  createRefreshCatalogEntityAction.createRefreshCatalogEntityAction(options);
 };
 
 exports.createCatalogActions = createCatalogActions;

package/dist/actions/index.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.cjs.js","sources":["../../src/actions/index.ts"],"sourcesContent":["/*\n * Copyright 2025 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { ActionsRegistryService } from '@backstage/backend-plugin-api/alpha';\nimport { CatalogService } from '@backstage/plugin-catalog-node';\nimport { ModelHolder } from '../model/ModelHolder';\nimport { createGetCatalogModelDescriptionAction } from './createGetCatalogModelDescriptionAction.ts';\nimport { createGetCatalogEntityAction } from './createGetCatalogEntityAction.ts';\nimport { createValidateEntityAction } from './createValidateEntityAction.ts';\nimport { createRegisterCatalogEntitiesAction } from './createRegisterCatalogEntitiesAction.ts';\nimport { createUnregisterCatalogEntitiesAction } from './createUnregisterCatalogEntitiesAction.ts';\nimport { createQueryCatalogEntitiesAction } from './createQueryCatalogEntitiesAction.ts';\n\nexport const createCatalogActions = (options: {\n  actionsRegistry: ActionsRegistryService;\n  catalog: CatalogService;\n  modelHolder: ModelHolder | undefined;\n  useExperimentalCatalogLayersDescriptions?: boolean;\n}) => {\n  createGetCatalogModelDescriptionAction(options);\n  createGetCatalogEntityAction(options);\n  createValidateEntityAction(options);\n  createRegisterCatalogEntitiesAction(options);\n  createUnregisterCatalogEntitiesAction(options);\n  createQueryCatalogEntitiesAction(options);\n};\n"],"names":["createGetCatalogModelDescriptionAction","createGetCatalogEntityAction","createValidateEntityAction","createRegisterCatalogEntitiesAction","createUnregisterCatalogEntitiesAction","createQueryCatalogEntitiesAction"],"mappings":";;;;;;;;;AA0BO,MAAM,oBAAA,GAAuB,CAAC,OAAA,KAK/B;AACJ,EAAAA,6EAAA,CAAuC,OAAO,CAAA;AAC9C,EAAAC,yDAAA,CAA6B,OAAO,CAAA;AACpC,EAAAC,qDAAA,CAA2B,OAAO,CAAA;AAClC,EAAAC,uEAAA,CAAoC,OAAO,CAAA;AAC3C,EAAAC,2EAAA,CAAsC,OAAO,CAAA;AAC7C,EAAAC,iEAAA,CAAiC,OAAO,CAAA;AAC1C;;;;"}
+{"version":3,"file":"index.cjs.js","sources":["../../src/actions/index.ts"],"sourcesContent":["/*\n * Copyright 2025 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { ActionsRegistryService } from '@backstage/backend-plugin-api/alpha';\nimport { CatalogService } from '@backstage/plugin-catalog-node';\nimport { ModelHolder } from '../model/ModelHolder';\nimport { createGetCatalogModelDescriptionAction } from './createGetCatalogModelDescriptionAction.ts';\nimport { createGetCatalogEntityAction } from './createGetCatalogEntityAction.ts';\nimport { createValidateEntityAction } from './createValidateEntityAction.ts';\nimport { createRegisterCatalogEntitiesAction } from './createRegisterCatalogEntitiesAction.ts';\nimport { createUnregisterCatalogEntitiesAction } from './createUnregisterCatalogEntitiesAction.ts';\nimport { createQueryCatalogEntitiesAction } from './createQueryCatalogEntitiesAction.ts';\nimport { createRefreshCatalogEntityAction } from './createRefreshCatalogEntityAction.ts';\n\nexport const createCatalogActions = (options: {\n  actionsRegistry: ActionsRegistryService;\n  catalog: CatalogService;\n  modelHolder: ModelHolder | undefined;\n  useExperimentalCatalogLayersDescriptions?: boolean;\n}) => {\n  createGetCatalogModelDescriptionAction(options);\n  createGetCatalogEntityAction(options);\n  createValidateEntityAction(options);\n  createRegisterCatalogEntitiesAction(options);\n  createUnregisterCatalogEntitiesAction(options);\n  createQueryCatalogEntitiesAction(options);\n  createRefreshCatalogEntityAction(options);\n};\n"],"names":["createGetCatalogModelDescriptionAction","createGetCatalogEntityAction","createValidateEntityAction","createRegisterCatalogEntitiesAction","createUnregisterCatalogEntitiesAction","createQueryCatalogEntitiesAction","createRefreshCatalogEntityAction"],"mappings":";;;;;;;;;;AA2BO,MAAM,oBAAA,GAAuB,CAAC,OAAA,KAK/B;AACJ,EAAAA,6EAAA,CAAuC,OAAO,CAAA;AAC9C,EAAAC,yDAAA,CAA6B,OAAO,CAAA;AACpC,EAAAC,qDAAA,CAA2B,OAAO,CAAA;AAClC,EAAAC,uEAAA,CAAoC,OAAO,CAAA;AAC3C,EAAAC,2EAAA,CAAsC,OAAO,CAAA;AAC7C,EAAAC,iEAAA,CAAiC,OAAO,CAAA;AACxC,EAAAC,iEAAA,CAAiC,OAAO,CAAA;AAC1C;;;;"}

package/dist/database/DefaultProcessingDatabase.cjs.js

@@ -2,20 +2,16 @@
 
 var catalogModel = require('@backstage/catalog-model');
 var errors = require('@backstage/errors');
-var lodash = require('lodash');
 var conversion = require('./conversion.cjs.js');
 var metrics = require('./metrics.cjs.js');
 var checkLocationKeyConflict = require('./operations/refreshState/checkLocationKeyConflict.cjs.js');
 var insertUnprocessedEntity = require('./operations/refreshState/insertUnprocessedEntity.cjs.js');
 var updateUnprocessedEntity = require('./operations/refreshState/updateUnprocessedEntity.cjs.js');
 var util = require('./util.cjs.js');
+var syncRelations = require('./operations/relations/syncRelations.cjs.js');
 var luxon = require('luxon');
 var constants = require('../constants.cjs.js');
 
-function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e : { default: e }; }
-
-var lodash__default = /*#__PURE__*/_interopDefaultCompat(lodash);
-
 const BATCH_SIZE = 50;
 class DefaultProcessingDatabase {
   options;
@@ -35,7 +31,6 @@ class DefaultProcessingDatabase {
       refreshKeys,
       locationKey
     } = options;
-    const configClient = tx.client.config.client;
     const refreshResult = await tx("refresh_state").update({
       processed_entity: JSON.stringify(processedEntity),
       result_hash: resultHash,
@@ -57,13 +52,6 @@ class DefaultProcessingDatabase {
       entities: deferredEntities,
       sourceEntityRef
     });
-    let previousRelationRows;
-    if (configClient.includes("sqlite3") || configClient.includes("mysql")) {
-      previousRelationRows = await tx("relations").select("*").where({ originating_entity_id: id });
-      await tx("relations").where({ originating_entity_id: id }).delete();
-    } else {
-      previousRelationRows = await tx("relations").where({ originating_entity_id: id }).delete().returning("*");
-    }
     const relationRows = relations.map(
       ({ source, target, type }) => ({
         originating_entity_id: id,
@@ -72,11 +60,7 @@ class DefaultProcessingDatabase {
         type
       })
     );
-    await tx.batchInsert(
-      "relations",
-      this.deduplicateRelations(relationRows),
-      BATCH_SIZE
-    );
+    const relationsResult = await syncRelations.syncRelations(tx, id, relationRows);
     await tx("refresh_keys").where({ entity_id: id }).delete();
     await tx.batchInsert(
       "refresh_keys",
@@ -87,9 +71,7 @@ class DefaultProcessingDatabase {
       BATCH_SIZE
     );
     return {
-      previous: {
-        relations: previousRelationRows
-      }
+      relationsChange: relationsResult
     };
   }
   async updateProcessedEntityErrors(txOpaque, options) {
@@ -186,12 +168,6 @@ class DefaultProcessingDatabase {
       throw conversion.rethrowError(e);
     }
   }
-  deduplicateRelations(rows) {
-    return lodash__default.default.uniqBy(
-      rows,
-      (r) => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`
-    );
-  }
   /**
    * Add a set of deferred entities for processing.
    * The entities will be added at the front of the processing queue.

package/dist/database/DefaultProcessingDatabase.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultProcessingDatabase.cjs.js","sources":["../../src/database/DefaultProcessingDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError } from '@backstage/errors';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { ProcessingIntervalFunction } from '../processing/refresh';\nimport { rethrowError, timestampToDateTime } from './conversion';\nimport { initDatabaseMetrics } from './metrics';\nimport {\n  DbRefreshKeysRow,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n} from './tables';\nimport {\n  GetProcessableEntitiesResult,\n  ListParentsOptions,\n  ListParentsResult,\n  ProcessingDatabase,\n  RefreshStateItem,\n  Transaction,\n  UpdateEntityCacheOptions,\n  UpdateProcessedEntityOptions,\n} from './types';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { generateStableHash, generateTargetKey } from './util';\nimport { EventParams, EventsService } from '@backstage/plugin-events-node';\nimport { DateTime } from 'luxon';\nimport { CATALOG_CONFLICTS_TOPIC } from '../constants';\nimport { CatalogConflictEventPayload } from '../catalog/types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\nimport { MetricsService } from '@backstage/backend-plugin-api/alpha';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProcessingDatabase implements ProcessingDatabase {\n  private readonly options: {\n    database: Knex;\n    logger: LoggerService;\n    refreshInterval: ProcessingIntervalFunction;\n    events: EventsService;\n    metrics: MetricsService;\n  };\n\n  constructor(options: {\n    database: Knex;\n    logger: LoggerService;\n    refreshInterval: ProcessingIntervalFunction;\n    events: EventsService;\n    metrics: MetricsService;\n  }) {\n    this.options = options;\n    initDatabaseMetrics(options.database, options.metrics);\n  }\n\n  async updateProcessedEntity(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<{ previous: { relations: DbRelationsRow[] } }> {\n    const tx = txOpaque as Knex.Transaction;\n    const {\n      id,\n      processedEntity,\n      resultHash,\n      errors,\n      relations,\n      deferredEntities,\n      refreshKeys,\n      locationKey,\n    } = options;\n    const configClient = tx.client.config.client;\n    const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        processed_entity: JSON.stringify(processedEntity),\n        result_hash: resultHash,\n        errors,\n        location_key: locationKey,\n      })\n      .where('entity_id', id)\n      .andWhere(inner => {\n        if (!locationKey) {\n          return inner.whereNull('location_key');\n        }\n        return inner\n          .where('location_key', locationKey)\n          .orWhereNull('location_key');\n      });\n    if (refreshResult === 0) {\n      throw new ConflictError(\n        `Conflicting write of processing result for ${id} with location key '${locationKey}'`,\n      );\n    }\n    const sourceEntityRef = stringifyEntityRef(processedEntity);\n\n    // Schedule all deferred entities for future processing.\n    await this.addUnprocessedEntities(tx, {\n      entities: deferredEntities,\n      sourceEntityRef,\n    });\n\n    // Delete old relations\n    // NOTE(freben): knex implemented support for returning() on update queries for sqlite, but at the current time of writing (Sep 2022) not for delete() queries.\n    let previousRelationRows: DbRelationsRow[];\n    if (configClient.includes('sqlite3') || configClient.includes('mysql')) {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .select('*')\n        .where({ originating_entity_id: id });\n      await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete();\n    } else {\n      previousRelationRows = await tx<DbRelationsRow>('relations')\n        .where({ originating_entity_id: id })\n        .delete()\n        .returning('*');\n    }\n\n    // Batch insert new relations\n    const relationRows: DbRelationsRow[] = relations.map(\n      ({ source, target, type }) => ({\n        originating_entity_id: id,\n        source_entity_ref: stringifyEntityRef(source),\n        target_entity_ref: stringifyEntityRef(target),\n        type,\n      }),\n    );\n\n    await tx.batchInsert(\n      'relations',\n      this.deduplicateRelations(relationRows),\n      BATCH_SIZE,\n    );\n\n    // Delete old refresh keys\n    await tx<DbRefreshKeysRow>('refresh_keys')\n      .where({ entity_id: id })\n      .delete();\n\n    // Insert the refresh keys for the processed entity\n    await tx.batchInsert(\n      'refresh_keys',\n      refreshKeys.map(k => ({\n        entity_id: id,\n        key: generateTargetKey(k.key),\n      })),\n      BATCH_SIZE,\n    );\n\n    return {\n      previous: {\n        relations: previousRelationRows,\n      },\n    };\n  }\n\n  async updateProcessedEntityErrors(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, errors, resultHash } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        errors,\n        result_hash: resultHash,\n      })\n      .where('entity_id', id);\n  }\n\n  async updateEntityCache(\n    txOpaque: Transaction,\n    options: UpdateEntityCacheOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, state } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({ cache: JSON.stringify(state ?? {}) })\n      .where('entity_id', id);\n  }\n\n  async getProcessableEntities(\n    maybeTx: Transaction | Knex,\n    request: { processBatchSize: number },\n  ): Promise<GetProcessableEntitiesResult> {\n    const knex = maybeTx as Knex.Transaction | Knex;\n    const useLocking = ['mysql', 'mysql2', 'pg'].includes(\n      knex.client.config.client,\n    );\n\n    const interval = this.options.refreshInterval();\n\n    const nextUpdateAt = (\n      tx: Knex | Knex.Transaction,\n      refreshInterval: number,\n    ) => {\n      if (tx.client.config.client.includes('sqlite3')) {\n        return tx.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);\n      } else if (tx.client.config.client.includes('mysql')) {\n        return tx.raw(`now() + interval ${refreshInterval} second`);\n      }\n      return tx.raw(`now() + interval '${refreshInterval} seconds'`);\n    };\n\n    // The SELECT FOR UPDATE SKIP LOCKED + UPDATE must run inside a\n    // single transaction so that the row locks persist until\n    // next_update_at has been bumped.\n    const run = async (tx: Knex | Knex.Transaction) => {\n      const items: DbRefreshStateRow[] = await tx('refresh_state')\n        .select([\n          'entity_id',\n          'entity_ref',\n          'unprocessed_entity',\n          'result_hash',\n          'cache',\n          'errors',\n          'location_key',\n          'next_update_at',\n        ])\n        .where('next_update_at', '<=', tx.fn.now())\n        .limit(request.processBatchSize)\n        .orderBy('next_update_at', 'asc')\n        .modify(qb => {\n          if (useLocking) {\n            qb.forUpdate().skipLocked();\n          }\n        });\n\n      if (items.length > 0) {\n        await tx('refresh_state')\n          .whereIn(\n            'entity_ref',\n            items.map(i => i.entity_ref),\n          )\n          .update({\n            next_update_at: nextUpdateAt(tx, interval),\n          });\n      }\n\n      return items;\n    };\n\n    const items =\n      knex.isTransaction || !useLocking\n        ? await run(knex)\n        : await knex.transaction(run);\n\n    return {\n      items: items.map(\n        i =>\n          ({\n            id: i.entity_id,\n            entityRef: i.entity_ref,\n            unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity,\n            resultHash: i.result_hash || '',\n            nextUpdateAt: timestampToDateTime(i.next_update_at),\n            state: i.cache ? JSON.parse(i.cache) : undefined,\n            errors: i.errors,\n            locationKey: i.location_key,\n          } satisfies RefreshStateItem),\n      ),\n    };\n  }\n\n  async listParents(\n    txOpaque: Transaction,\n    options: ListParentsOptions,\n  ): Promise<ListParentsResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .whereIn('target_entity_ref', options.entityRefs)\n      .select();\n\n    const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);\n\n    return { entityRefs };\n  }\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] {\n    return lodash.uniqBy(\n      rows,\n      r => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`,\n    );\n  }\n\n  /**\n   * Add a set of deferred entities for processing.\n   * The entities will be added at the front of the processing queue.\n   */\n  private async addUnprocessedEntities(\n    txOpaque: Transaction,\n    options: {\n      sourceEntityRef: string;\n      entities: DeferredEntity[];\n    },\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n\n    // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards\n    const stateReferences = new Array<string>();\n\n    // Upsert all of the unprocessed entities into the refresh_state table, by\n    // their entity ref.\n    for (const { entity, locationKey } of options.entities) {\n      const entityRef = stringifyEntityRef(entity);\n      const hash = generateStableHash(entity);\n\n      const updated = await updateUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n      });\n      if (updated) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      const inserted = await insertUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n        logger: this.options.logger,\n      });\n      if (inserted) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      // If the row can't be inserted, we have a conflict, but it could be either\n      // because of a conflicting locationKey or a race with another instance, so check\n      // whether the conflicting entity has the same entityRef but a different locationKey\n      const conflictingKey = await checkLocationKeyConflict({\n        tx,\n        entityRef,\n        locationKey,\n      });\n      if (conflictingKey) {\n        this.options.logger.warn(\n          `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n        );\n        if (locationKey) {\n          const eventParams: EventParams<CatalogConflictEventPayload> = {\n            topic: CATALOG_CONFLICTS_TOPIC,\n            eventPayload: {\n              unprocessedEntity: entity,\n              entityRef,\n              newLocationKey: locationKey,\n              existingLocationKey: conflictingKey,\n              lastConflictAt: DateTime.now().toISO()!,\n            },\n          };\n          await this.options.events.publish(eventParams);\n        }\n      }\n    }\n\n    // Lastly, replace refresh state references for the originating entity and any successfully added entities\n    await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n      // Remove all existing references from the originating entity\n      .where({ source_entity_ref: options.sourceEntityRef })\n      // And remove any existing references to entities that we're inserting new references for\n      .orWhereIn('target_entity_ref', stateReferences)\n      .delete();\n    await tx.batchInsert(\n      'refresh_state_references',\n      stateReferences.map(entityRef => ({\n        source_entity_ref: options.sourceEntityRef,\n        target_entity_ref: entityRef,\n      })),\n      BATCH_SIZE,\n    );\n  }\n}\n"],"names":["initDatabaseMetrics","errors","ConflictError","stringifyEntityRef","generateTargetKey","items","timestampToDateTime","rethrowError","lodash","generateStableHash","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","CATALOG_CONFLICTS_TOPIC","DateTime"],"mappings":";;;;;;;;;;;;;;;;;;AAuDA,MAAM,UAAA,GAAa,EAAA;AAEZ,MAAM,yBAAA,CAAwD;AAAA,EAClD,OAAA;AAAA,EAQjB,YAAY,OAAA,EAMT;AACD,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA;AACf,IAAAA,2BAAA,CAAoB,OAAA,CAAQ,QAAA,EAAU,OAAA,CAAQ,OAAO,CAAA;AAAA,EACvD;AAAA,EAEA,MAAM,qBAAA,CACJ,QAAA,EACA,OAAA,EACwD;AACxD,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM;AAAA,MACJ,EAAA;AAAA,MACA,eAAA;AAAA,MACA,UAAA;AAAA,cACAC,QAAA;AAAA,MACA,SAAA;AAAA,MACA,gBAAA;AAAA,MACA,WAAA;AAAA,MACA;AAAA,KACF,GAAI,OAAA;AACJ,IAAA,MAAM,YAAA,GAAe,EAAA,CAAG,MAAA,CAAO,MAAA,CAAO,MAAA;AACtC,IAAA,MAAM,aAAA,GAAgB,MAAM,EAAA,CAAsB,eAAe,EAC9D,MAAA,CAAO;AAAA,MACN,gBAAA,EAAkB,IAAA,CAAK,SAAA,CAAU,eAAe,CAAA;AAAA,MAChD,WAAA,EAAa,UAAA;AAAA,cACbA,QAAA;AAAA,MACA,YAAA,EAAc;AAAA,KACf,CAAA,CACA,KAAA,CAAM,aAAa,EAAE,CAAA,CACrB,SAAS,CAAA,KAAA,KAAS;AACjB,MAAA,IAAI,CAAC,WAAA,EAAa;AAChB,QAAA,OAAO,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA,MACvC;AACA,MAAA,OAAO,MACJ,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,IAC/B,CAAC,CAAA;AACH,IAAA,IAAI,kBAAkB,CAAA,EAAG;AACvB,MAAA,MAAM,IAAIC,oBAAA;AAAA,QACR,CAAA,2CAAA,EAA8C,EAAE,CAAA,oBAAA,EAAuB,WAAW,CAAA,CAAA;AAAA,OACpF;AAAA,IACF;AACA,IAAA,MAAM,eAAA,GAAkBC,gCAAmB,eAAe,CAAA;AAG1D,IAAA,MAAM,IAAA,CAAK,uBAAuB,EAAA,EAAI;AAAA,MACpC,QAAA,EAAU,gBAAA;AAAA,MACV;AAAA,KACD,CAAA;AAID,IAAA,IAAI,oBAAA;AACJ,IAAA,IAAI,aAAa,QAAA,CAAS,SAAS,KAAK,YAAA,CAAa,QAAA,CAAS,OAAO,CAAA,EAAG;AACtE,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,MAAA,CAAO,GAAG,CAAA,CACV,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA;AACtC,MAAA,MAAM,EAAA,CAAmB,WAAW,CAAA,CACjC,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO;AAAA,IACZ,CAAA,MAAO;AACL,MAAA,oBAAA,GAAuB,MAAM,EAAA,CAAmB,WAAW,CAAA,CACxD,KAAA,CAAM,EAAE,qBAAA,EAAuB,EAAA,EAAI,CAAA,CACnC,MAAA,EAAO,CACP,UAAU,GAAG,CAAA;AAAA,IAClB;AAGA,IAAA,MAAM,eAAiC,SAAA,CAAU,GAAA;AAAA,MAC/C,CAAC,EAAE,MAAA,EAAQ,MAAA,EAAQ,MAAK,MAAO;AAAA,QAC7B,qBAAA,EAAuB,EAAA;AAAA,QACvB,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C;AAAA,OACF;AAAA,KACF;AAEA,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,WAAA;AAAA,MACA,IAAA,CAAK,qBAAqB,YAAY,CAAA;AAAA,MACtC;AAAA,KACF;AAGA,IAAA,MAAM,EAAA,CAAqB,cAAc,CAAA,CACtC,KAAA,CAAM,EAAE,SAAA,EAAW,EAAA,EAAI,CAAA,CACvB,MAAA,EAAO;AAGV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,cAAA;AAAA,MACA,WAAA,CAAY,IAAI,CAAA,CAAA,MAAM;AAAA,QACpB,SAAA,EAAW,EAAA;AAAA,QACX,GAAA,EAAKC,sBAAA,CAAkB,CAAA,CAAE,GAAG;AAAA,OAC9B,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAEA,IAAA,OAAO;AAAA,MACL,QAAA,EAAU;AAAA,QACR,SAAA,EAAW;AAAA;AACb,KACF;AAAA,EACF;AAAA,EAEA,MAAM,2BAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,MAAA,EAAQ,UAAA,EAAW,GAAI,OAAA;AAEnC,IAAA,MAAM,EAAA,CAAsB,eAAe,CAAA,CACxC,MAAA,CAAO;AAAA,MACN,MAAA;AAAA,MACA,WAAA,EAAa;AAAA,KACd,CAAA,CACA,KAAA,CAAM,WAAA,EAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,iBAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,KAAA,EAAM,GAAI,OAAA;AAEtB,IAAA,MAAM,GAAsB,eAAe,CAAA,CACxC,MAAA,CAAO,EAAE,OAAO,IAAA,CAAK,SAAA,CAAU,KAAA,IAAS,EAAE,CAAA,EAAG,CAAA,CAC7C,KAAA,CAAM,aAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,sBAAA,CACJ,OAAA,EACA,OAAA,EACuC;AACvC,IAAA,MAAM,IAAA,GAAO,OAAA;AACb,IAAA,MAAM,UAAA,GAAa,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,QAAA;AAAA,MAC3C,IAAA,CAAK,OAAO,MAAA,CAAO;AAAA,KACrB;AAEA,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,OAAA,CAAQ,eAAA,EAAgB;AAE9C,IAAA,MAAM,YAAA,GAAe,CACnB,EAAA,EACA,eAAA,KACG;AACH,MAAA,IAAI,GAAG,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AAC/C,QAAA,OAAO,GAAG,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,eAAe,UAAU,CAAC,CAAA;AAAA,MACpE,WAAW,EAAA,CAAG,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACpD,QAAA,OAAO,EAAA,CAAG,GAAA,CAAI,CAAA,iBAAA,EAAoB,eAAe,CAAA,OAAA,CAAS,CAAA;AAAA,MAC5D;AACA,MAAA,OAAO,EAAA,CAAG,GAAA,CAAI,CAAA,kBAAA,EAAqB,eAAe,CAAA,SAAA,CAAW,CAAA;AAAA,IAC/D,CAAA;AAKA,IAAA,MAAM,GAAA,GAAM,OAAO,EAAA,KAAgC;AACjD,MAAA,MAAMC,MAAAA,GAA6B,MAAM,EAAA,CAAG,eAAe,EACxD,MAAA,CAAO;AAAA,QACN,WAAA;AAAA,QACA,YAAA;AAAA,QACA,oBAAA;AAAA,QACA,aAAA;AAAA,QACA,OAAA;AAAA,QACA,QAAA;AAAA,QACA,cAAA;AAAA,QACA;AAAA,OACD,CAAA,CACA,KAAA,CAAM,kBAAkB,IAAA,EAAM,EAAA,CAAG,GAAG,GAAA,EAAK,EACzC,KAAA,CAAM,OAAA,CAAQ,gBAAgB,CAAA,CAC9B,OAAA,CAAQ,kBAAkB,KAAK,CAAA,CAC/B,OAAO,CAAA,EAAA,KAAM;AACZ,QAAA,IAAI,UAAA,EAAY;AACd,UAAA,EAAA,CAAG,SAAA,GAAY,UAAA,EAAW;AAAA,QAC5B;AAAA,MACF,CAAC,CAAA;AAEH,MAAA,IAAIA,MAAAA,CAAM,SAAS,CAAA,EAAG;AACpB,QAAA,MAAM,EAAA,CAAG,eAAe,CAAA,CACrB,OAAA;AAAA,UACC,YAAA;AAAA,UACAA,MAAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,UAE5B,MAAA,CAAO;AAAA,UACN,cAAA,EAAgB,YAAA,CAAa,EAAA,EAAI,QAAQ;AAAA,SAC1C,CAAA;AAAA,MACL;AAEA,MAAA,OAAOA,MAAAA;AAAA,IACT,CAAA;AAEA,IAAA,MAAM,KAAA,GACJ,IAAA,CAAK,aAAA,IAAiB,CAAC,UAAA,GACnB,MAAM,GAAA,CAAI,IAAI,CAAA,GACd,MAAM,IAAA,CAAK,WAAA,CAAY,GAAG,CAAA;AAEhC,IAAA,OAAO;AAAA,MACL,OAAO,KAAA,CAAM,GAAA;AAAA,QACX,CAAA,CAAA,MACG;AAAA,UACC,IAAI,CAAA,CAAE,SAAA;AAAA,UACN,WAAW,CAAA,CAAE,UAAA;AAAA,UACb,iBAAA,EAAmB,IAAA,CAAK,KAAA,CAAM,CAAA,CAAE,kBAAkB,CAAA;AAAA,UAClD,UAAA,EAAY,EAAE,WAAA,IAAe,EAAA;AAAA,UAC7B,YAAA,EAAcC,8BAAA,CAAoB,CAAA,CAAE,cAAc,CAAA;AAAA,UAClD,OAAO,CAAA,CAAE,KAAA,GAAQ,KAAK,KAAA,CAAM,CAAA,CAAE,KAAK,CAAA,GAAI,MAAA;AAAA,UACvC,QAAQ,CAAA,CAAE,MAAA;AAAA,UACV,aAAa,CAAA,CAAE;AAAA,SACjB;AAAA;AACJ,KACF;AAAA,EACF;AAAA,EAEA,MAAM,WAAA,CACJ,QAAA,EACA,OAAA,EAC4B;AAC5B,IAAA,MAAM,EAAA,GAAK,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,MAEC,OAAA,CAAQ,mBAAA,EAAqB,OAAA,CAAQ,UAAU,EAC/C,MAAA,EAAO;AAEV,IAAA,MAAM,UAAA,GAAa,KAAK,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,iBAAkB,CAAA,CAAE,OAAO,OAAO,CAAA;AAErE,IAAA,OAAO,EAAE,UAAA,EAAW;AAAA,EACtB;AAAA,EAEA,MAAM,YAAe,EAAA,EAAiD;AACpE,IAAA,IAAI;AACF,MAAA,IAAI,MAAA,GAAwB,KAAA,CAAA;AAE5B,MAAA,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,WAAA;AAAA,QAC1B,OAAM,EAAA,KAAM;AAGV,UAAA,MAAA,GAAS,MAAM,GAAG,EAAE,CAAA;AAAA,QACtB,CAAA;AAAA,QACA;AAAA;AAAA,UAEE,qBAAA,EAAuB;AAAA;AACzB,OACF;AAEA,MAAA,OAAO,MAAA;AAAA,IACT,SAAS,CAAA,EAAG;AACV,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,0BAAA,EAA6B,CAAC,CAAA,CAAE,CAAA;AAC1D,MAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA,IACtB;AAAA,EACF;AAAA,EAEQ,qBAAqB,IAAA,EAA0C;AACrE,IAAA,OAAOC,uBAAA,CAAO,MAAA;AAAA,MACZ,IAAA;AAAA,MACA,CAAA,CAAA,KAAK,GAAG,CAAA,CAAE,iBAAiB,IAAI,CAAA,CAAE,iBAAiB,CAAA,CAAA,EAAI,CAAA,CAAE,IAAI,CAAA;AAAA,KAC9D;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,sBAAA,CACZ,QAAA,EACA,OAAA,EAIe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AAGX,IAAA,MAAM,eAAA,GAAkB,IAAI,KAAA,EAAc;AAI1C,IAAA,KAAA,MAAW,EAAE,MAAA,EAAQ,WAAA,EAAY,IAAK,QAAQ,QAAA,EAAU;AACtD,MAAA,MAAM,SAAA,GAAYL,gCAAmB,MAAM,CAAA;AAC3C,MAAA,MAAM,IAAA,GAAOM,wBAAmB,MAAM,CAAA;AAEtC,MAAA,MAAM,OAAA,GAAU,MAAMC,+CAAA,CAAwB;AAAA,QAC5C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,OAAA,EAAS;AACX,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,QAAA,GAAW,MAAMC,+CAAA,CAAwB;AAAA,QAC7C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA,WAAA;AAAA,QACA,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAKA,MAAA,MAAM,cAAA,GAAiB,MAAMC,iDAAA,CAAyB;AAAA,QACpD,EAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,cAAA,EAAgB;AAClB,QAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,UAClB,CAAA,+BAAA,EAAkC,SAAS,CAAA,uBAAA,EAA0B,cAAc,iBAAiB,WAAW,CAAA;AAAA,SACjH;AACA,QAAA,IAAI,WAAA,EAAa;AACf,UAAA,MAAM,WAAA,GAAwD;AAAA,YAC5D,KAAA,EAAOC,iCAAA;AAAA,YACP,YAAA,EAAc;AAAA,cACZ,iBAAA,EAAmB,MAAA;AAAA,cACnB,SAAA;AAAA,cACA,cAAA,EAAgB,WAAA;AAAA,cAChB,mBAAA,EAAqB,cAAA;AAAA,cACrB,cAAA,EAAgBC,cAAA,CAAS,GAAA,EAAI,CAAE,KAAA;AAAM;AACvC,WACF;AACA,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,OAAA,CAAQ,WAAW,CAAA;AAAA,QAC/C;AAAA,MACF;AAAA,IACF;AAGA,IAAA,MAAM,EAAA,CAAgC,0BAA0B,CAAA,CAE7D,KAAA,CAAM,EAAE,iBAAA,EAAmB,OAAA,CAAQ,eAAA,EAAiB,CAAA,CAEpD,SAAA,CAAU,mBAAA,EAAqB,eAAe,EAC9C,MAAA,EAAO;AACV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,0BAAA;AAAA,MACA,eAAA,CAAgB,IAAI,CAAA,SAAA,MAAc;AAAA,QAChC,mBAAmB,OAAA,CAAQ,eAAA;AAAA,QAC3B,iBAAA,EAAmB;AAAA,OACrB,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAAA,EACF;AACF;;;;"}
+{"version":3,"file":"DefaultProcessingDatabase.cjs.js","sources":["../../src/database/DefaultProcessingDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Entity, stringifyEntityRef } from '@backstage/catalog-model';\nimport { ConflictError } from '@backstage/errors';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport { ProcessingIntervalFunction } from '../processing/refresh';\nimport { rethrowError, timestampToDateTime } from './conversion';\nimport { initDatabaseMetrics } from './metrics';\nimport {\n  DbRefreshKeysRow,\n  DbRefreshStateReferencesRow,\n  DbRefreshStateRow,\n  DbRelationsRow,\n} from './tables';\nimport {\n  GetProcessableEntitiesResult,\n  ListParentsOptions,\n  ListParentsResult,\n  ProcessingDatabase,\n  RefreshStateItem,\n  Transaction,\n  UpdateEntityCacheOptions,\n  UpdateProcessedEntityOptions,\n} from './types';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { generateStableHash, generateTargetKey } from './util';\nimport {\n  syncRelations,\n  SyncRelationsResult,\n} from './operations/relations/syncRelations';\nimport { EventParams, EventsService } from '@backstage/plugin-events-node';\nimport { DateTime } from 'luxon';\nimport { CATALOG_CONFLICTS_TOPIC } from '../constants';\nimport { CatalogConflictEventPayload } from '../catalog/types';\nimport { LoggerService } from '@backstage/backend-plugin-api';\nimport { MetricsService } from '@backstage/backend-plugin-api/alpha';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProcessingDatabase implements ProcessingDatabase {\n  private readonly options: {\n    database: Knex;\n    logger: LoggerService;\n    refreshInterval: ProcessingIntervalFunction;\n    events: EventsService;\n    metrics: MetricsService;\n  };\n\n  constructor(options: {\n    database: Knex;\n    logger: LoggerService;\n    refreshInterval: ProcessingIntervalFunction;\n    events: EventsService;\n    metrics: MetricsService;\n  }) {\n    this.options = options;\n    initDatabaseMetrics(options.database, options.metrics);\n  }\n\n  async updateProcessedEntity(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<{ relationsChange: SyncRelationsResult }> {\n    const tx = txOpaque as Knex.Transaction;\n    const {\n      id,\n      processedEntity,\n      resultHash,\n      errors,\n      relations,\n      deferredEntities,\n      refreshKeys,\n      locationKey,\n    } = options;\n    const refreshResult = await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        processed_entity: JSON.stringify(processedEntity),\n        result_hash: resultHash,\n        errors,\n        location_key: locationKey,\n      })\n      .where('entity_id', id)\n      .andWhere(inner => {\n        if (!locationKey) {\n          return inner.whereNull('location_key');\n        }\n        return inner\n          .where('location_key', locationKey)\n          .orWhereNull('location_key');\n      });\n    if (refreshResult === 0) {\n      throw new ConflictError(\n        `Conflicting write of processing result for ${id} with location key '${locationKey}'`,\n      );\n    }\n    const sourceEntityRef = stringifyEntityRef(processedEntity);\n\n    // Schedule all deferred entities for future processing.\n    await this.addUnprocessedEntities(tx, {\n      entities: deferredEntities,\n      sourceEntityRef,\n    });\n\n    // Sync relations using a diff-based approach — only touches rows that\n    // actually changed, eliminating steady-state write churn.\n    const relationRows: DbRelationsRow[] = relations.map(\n      ({ source, target, type }) => ({\n        originating_entity_id: id,\n        source_entity_ref: stringifyEntityRef(source),\n        target_entity_ref: stringifyEntityRef(target),\n        type,\n      }),\n    );\n\n    const relationsResult = await syncRelations(tx, id, relationRows);\n\n    // Delete old refresh keys\n    await tx<DbRefreshKeysRow>('refresh_keys')\n      .where({ entity_id: id })\n      .delete();\n\n    // Insert the refresh keys for the processed entity\n    await tx.batchInsert(\n      'refresh_keys',\n      refreshKeys.map(k => ({\n        entity_id: id,\n        key: generateTargetKey(k.key),\n      })),\n      BATCH_SIZE,\n    );\n\n    return {\n      relationsChange: relationsResult,\n    };\n  }\n\n  async updateProcessedEntityErrors(\n    txOpaque: Transaction,\n    options: UpdateProcessedEntityOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, errors, resultHash } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({\n        errors,\n        result_hash: resultHash,\n      })\n      .where('entity_id', id);\n  }\n\n  async updateEntityCache(\n    txOpaque: Transaction,\n    options: UpdateEntityCacheOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n    const { id, state } = options;\n\n    await tx<DbRefreshStateRow>('refresh_state')\n      .update({ cache: JSON.stringify(state ?? {}) })\n      .where('entity_id', id);\n  }\n\n  async getProcessableEntities(\n    maybeTx: Transaction | Knex,\n    request: { processBatchSize: number },\n  ): Promise<GetProcessableEntitiesResult> {\n    const knex = maybeTx as Knex.Transaction | Knex;\n    const useLocking = ['mysql', 'mysql2', 'pg'].includes(\n      knex.client.config.client,\n    );\n\n    const interval = this.options.refreshInterval();\n\n    const nextUpdateAt = (\n      tx: Knex | Knex.Transaction,\n      refreshInterval: number,\n    ) => {\n      if (tx.client.config.client.includes('sqlite3')) {\n        return tx.raw(`datetime('now', ?)`, [`${refreshInterval} seconds`]);\n      } else if (tx.client.config.client.includes('mysql')) {\n        return tx.raw(`now() + interval ${refreshInterval} second`);\n      }\n      return tx.raw(`now() + interval '${refreshInterval} seconds'`);\n    };\n\n    // The SELECT FOR UPDATE SKIP LOCKED + UPDATE must run inside a\n    // single transaction so that the row locks persist until\n    // next_update_at has been bumped.\n    const run = async (tx: Knex | Knex.Transaction) => {\n      const items: DbRefreshStateRow[] = await tx('refresh_state')\n        .select([\n          'entity_id',\n          'entity_ref',\n          'unprocessed_entity',\n          'result_hash',\n          'cache',\n          'errors',\n          'location_key',\n          'next_update_at',\n        ])\n        .where('next_update_at', '<=', tx.fn.now())\n        .limit(request.processBatchSize)\n        .orderBy('next_update_at', 'asc')\n        .modify(qb => {\n          if (useLocking) {\n            qb.forUpdate().skipLocked();\n          }\n        });\n\n      if (items.length > 0) {\n        await tx('refresh_state')\n          .whereIn(\n            'entity_ref',\n            items.map(i => i.entity_ref),\n          )\n          .update({\n            next_update_at: nextUpdateAt(tx, interval),\n          });\n      }\n\n      return items;\n    };\n\n    const items =\n      knex.isTransaction || !useLocking\n        ? await run(knex)\n        : await knex.transaction(run);\n\n    return {\n      items: items.map(\n        i =>\n          ({\n            id: i.entity_id,\n            entityRef: i.entity_ref,\n            unprocessedEntity: JSON.parse(i.unprocessed_entity) as Entity,\n            resultHash: i.result_hash || '',\n            nextUpdateAt: timestampToDateTime(i.next_update_at),\n            state: i.cache ? JSON.parse(i.cache) : undefined,\n            errors: i.errors,\n            locationKey: i.location_key,\n          } satisfies RefreshStateItem),\n      ),\n    };\n  }\n\n  async listParents(\n    txOpaque: Transaction,\n    options: ListParentsOptions,\n  ): Promise<ListParentsResult> {\n    const tx = txOpaque as Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .whereIn('target_entity_ref', options.entityRefs)\n      .select();\n\n    const entityRefs = rows.map(r => r.source_entity_ref!).filter(Boolean);\n\n    return { entityRefs };\n  }\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  /**\n   * Add a set of deferred entities for processing.\n   * The entities will be added at the front of the processing queue.\n   */\n  private async addUnprocessedEntities(\n    txOpaque: Transaction,\n    options: {\n      sourceEntityRef: string;\n      entities: DeferredEntity[];\n    },\n  ): Promise<void> {\n    const tx = txOpaque as Knex.Transaction;\n\n    // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards\n    const stateReferences = new Array<string>();\n\n    // Upsert all of the unprocessed entities into the refresh_state table, by\n    // their entity ref.\n    for (const { entity, locationKey } of options.entities) {\n      const entityRef = stringifyEntityRef(entity);\n      const hash = generateStableHash(entity);\n\n      const updated = await updateUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n      });\n      if (updated) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      const inserted = await insertUnprocessedEntity({\n        tx,\n        entity,\n        hash,\n        locationKey,\n        logger: this.options.logger,\n      });\n      if (inserted) {\n        stateReferences.push(entityRef);\n        continue;\n      }\n\n      // If the row can't be inserted, we have a conflict, but it could be either\n      // because of a conflicting locationKey or a race with another instance, so check\n      // whether the conflicting entity has the same entityRef but a different locationKey\n      const conflictingKey = await checkLocationKeyConflict({\n        tx,\n        entityRef,\n        locationKey,\n      });\n      if (conflictingKey) {\n        this.options.logger.warn(\n          `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n        );\n        if (locationKey) {\n          const eventParams: EventParams<CatalogConflictEventPayload> = {\n            topic: CATALOG_CONFLICTS_TOPIC,\n            eventPayload: {\n              unprocessedEntity: entity,\n              entityRef,\n              newLocationKey: locationKey,\n              existingLocationKey: conflictingKey,\n              lastConflictAt: DateTime.now().toISO()!,\n            },\n          };\n          await this.options.events.publish(eventParams);\n        }\n      }\n    }\n\n    // Lastly, replace refresh state references for the originating entity and any successfully added entities\n    await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n      // Remove all existing references from the originating entity\n      .where({ source_entity_ref: options.sourceEntityRef })\n      // And remove any existing references to entities that we're inserting new references for\n      .orWhereIn('target_entity_ref', stateReferences)\n      .delete();\n    await tx.batchInsert(\n      'refresh_state_references',\n      stateReferences.map(entityRef => ({\n        source_entity_ref: options.sourceEntityRef,\n        target_entity_ref: entityRef,\n      })),\n      BATCH_SIZE,\n    );\n  }\n}\n"],"names":["initDatabaseMetrics","errors","ConflictError","stringifyEntityRef","syncRelations","generateTargetKey","items","timestampToDateTime","rethrowError","generateStableHash","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","CATALOG_CONFLICTS_TOPIC","DateTime"],"mappings":";;;;;;;;;;;;;;AA0DA,MAAM,UAAA,GAAa,EAAA;AAEZ,MAAM,yBAAA,CAAwD;AAAA,EAClD,OAAA;AAAA,EAQjB,YAAY,OAAA,EAMT;AACD,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA;AACf,IAAAA,2BAAA,CAAoB,OAAA,CAAQ,QAAA,EAAU,OAAA,CAAQ,OAAO,CAAA;AAAA,EACvD;AAAA,EAEA,MAAM,qBAAA,CACJ,QAAA,EACA,OAAA,EACmD;AACnD,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM;AAAA,MACJ,EAAA;AAAA,MACA,eAAA;AAAA,MACA,UAAA;AAAA,cACAC,QAAA;AAAA,MACA,SAAA;AAAA,MACA,gBAAA;AAAA,MACA,WAAA;AAAA,MACA;AAAA,KACF,GAAI,OAAA;AACJ,IAAA,MAAM,aAAA,GAAgB,MAAM,EAAA,CAAsB,eAAe,EAC9D,MAAA,CAAO;AAAA,MACN,gBAAA,EAAkB,IAAA,CAAK,SAAA,CAAU,eAAe,CAAA;AAAA,MAChD,WAAA,EAAa,UAAA;AAAA,cACbA,QAAA;AAAA,MACA,YAAA,EAAc;AAAA,KACf,CAAA,CACA,KAAA,CAAM,aAAa,EAAE,CAAA,CACrB,SAAS,CAAA,KAAA,KAAS;AACjB,MAAA,IAAI,CAAC,WAAA,EAAa;AAChB,QAAA,OAAO,KAAA,CAAM,UAAU,cAAc,CAAA;AAAA,MACvC;AACA,MAAA,OAAO,MACJ,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,YAAY,cAAc,CAAA;AAAA,IAC/B,CAAC,CAAA;AACH,IAAA,IAAI,kBAAkB,CAAA,EAAG;AACvB,MAAA,MAAM,IAAIC,oBAAA;AAAA,QACR,CAAA,2CAAA,EAA8C,EAAE,CAAA,oBAAA,EAAuB,WAAW,CAAA,CAAA;AAAA,OACpF;AAAA,IACF;AACA,IAAA,MAAM,eAAA,GAAkBC,gCAAmB,eAAe,CAAA;AAG1D,IAAA,MAAM,IAAA,CAAK,uBAAuB,EAAA,EAAI;AAAA,MACpC,QAAA,EAAU,gBAAA;AAAA,MACV;AAAA,KACD,CAAA;AAID,IAAA,MAAM,eAAiC,SAAA,CAAU,GAAA;AAAA,MAC/C,CAAC,EAAE,MAAA,EAAQ,MAAA,EAAQ,MAAK,MAAO;AAAA,QAC7B,qBAAA,EAAuB,EAAA;AAAA,QACvB,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C,iBAAA,EAAmBA,gCAAmB,MAAM,CAAA;AAAA,QAC5C;AAAA,OACF;AAAA,KACF;AAEA,IAAA,MAAM,eAAA,GAAkB,MAAMC,2BAAA,CAAc,EAAA,EAAI,IAAI,YAAY,CAAA;AAGhE,IAAA,MAAM,EAAA,CAAqB,cAAc,CAAA,CACtC,KAAA,CAAM,EAAE,SAAA,EAAW,EAAA,EAAI,CAAA,CACvB,MAAA,EAAO;AAGV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,cAAA;AAAA,MACA,WAAA,CAAY,IAAI,CAAA,CAAA,MAAM;AAAA,QACpB,SAAA,EAAW,EAAA;AAAA,QACX,GAAA,EAAKC,sBAAA,CAAkB,CAAA,CAAE,GAAG;AAAA,OAC9B,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAEA,IAAA,OAAO;AAAA,MACL,eAAA,EAAiB;AAAA,KACnB;AAAA,EACF;AAAA,EAEA,MAAM,2BAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,MAAA,EAAQ,UAAA,EAAW,GAAI,OAAA;AAEnC,IAAA,MAAM,EAAA,CAAsB,eAAe,CAAA,CACxC,MAAA,CAAO;AAAA,MACN,MAAA;AAAA,MACA,WAAA,EAAa;AAAA,KACd,CAAA,CACA,KAAA,CAAM,WAAA,EAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,iBAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,EAAA,EAAI,KAAA,EAAM,GAAI,OAAA;AAEtB,IAAA,MAAM,GAAsB,eAAe,CAAA,CACxC,MAAA,CAAO,EAAE,OAAO,IAAA,CAAK,SAAA,CAAU,KAAA,IAAS,EAAE,CAAA,EAAG,CAAA,CAC7C,KAAA,CAAM,aAAa,EAAE,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,sBAAA,CACJ,OAAA,EACA,OAAA,EACuC;AACvC,IAAA,MAAM,IAAA,GAAO,OAAA;AACb,IAAA,MAAM,UAAA,GAAa,CAAC,OAAA,EAAS,QAAA,EAAU,IAAI,CAAA,CAAE,QAAA;AAAA,MAC3C,IAAA,CAAK,OAAO,MAAA,CAAO;AAAA,KACrB;AAEA,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,OAAA,CAAQ,eAAA,EAAgB;AAE9C,IAAA,MAAM,YAAA,GAAe,CACnB,EAAA,EACA,eAAA,KACG;AACH,MAAA,IAAI,GAAG,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AAC/C,QAAA,OAAO,GAAG,GAAA,CAAI,CAAA,kBAAA,CAAA,EAAsB,CAAC,CAAA,EAAG,eAAe,UAAU,CAAC,CAAA;AAAA,MACpE,WAAW,EAAA,CAAG,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AACpD,QAAA,OAAO,EAAA,CAAG,GAAA,CAAI,CAAA,iBAAA,EAAoB,eAAe,CAAA,OAAA,CAAS,CAAA;AAAA,MAC5D;AACA,MAAA,OAAO,EAAA,CAAG,GAAA,CAAI,CAAA,kBAAA,EAAqB,eAAe,CAAA,SAAA,CAAW,CAAA;AAAA,IAC/D,CAAA;AAKA,IAAA,MAAM,GAAA,GAAM,OAAO,EAAA,KAAgC;AACjD,MAAA,MAAMC,MAAAA,GAA6B,MAAM,EAAA,CAAG,eAAe,EACxD,MAAA,CAAO;AAAA,QACN,WAAA;AAAA,QACA,YAAA;AAAA,QACA,oBAAA;AAAA,QACA,aAAA;AAAA,QACA,OAAA;AAAA,QACA,QAAA;AAAA,QACA,cAAA;AAAA,QACA;AAAA,OACD,CAAA,CACA,KAAA,CAAM,kBAAkB,IAAA,EAAM,EAAA,CAAG,GAAG,GAAA,EAAK,EACzC,KAAA,CAAM,OAAA,CAAQ,gBAAgB,CAAA,CAC9B,OAAA,CAAQ,kBAAkB,KAAK,CAAA,CAC/B,OAAO,CAAA,EAAA,KAAM;AACZ,QAAA,IAAI,UAAA,EAAY;AACd,UAAA,EAAA,CAAG,SAAA,GAAY,UAAA,EAAW;AAAA,QAC5B;AAAA,MACF,CAAC,CAAA;AAEH,MAAA,IAAIA,MAAAA,CAAM,SAAS,CAAA,EAAG;AACpB,QAAA,MAAM,EAAA,CAAG,eAAe,CAAA,CACrB,OAAA;AAAA,UACC,YAAA;AAAA,UACAA,MAAAA,CAAM,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,UAAU;AAAA,UAE5B,MAAA,CAAO;AAAA,UACN,cAAA,EAAgB,YAAA,CAAa,EAAA,EAAI,QAAQ;AAAA,SAC1C,CAAA;AAAA,MACL;AAEA,MAAA,OAAOA,MAAAA;AAAA,IACT,CAAA;AAEA,IAAA,MAAM,KAAA,GACJ,IAAA,CAAK,aAAA,IAAiB,CAAC,UAAA,GACnB,MAAM,GAAA,CAAI,IAAI,CAAA,GACd,MAAM,IAAA,CAAK,WAAA,CAAY,GAAG,CAAA;AAEhC,IAAA,OAAO;AAAA,MACL,OAAO,KAAA,CAAM,GAAA;AAAA,QACX,CAAA,CAAA,MACG;AAAA,UACC,IAAI,CAAA,CAAE,SAAA;AAAA,UACN,WAAW,CAAA,CAAE,UAAA;AAAA,UACb,iBAAA,EAAmB,IAAA,CAAK,KAAA,CAAM,CAAA,CAAE,kBAAkB,CAAA;AAAA,UAClD,UAAA,EAAY,EAAE,WAAA,IAAe,EAAA;AAAA,UAC7B,YAAA,EAAcC,8BAAA,CAAoB,CAAA,CAAE,cAAc,CAAA;AAAA,UAClD,OAAO,CAAA,CAAE,KAAA,GAAQ,KAAK,KAAA,CAAM,CAAA,CAAE,KAAK,CAAA,GAAI,MAAA;AAAA,UACvC,QAAQ,CAAA,CAAE,MAAA;AAAA,UACV,aAAa,CAAA,CAAE;AAAA,SACjB;AAAA;AACJ,KACF;AAAA,EACF;AAAA,EAEA,MAAM,WAAA,CACJ,QAAA,EACA,OAAA,EAC4B;AAC5B,IAAA,MAAM,EAAA,GAAK,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,MAEC,OAAA,CAAQ,mBAAA,EAAqB,OAAA,CAAQ,UAAU,EAC/C,MAAA,EAAO;AAEV,IAAA,MAAM,UAAA,GAAa,KAAK,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,iBAAkB,CAAA,CAAE,OAAO,OAAO,CAAA;AAErE,IAAA,OAAO,EAAE,UAAA,EAAW;AAAA,EACtB;AAAA,EAEA,MAAM,YAAe,EAAA,EAAiD;AACpE,IAAA,IAAI;AACF,MAAA,IAAI,MAAA,GAAwB,KAAA,CAAA;AAE5B,MAAA,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,WAAA;AAAA,QAC1B,OAAM,EAAA,KAAM;AAGV,UAAA,MAAA,GAAS,MAAM,GAAG,EAAE,CAAA;AAAA,QACtB,CAAA;AAAA,QACA;AAAA;AAAA,UAEE,qBAAA,EAAuB;AAAA;AACzB,OACF;AAEA,MAAA,OAAO,MAAA;AAAA,IACT,SAAS,CAAA,EAAG;AACV,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,0BAAA,EAA6B,CAAC,CAAA,CAAE,CAAA;AAC1D,MAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA,IACtB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,sBAAA,CACZ,QAAA,EACA,OAAA,EAIe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AAGX,IAAA,MAAM,eAAA,GAAkB,IAAI,KAAA,EAAc;AAI1C,IAAA,KAAA,MAAW,EAAE,MAAA,EAAQ,WAAA,EAAY,IAAK,QAAQ,QAAA,EAAU;AACtD,MAAA,MAAM,SAAA,GAAYL,gCAAmB,MAAM,CAAA;AAC3C,MAAA,MAAM,IAAA,GAAOM,wBAAmB,MAAM,CAAA;AAEtC,MAAA,MAAM,OAAA,GAAU,MAAMC,+CAAA,CAAwB;AAAA,QAC5C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,OAAA,EAAS;AACX,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,QAAA,GAAW,MAAMC,+CAAA,CAAwB;AAAA,QAC7C,EAAA;AAAA,QACA,MAAA;AAAA,QACA,IAAA;AAAA,QACA,WAAA;AAAA,QACA,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,eAAA,CAAgB,KAAK,SAAS,CAAA;AAC9B,QAAA;AAAA,MACF;AAKA,MAAA,MAAM,cAAA,GAAiB,MAAMC,iDAAA,CAAyB;AAAA,QACpD,EAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,cAAA,EAAgB;AAClB,QAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,UAClB,CAAA,+BAAA,EAAkC,SAAS,CAAA,uBAAA,EAA0B,cAAc,iBAAiB,WAAW,CAAA;AAAA,SACjH;AACA,QAAA,IAAI,WAAA,EAAa;AACf,UAAA,MAAM,WAAA,GAAwD;AAAA,YAC5D,KAAA,EAAOC,iCAAA;AAAA,YACP,YAAA,EAAc;AAAA,cACZ,iBAAA,EAAmB,MAAA;AAAA,cACnB,SAAA;AAAA,cACA,cAAA,EAAgB,WAAA;AAAA,cAChB,mBAAA,EAAqB,cAAA;AAAA,cACrB,cAAA,EAAgBC,cAAA,CAAS,GAAA,EAAI,CAAE,KAAA;AAAM;AACvC,WACF;AACA,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,OAAA,CAAQ,WAAW,CAAA;AAAA,QAC/C;AAAA,MACF;AAAA,IACF;AAGA,IAAA,MAAM,EAAA,CAAgC,0BAA0B,CAAA,CAE7D,KAAA,CAAM,EAAE,iBAAA,EAAmB,OAAA,CAAQ,eAAA,EAAiB,CAAA,CAEpD,SAAA,CAAU,mBAAA,EAAqB,eAAe,EAC9C,MAAA,EAAO;AACV,IAAA,MAAM,EAAA,CAAG,WAAA;AAAA,MACP,0BAAA;AAAA,MACA,eAAA,CAAgB,IAAI,CAAA,SAAA,MAAc;AAAA,QAChC,mBAAmB,OAAA,CAAQ,eAAA;AAAA,QAC3B,iBAAA,EAAmB;AAAA,OACrB,CAAE,CAAA;AAAA,MACF;AAAA,KACF;AAAA,EACF;AACF;;;;"}

package/dist/database/DefaultProviderDatabase.cjs.js

@@ -22,23 +22,31 @@ class DefaultProviderDatabase {
   constructor(options) {
     this.options = options;
   }
+  /**
+   * Executes `fn` inside a database transaction, retrying automatically on
+   * deadlock (PostgreSQL error 40P01). Because the callback may be invoked
+   * more than once, it must not perform non-database side effects such as
+   * emitting events, mutating in-memory state, or calling external services.
+   */
   async transaction(fn) {
-    try {
-      let result = void 0;
-      await this.options.database.transaction(
-        async (tx) => {
-          result = await fn(tx);
-        },
-        {
-          // If we explicitly trigger a rollback, don't fail.
-          doNotRejectOnRollback: true
-        }
-      );
-      return result;
-    } catch (e) {
-      this.options.logger.debug(`Error during transaction, ${e}`);
-      throw conversion.rethrowError(e);
-    }
+    return util.retryOnDeadlock(async () => {
+      try {
+        let result = void 0;
+        await this.options.database.transaction(
+          async (tx) => {
+            result = await fn(tx);
+          },
+          {
+            // If we explicitly trigger a rollback, don't fail.
+            doNotRejectOnRollback: true
+          }
+        );
+        return result;
+      } catch (e) {
+        this.options.logger.debug(`Error during transaction, ${e}`);
+        throw conversion.rethrowError(e);
+      }
+    }, this.options.database);
   }
   async replaceUnprocessedEntities(txOpaque, options) {
     const tx = txOpaque;
@@ -134,6 +142,9 @@ class DefaultProviderDatabase {
             }
           }
         } catch (error) {
+          if (util.isDeadlockError(tx, error)) {
+            throw error;
+          }
           this.options.logger.error(
             `Failed to add '${entityRef}' from source '${options.sourceKey}', ${error}`
           );

package/dist/database/DefaultProviderDatabase.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DefaultProviderDatabase.cjs.js","sources":["../../src/database/DefaultProviderDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { stringifyEntityRef } from '@backstage/catalog-model';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { randomUUID as uuid } from 'node:crypto';\nimport { rethrowError } from './conversion';\nimport { deleteWithEagerPruningOfChildren } from './operations/provider/deleteWithEagerPruningOfChildren';\nimport { refreshByRefreshKeys } from './operations/provider/refreshByRefreshKeys';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { DbRefreshStateReferencesRow, DbRefreshStateRow } from './tables';\nimport {\n  ProviderDatabase,\n  RefreshByKeyOptions,\n  ReplaceUnprocessedEntitiesOptions,\n  Transaction,\n} from './types';\nimport { generateStableHash } from './util';\nimport {\n  LoggerService,\n  isDatabaseConflictError,\n} from '@backstage/backend-plugin-api';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProviderDatabase implements ProviderDatabase {\n  private readonly options: {\n    database: Knex;\n    logger: LoggerService;\n  };\n\n  constructor(options: { database: Knex; logger: LoggerService }) {\n    this.options = options;\n  }\n\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    try {\n      let result: T | undefined = undefined;\n      await this.options.database.transaction(\n        async tx => {\n          // We can't return here, as knex swallows the return type in case the\n          // transaction is rolled back:\n          // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n          result = await fn(tx);\n        },\n        {\n          // If we explicitly trigger a rollback, don't fail.\n          doNotRejectOnRollback: true,\n        },\n      );\n      return result!;\n    } catch (e) {\n      this.options.logger.debug(`Error during transaction, ${e}`);\n      throw rethrowError(e);\n    }\n  }\n\n  async replaceUnprocessedEntities(\n    txOpaque: Knex | Transaction,\n    options: ReplaceUnprocessedEntitiesOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex | Knex.Transaction;\n    const { toAdd, toUpsert, toRemove } = await this.createDelta(tx, options);\n\n    if (toRemove.length) {\n      const removedCount = await deleteWithEagerPruningOfChildren({\n        knex: tx,\n        entityRefs: toRemove,\n        sourceKey: options.sourceKey,\n      });\n      this.options.logger.debug(\n        `removed, ${removedCount} entities: ${JSON.stringify(toRemove)}`,\n      );\n    }\n\n    if (toAdd.length) {\n      // The reason for this chunking, rather than just massively batch\n      // inserting the entire payload, is that we fall back to the individual\n      // upsert mechanism below on conflicts. That path is massively slower than\n      // the fast batch path, so we don't want to end up accidentally having to\n      // for example item-by-item upsert tens of thousands of entities in a\n      // large initial delivery dump. The implication is that the size of these\n      // chunks needs to weigh the benefit of fast successful inserts, against\n      // the drawback of super slow but more rare fallbacks. There's quickly\n      // diminishing returns though with turning up this value way high.\n      for (const chunk of lodash.chunk(toAdd, 50)) {\n        try {\n          await tx.batchInsert(\n            'refresh_state',\n            chunk.map(item => ({\n              entity_id: uuid(),\n              entity_ref: stringifyEntityRef(item.deferred.entity),\n              unprocessed_entity: JSON.stringify(item.deferred.entity),\n              unprocessed_hash: item.hash,\n              errors: '',\n              location_key: item.deferred.locationKey,\n              next_update_at: tx.fn.now(),\n              last_discovery_at: tx.fn.now(),\n            })),\n            BATCH_SIZE,\n          );\n          await tx.batchInsert(\n            'refresh_state_references',\n            chunk.map(item => ({\n              source_key: options.sourceKey,\n              target_entity_ref: stringifyEntityRef(item.deferred.entity),\n            })),\n            BATCH_SIZE,\n          );\n        } catch (error) {\n          if (!isDatabaseConflictError(error)) {\n            throw error;\n          } else {\n            this.options.logger.debug(\n              `Fast insert path failed, falling back to slow path, ${error}`,\n            );\n            toUpsert.push(...chunk);\n          }\n        }\n      }\n    }\n\n    if (toUpsert.length) {\n      for (const {\n        deferred: { entity, locationKey },\n        hash,\n      } of toUpsert) {\n        const entityRef = stringifyEntityRef(entity);\n\n        try {\n          let ok = await updateUnprocessedEntity({\n            tx,\n            entity,\n            hash,\n            locationKey,\n          });\n          if (!ok) {\n            ok = await insertUnprocessedEntity({\n              tx,\n              entity,\n              hash,\n              locationKey,\n              logger: this.options.logger,\n            });\n          }\n          if (ok) {\n            await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n              .where('target_entity_ref', entityRef)\n              .delete();\n\n            await tx<DbRefreshStateReferencesRow>(\n              'refresh_state_references',\n            ).insert({\n              source_key: options.sourceKey,\n              target_entity_ref: entityRef,\n            });\n          } else {\n            await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n              .where('target_entity_ref', entityRef)\n              .andWhere({ source_key: options.sourceKey })\n              .delete();\n\n            const conflictingKey = await checkLocationKeyConflict({\n              tx,\n              entityRef,\n              locationKey,\n            });\n            if (conflictingKey) {\n              this.options.logger.warn(\n                `Source ${options.sourceKey} detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n              );\n            }\n          }\n        } catch (error) {\n          this.options.logger.error(\n            `Failed to add '${entityRef}' from source '${options.sourceKey}', ${error}`,\n          );\n        }\n      }\n    }\n  }\n\n  async listReferenceSourceKeys(txOpaque: Transaction): Promise<string[]> {\n    const tx = txOpaque as Knex | Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .distinct('source_key')\n      .whereNotNull('source_key');\n\n    return rows\n      .map(row => row.source_key)\n      .filter((key): key is string => !!key);\n  }\n\n  async refreshByRefreshKeys(\n    txOpaque: Transaction,\n    options: RefreshByKeyOptions,\n  ) {\n    const tx = txOpaque as Knex.Transaction;\n    await refreshByRefreshKeys({ tx, keys: options.keys });\n  }\n\n  private async createDelta(\n    tx: Knex | Knex.Transaction,\n    options: ReplaceUnprocessedEntitiesOptions,\n  ): Promise<{\n    toAdd: { deferred: DeferredEntity; hash: string }[];\n    toUpsert: { deferred: DeferredEntity; hash: string }[];\n    toRemove: string[];\n  }> {\n    if (options.type === 'delta') {\n      const toAdd = new Array<{ deferred: DeferredEntity; hash: string }>();\n      const toUpsert = new Array<{ deferred: DeferredEntity; hash: string }>();\n      const toRemove = options.removed.map(e => e.entityRef);\n\n      for (const chunk of lodash.chunk(options.added, 1000)) {\n        const entityRefs = chunk.map(e => stringifyEntityRef(e.entity));\n        const rows = await tx<DbRefreshStateRow>('refresh_state')\n          .select(['entity_ref', 'unprocessed_hash', 'location_key'])\n          .whereIn('entity_ref', entityRefs);\n        const oldStates = new Map(\n          rows.map(row => [\n            row.entity_ref,\n            {\n              unprocessed_hash: row.unprocessed_hash,\n              location_key: row.location_key,\n            },\n          ]),\n        );\n\n        chunk.forEach((deferred, i) => {\n          const entityRef = entityRefs[i];\n          const newHash = generateStableHash(deferred.entity);\n          const oldState = oldStates.get(entityRef);\n          if (oldState === undefined) {\n            // Add any entity that does not exist in the database\n            toAdd.push({ deferred, hash: newHash });\n          } else if (\n            (deferred.locationKey ?? null) !== (oldState.location_key ?? null)\n          ) {\n            // Remove and then re-add any entity that exists, but with a different location key\n            toRemove.push(entityRef);\n            toAdd.push({ deferred, hash: newHash });\n          } else if (newHash !== oldState.unprocessed_hash) {\n            // Entities with modifications should be pushed through too\n            toUpsert.push({ deferred, hash: newHash });\n          }\n        });\n      }\n\n      return { toAdd, toUpsert, toRemove };\n    }\n\n    // Grab all of the existing references from the same source, and their locationKeys as well\n    const oldRefs = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .leftJoin<DbRefreshStateRow>('refresh_state', {\n        target_entity_ref: 'entity_ref',\n      })\n      .where({ source_key: options.sourceKey })\n      .select({\n        target_entity_ref: 'refresh_state_references.target_entity_ref',\n        location_key: 'refresh_state.location_key',\n        unprocessed_hash: 'refresh_state.unprocessed_hash',\n      });\n\n    const items = options.items.map(deferred => ({\n      deferred,\n      ref: stringifyEntityRef(deferred.entity),\n      hash: generateStableHash(deferred.entity),\n    }));\n\n    const oldRefsSet = new Map(\n      oldRefs.map(r => [\n        r.target_entity_ref,\n        {\n          locationKey: r.location_key,\n          oldEntityHash: r.unprocessed_hash,\n        },\n      ]),\n    );\n    const newRefsSet = new Set(items.map(item => item.ref));\n\n    const toAdd = new Array<{ deferred: DeferredEntity; hash: string }>();\n    const toUpsert = new Array<{ deferred: DeferredEntity; hash: string }>();\n    const toRemove = oldRefs\n      .map(row => row.target_entity_ref)\n      .filter(ref => !newRefsSet.has(ref));\n\n    for (const item of items) {\n      const oldRef = oldRefsSet.get(item.ref);\n      const upsertItem = { deferred: item.deferred, hash: item.hash };\n      if (!oldRef) {\n        // Add any entity that does not exist in the database\n        toAdd.push(upsertItem);\n      } else if (\n        (oldRef.locationKey ?? undefined) !==\n        (item.deferred.locationKey ?? undefined)\n      ) {\n        // Remove and then re-add any entity that exists, but with a different location key\n        toRemove.push(item.ref);\n        toAdd.push(upsertItem);\n      } else if (oldRef.oldEntityHash !== item.hash) {\n        // Entities with modifications should be pushed through too\n        toUpsert.push(upsertItem);\n      }\n    }\n\n    return { toAdd, toUpsert, toRemove };\n  }\n}\n"],"names":["rethrowError","deleteWithEagerPruningOfChildren","lodash","uuid","stringifyEntityRef","isDatabaseConflictError","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","refreshByRefreshKeys","toAdd","toUpsert","toRemove","generateStableHash"],"mappings":";;;;;;;;;;;;;;;;;;AA4CA,MAAM,UAAA,GAAa,EAAA;AAEZ,MAAM,uBAAA,CAAoD;AAAA,EAC9C,OAAA;AAAA,EAKjB,YAAY,OAAA,EAAoD;AAC9D,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA;AAAA,EACjB;AAAA,EAEA,MAAM,YAAe,EAAA,EAAiD;AACpE,IAAA,IAAI;AACF,MAAA,IAAI,MAAA,GAAwB,KAAA,CAAA;AAC5B,MAAA,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,WAAA;AAAA,QAC1B,OAAM,EAAA,KAAM;AAIV,UAAA,MAAA,GAAS,MAAM,GAAG,EAAE,CAAA;AAAA,QACtB,CAAA;AAAA,QACA;AAAA;AAAA,UAEE,qBAAA,EAAuB;AAAA;AACzB,OACF;AACA,MAAA,OAAO,MAAA;AAAA,IACT,SAAS,CAAA,EAAG;AACV,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,0BAAA,EAA6B,CAAC,CAAA,CAAE,CAAA;AAC1D,MAAA,MAAMA,wBAAa,CAAC,CAAA;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,MAAM,0BAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,OAAO,QAAA,EAAU,QAAA,KAAa,MAAM,IAAA,CAAK,WAAA,CAAY,EAAA,EAAI,OAAO,CAAA;AAExE,IAAA,IAAI,SAAS,MAAA,EAAQ;AACnB,MAAA,MAAM,YAAA,GAAe,MAAMC,iEAAA,CAAiC;AAAA,QAC1D,IAAA,EAAM,EAAA;AAAA,QACN,UAAA,EAAY,QAAA;AAAA,QACZ,WAAW,OAAA,CAAQ;AAAA,OACpB,CAAA;AACD,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,QAClB,YAAY,YAAY,CAAA,WAAA,EAAc,IAAA,CAAK,SAAA,CAAU,QAAQ,CAAC,CAAA;AAAA,OAChE;AAAA,IACF;AAEA,IAAA,IAAI,MAAM,MAAA,EAAQ;AAUhB,MAAA,KAAA,MAAW,KAAA,IAASC,uBAAA,CAAO,KAAA,CAAM,KAAA,EAAO,EAAE,CAAA,EAAG;AAC3C,QAAA,IAAI;AACF,UAAA,MAAM,EAAA,CAAG,WAAA;AAAA,YACP,eAAA;AAAA,YACA,KAAA,CAAM,IAAI,CAAA,IAAA,MAAS;AAAA,cACjB,WAAWC,sBAAA,EAAK;AAAA,cAChB,UAAA,EAAYC,+BAAA,CAAmB,IAAA,CAAK,QAAA,CAAS,MAAM,CAAA;AAAA,cACnD,kBAAA,EAAoB,IAAA,CAAK,SAAA,CAAU,IAAA,CAAK,SAAS,MAAM,CAAA;AAAA,cACvD,kBAAkB,IAAA,CAAK,IAAA;AAAA,cACvB,MAAA,EAAQ,EAAA;AAAA,cACR,YAAA,EAAc,KAAK,QAAA,CAAS,WAAA;AAAA,cAC5B,cAAA,EAAgB,EAAA,CAAG,EAAA,CAAG,GAAA,EAAI;AAAA,cAC1B,iBAAA,EAAmB,EAAA,CAAG,EAAA,CAAG,GAAA;AAAI,aAC/B,CAAE,CAAA;AAAA,YACF;AAAA,WACF;AACA,UAAA,MAAM,EAAA,CAAG,WAAA;AAAA,YACP,0BAAA;AAAA,YACA,KAAA,CAAM,IAAI,CAAA,IAAA,MAAS;AAAA,cACjB,YAAY,OAAA,CAAQ,SAAA;AAAA,cACpB,iBAAA,EAAmBA,+BAAA,CAAmB,IAAA,CAAK,QAAA,CAAS,MAAM;AAAA,aAC5D,CAAE,CAAA;AAAA,YACF;AAAA,WACF;AAAA,QACF,SAAS,KAAA,EAAO;AACd,UAAA,IAAI,CAACC,wCAAA,CAAwB,KAAK,CAAA,EAAG;AACnC,YAAA,MAAM,KAAA;AAAA,UACR,CAAA,MAAO;AACL,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,cAClB,uDAAuD,KAAK,CAAA;AAAA,aAC9D;AACA,YAAA,QAAA,CAAS,IAAA,CAAK,GAAG,KAAK,CAAA;AAAA,UACxB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,IAAA,IAAI,SAAS,MAAA,EAAQ;AACnB,MAAA,KAAA,MAAW;AAAA,QACT,QAAA,EAAU,EAAE,MAAA,EAAQ,WAAA,EAAY;AAAA,QAChC;AAAA,WACG,QAAA,EAAU;AACb,QAAA,MAAM,SAAA,GAAYD,gCAAmB,MAAM,CAAA;AAE3C,QAAA,IAAI;AACF,UAAA,IAAI,EAAA,GAAK,MAAME,+CAAA,CAAwB;AAAA,YACrC,EAAA;AAAA,YACA,MAAA;AAAA,YACA,IAAA;AAAA,YACA;AAAA,WACD,CAAA;AACD,UAAA,IAAI,CAAC,EAAA,EAAI;AACP,YAAA,EAAA,GAAK,MAAMC,+CAAA,CAAwB;AAAA,cACjC,EAAA;AAAA,cACA,MAAA;AAAA,cACA,IAAA;AAAA,cACA,WAAA;AAAA,cACA,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,aACtB,CAAA;AAAA,UACH;AACA,UAAA,IAAI,EAAA,EAAI;AACN,YAAA,MAAM,GAAgC,0BAA0B,CAAA,CAC7D,MAAM,mBAAA,EAAqB,SAAS,EACpC,MAAA,EAAO;AAEV,YAAA,MAAM,EAAA;AAAA,cACJ;AAAA,cACA,MAAA,CAAO;AAAA,cACP,YAAY,OAAA,CAAQ,SAAA;AAAA,cACpB,iBAAA,EAAmB;AAAA,aACpB,CAAA;AAAA,UACH,CAAA,MAAO;AACL,YAAA,MAAM,EAAA,CAAgC,0BAA0B,CAAA,CAC7D,KAAA,CAAM,qBAAqB,SAAS,CAAA,CACpC,QAAA,CAAS,EAAE,UAAA,EAAY,OAAA,CAAQ,SAAA,EAAW,EAC1C,MAAA,EAAO;AAEV,YAAA,MAAM,cAAA,GAAiB,MAAMC,iDAAA,CAAyB;AAAA,cACpD,EAAA;AAAA,cACA,SAAA;AAAA,cACA;AAAA,aACD,CAAA;AACD,YAAA,IAAI,cAAA,EAAgB;AAClB,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,gBAClB,CAAA,OAAA,EAAU,QAAQ,SAAS,CAAA,gCAAA,EAAmC,SAAS,CAAA,uBAAA,EAA0B,cAAc,iBAAiB,WAAW,CAAA;AAAA,eAC7I;AAAA,YACF;AAAA,UACF;AAAA,QACF,SAAS,KAAA,EAAO;AACd,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,YAClB,kBAAkB,SAAS,CAAA,eAAA,EAAkB,OAAA,CAAQ,SAAS,MAAM,KAAK,CAAA;AAAA,WAC3E;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,wBAAwB,QAAA,EAA0C;AACtE,IAAA,MAAM,EAAA,GAAK,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,KACF,CACG,QAAA,CAAS,YAAY,CAAA,CACrB,aAAa,YAAY,CAAA;AAE5B,IAAA,OAAO,IAAA,CACJ,GAAA,CAAI,CAAA,GAAA,KAAO,GAAA,CAAI,UAAU,CAAA,CACzB,MAAA,CAAO,CAAC,GAAA,KAAuB,CAAC,CAAC,GAAG,CAAA;AAAA,EACzC;AAAA,EAEA,MAAM,oBAAA,CACJ,QAAA,EACA,OAAA,EACA;AACA,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAMC,0CAAqB,EAAE,EAAA,EAAI,IAAA,EAAM,OAAA,CAAQ,MAAM,CAAA;AAAA,EACvD;AAAA,EAEA,MAAc,WAAA,CACZ,EAAA,EACA,OAAA,EAKC;AACD,IAAA,IAAI,OAAA,CAAQ,SAAS,OAAA,EAAS;AAC5B,MAAA,MAAMC,MAAAA,GAAQ,IAAI,KAAA,EAAkD;AACpE,MAAA,MAAMC,SAAAA,GAAW,IAAI,KAAA,EAAkD;AACvE,MAAA,MAAMC,YAAW,OAAA,CAAQ,OAAA,CAAQ,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,SAAS,CAAA;AAErD,MAAA,KAAA,MAAW,SAASV,uBAAA,CAAO,KAAA,CAAM,OAAA,CAAQ,KAAA,EAAO,GAAI,CAAA,EAAG;AACrD,QAAA,MAAM,aAAa,KAAA,CAAM,GAAA,CAAI,OAAKE,+BAAA,CAAmB,CAAA,CAAE,MAAM,CAAC,CAAA;AAC9D,QAAA,MAAM,IAAA,GAAO,MAAM,EAAA,CAAsB,eAAe,EACrD,MAAA,CAAO,CAAC,YAAA,EAAc,kBAAA,EAAoB,cAAc,CAAC,CAAA,CACzD,OAAA,CAAQ,cAAc,UAAU,CAAA;AACnC,QAAA,MAAM,YAAY,IAAI,GAAA;AAAA,UACpB,IAAA,CAAK,IAAI,CAAA,GAAA,KAAO;AAAA,YACd,GAAA,CAAI,UAAA;AAAA,YACJ;AAAA,cACE,kBAAkB,GAAA,CAAI,gBAAA;AAAA,cACtB,cAAc,GAAA,CAAI;AAAA;AACpB,WACD;AAAA,SACH;AAEA,QAAA,KAAA,CAAM,OAAA,CAAQ,CAAC,QAAA,EAAU,CAAA,KAAM;AAC7B,UAAA,MAAM,SAAA,GAAY,WAAW,CAAC,CAAA;AAC9B,UAAA,MAAM,OAAA,GAAUS,uBAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAClD,UAAA,MAAM,QAAA,GAAW,SAAA,CAAU,GAAA,CAAI,SAAS,CAAA;AACxC,UAAA,IAAI,aAAa,MAAA,EAAW;AAE1B,YAAAH,OAAM,IAAA,CAAK,EAAE,QAAA,EAAU,IAAA,EAAM,SAAS,CAAA;AAAA,UACxC,YACG,QAAA,CAAS,WAAA,IAAe,IAAA,OAAW,QAAA,CAAS,gBAAgB,IAAA,CAAA,EAC7D;AAEA,YAAAE,SAAAA,CAAS,KAAK,SAAS,CAAA;AACvB,YAAAF,OAAM,IAAA,CAAK,EAAE,QAAA,EAAU,IAAA,EAAM,SAAS,CAAA;AAAA,UACxC,CAAA,MAAA,IAAW,OAAA,KAAY,QAAA,CAAS,gBAAA,EAAkB;AAEhD,YAAAC,UAAS,IAAA,CAAK,EAAE,QAAA,EAAU,IAAA,EAAM,SAAS,CAAA;AAAA,UAC3C;AAAA,QACF,CAAC,CAAA;AAAA,MACH;AAEA,MAAA,OAAO,EAAE,KAAA,EAAAD,MAAAA,EAAO,QAAA,EAAAC,SAAAA,EAAU,UAAAC,SAAAA,EAAS;AAAA,IACrC;AAGA,IAAA,MAAM,UAAU,MAAM,EAAA;AAAA,MACpB;AAAA,KACF,CACG,SAA4B,eAAA,EAAiB;AAAA,MAC5C,iBAAA,EAAmB;AAAA,KACpB,EACA,KAAA,CAAM,EAAE,YAAY,OAAA,CAAQ,SAAA,EAAW,CAAA,CACvC,MAAA,CAAO;AAAA,MACN,iBAAA,EAAmB,4CAAA;AAAA,MACnB,YAAA,EAAc,4BAAA;AAAA,MACd,gBAAA,EAAkB;AAAA,KACnB,CAAA;AAEH,IAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,KAAA,CAAM,GAAA,CAAI,CAAA,QAAA,MAAa;AAAA,MAC3C,QAAA;AAAA,MACA,GAAA,EAAKR,+BAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAAA,MACvC,IAAA,EAAMS,uBAAA,CAAmB,QAAA,CAAS,MAAM;AAAA,KAC1C,CAAE,CAAA;AAEF,IAAA,MAAM,aAAa,IAAI,GAAA;AAAA,MACrB,OAAA,CAAQ,IAAI,CAAA,CAAA,KAAK;AAAA,QACf,CAAA,CAAE,iBAAA;AAAA,QACF;AAAA,UACE,aAAa,CAAA,CAAE,YAAA;AAAA,UACf,eAAe,CAAA,CAAE;AAAA;AACnB,OACD;AAAA,KACH;AACA,IAAA,MAAM,UAAA,GAAa,IAAI,GAAA,CAAI,KAAA,CAAM,IAAI,CAAA,IAAA,KAAQ,IAAA,CAAK,GAAG,CAAC,CAAA;AAEtD,IAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAkD;AACpE,IAAA,MAAM,QAAA,GAAW,IAAI,KAAA,EAAkD;AACvE,IAAA,MAAM,QAAA,GAAW,OAAA,CACd,GAAA,CAAI,CAAA,GAAA,KAAO,GAAA,CAAI,iBAAiB,CAAA,CAChC,MAAA,CAAO,CAAA,GAAA,KAAO,CAAC,UAAA,CAAW,GAAA,CAAI,GAAG,CAAC,CAAA;AAErC,IAAA,KAAA,MAAW,QAAQ,KAAA,EAAO;AACxB,MAAA,MAAM,MAAA,GAAS,UAAA,CAAW,GAAA,CAAI,IAAA,CAAK,GAAG,CAAA;AACtC,MAAA,MAAM,aAAa,EAAE,QAAA,EAAU,KAAK,QAAA,EAAU,IAAA,EAAM,KAAK,IAAA,EAAK;AAC9D,MAAA,IAAI,CAAC,MAAA,EAAQ;AAEX,QAAA,KAAA,CAAM,KAAK,UAAU,CAAA;AAAA,MACvB,YACG,MAAA,CAAO,WAAA,IAAe,aACtB,IAAA,CAAK,QAAA,CAAS,eAAe,MAAA,CAAA,EAC9B;AAEA,QAAA,QAAA,CAAS,IAAA,CAAK,KAAK,GAAG,CAAA;AACtB,QAAA,KAAA,CAAM,KAAK,UAAU,CAAA;AAAA,MACvB,CAAA,MAAA,IAAW,MAAA,CAAO,aAAA,KAAkB,IAAA,CAAK,IAAA,EAAM;AAE7C,QAAA,QAAA,CAAS,KAAK,UAAU,CAAA;AAAA,MAC1B;AAAA,IACF;AAEA,IAAA,OAAO,EAAE,KAAA,EAAO,QAAA,EAAU,QAAA,EAAS;AAAA,EACrC;AACF;;;;"}
+{"version":3,"file":"DefaultProviderDatabase.cjs.js","sources":["../../src/database/DefaultProviderDatabase.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { stringifyEntityRef } from '@backstage/catalog-model';\nimport { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Knex } from 'knex';\nimport lodash from 'lodash';\nimport { randomUUID as uuid } from 'node:crypto';\nimport { rethrowError } from './conversion';\nimport { deleteWithEagerPruningOfChildren } from './operations/provider/deleteWithEagerPruningOfChildren';\nimport { refreshByRefreshKeys } from './operations/provider/refreshByRefreshKeys';\nimport { checkLocationKeyConflict } from './operations/refreshState/checkLocationKeyConflict';\nimport { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity';\nimport { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity';\nimport { DbRefreshStateReferencesRow, DbRefreshStateRow } from './tables';\nimport {\n  ProviderDatabase,\n  RefreshByKeyOptions,\n  ReplaceUnprocessedEntitiesOptions,\n  Transaction,\n} from './types';\nimport { generateStableHash, isDeadlockError, retryOnDeadlock } from './util';\nimport {\n  LoggerService,\n  isDatabaseConflictError,\n} from '@backstage/backend-plugin-api';\n\n// The number of items that are sent per batch to the database layer, when\n// doing .batchInsert calls to knex. This needs to be low enough to not cause\n// errors in the underlying engine due to exceeding query limits, but large\n// enough to get the speed benefits.\nconst BATCH_SIZE = 50;\n\nexport class DefaultProviderDatabase implements ProviderDatabase {\n  private readonly options: {\n    database: Knex;\n    logger: LoggerService;\n  };\n\n  constructor(options: { database: Knex; logger: LoggerService }) {\n    this.options = options;\n  }\n\n  /**\n   * Executes `fn` inside a database transaction, retrying automatically on\n   * deadlock (PostgreSQL error 40P01). Because the callback may be invoked\n   * more than once, it must not perform non-database side effects such as\n   * emitting events, mutating in-memory state, or calling external services.\n   */\n  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {\n    return retryOnDeadlock(async () => {\n      try {\n        let result: T | undefined = undefined;\n        await this.options.database.transaction(\n          async tx => {\n            // We can't return here, as knex swallows the return type in case the\n            // transaction is rolled back:\n            // https://github.com/knex/knex/blob/e37aeaa31c8ef9c1b07d2e4d3ec6607e557d800d/lib/transaction.js#L136\n            result = await fn(tx);\n          },\n          {\n            // If we explicitly trigger a rollback, don't fail.\n            doNotRejectOnRollback: true,\n          },\n        );\n        return result!;\n      } catch (e) {\n        this.options.logger.debug(`Error during transaction, ${e}`);\n        throw rethrowError(e);\n      }\n    }, this.options.database);\n  }\n\n  async replaceUnprocessedEntities(\n    txOpaque: Knex | Transaction,\n    options: ReplaceUnprocessedEntitiesOptions,\n  ): Promise<void> {\n    const tx = txOpaque as Knex | Knex.Transaction;\n    const { toAdd, toUpsert, toRemove } = await this.createDelta(tx, options);\n\n    if (toRemove.length) {\n      const removedCount = await deleteWithEagerPruningOfChildren({\n        knex: tx,\n        entityRefs: toRemove,\n        sourceKey: options.sourceKey,\n      });\n      this.options.logger.debug(\n        `removed, ${removedCount} entities: ${JSON.stringify(toRemove)}`,\n      );\n    }\n\n    if (toAdd.length) {\n      // The reason for this chunking, rather than just massively batch\n      // inserting the entire payload, is that we fall back to the individual\n      // upsert mechanism below on conflicts. That path is massively slower than\n      // the fast batch path, so we don't want to end up accidentally having to\n      // for example item-by-item upsert tens of thousands of entities in a\n      // large initial delivery dump. The implication is that the size of these\n      // chunks needs to weigh the benefit of fast successful inserts, against\n      // the drawback of super slow but more rare fallbacks. There's quickly\n      // diminishing returns though with turning up this value way high.\n      for (const chunk of lodash.chunk(toAdd, 50)) {\n        try {\n          await tx.batchInsert(\n            'refresh_state',\n            chunk.map(item => ({\n              entity_id: uuid(),\n              entity_ref: stringifyEntityRef(item.deferred.entity),\n              unprocessed_entity: JSON.stringify(item.deferred.entity),\n              unprocessed_hash: item.hash,\n              errors: '',\n              location_key: item.deferred.locationKey,\n              next_update_at: tx.fn.now(),\n              last_discovery_at: tx.fn.now(),\n            })),\n            BATCH_SIZE,\n          );\n          await tx.batchInsert(\n            'refresh_state_references',\n            chunk.map(item => ({\n              source_key: options.sourceKey,\n              target_entity_ref: stringifyEntityRef(item.deferred.entity),\n            })),\n            BATCH_SIZE,\n          );\n        } catch (error) {\n          if (!isDatabaseConflictError(error)) {\n            throw error;\n          } else {\n            this.options.logger.debug(\n              `Fast insert path failed, falling back to slow path, ${error}`,\n            );\n            toUpsert.push(...chunk);\n          }\n        }\n      }\n    }\n\n    if (toUpsert.length) {\n      for (const {\n        deferred: { entity, locationKey },\n        hash,\n      } of toUpsert) {\n        const entityRef = stringifyEntityRef(entity);\n\n        try {\n          let ok = await updateUnprocessedEntity({\n            tx,\n            entity,\n            hash,\n            locationKey,\n          });\n          if (!ok) {\n            ok = await insertUnprocessedEntity({\n              tx,\n              entity,\n              hash,\n              locationKey,\n              logger: this.options.logger,\n            });\n          }\n          if (ok) {\n            await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n              .where('target_entity_ref', entityRef)\n              .delete();\n\n            await tx<DbRefreshStateReferencesRow>(\n              'refresh_state_references',\n            ).insert({\n              source_key: options.sourceKey,\n              target_entity_ref: entityRef,\n            });\n          } else {\n            await tx<DbRefreshStateReferencesRow>('refresh_state_references')\n              .where('target_entity_ref', entityRef)\n              .andWhere({ source_key: options.sourceKey })\n              .delete();\n\n            const conflictingKey = await checkLocationKeyConflict({\n              tx,\n              entityRef,\n              locationKey,\n            });\n            if (conflictingKey) {\n              this.options.logger.warn(\n                `Source ${options.sourceKey} detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`,\n              );\n            }\n          }\n        } catch (error) {\n          if (isDeadlockError(tx, error)) {\n            throw error;\n          }\n          this.options.logger.error(\n            `Failed to add '${entityRef}' from source '${options.sourceKey}', ${error}`,\n          );\n        }\n      }\n    }\n  }\n\n  async listReferenceSourceKeys(txOpaque: Transaction): Promise<string[]> {\n    const tx = txOpaque as Knex | Knex.Transaction;\n\n    const rows = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .distinct('source_key')\n      .whereNotNull('source_key');\n\n    return rows\n      .map(row => row.source_key)\n      .filter((key): key is string => !!key);\n  }\n\n  async refreshByRefreshKeys(\n    txOpaque: Transaction,\n    options: RefreshByKeyOptions,\n  ) {\n    const tx = txOpaque as Knex.Transaction;\n    await refreshByRefreshKeys({ tx, keys: options.keys });\n  }\n\n  private async createDelta(\n    tx: Knex | Knex.Transaction,\n    options: ReplaceUnprocessedEntitiesOptions,\n  ): Promise<{\n    toAdd: { deferred: DeferredEntity; hash: string }[];\n    toUpsert: { deferred: DeferredEntity; hash: string }[];\n    toRemove: string[];\n  }> {\n    if (options.type === 'delta') {\n      const toAdd = new Array<{ deferred: DeferredEntity; hash: string }>();\n      const toUpsert = new Array<{ deferred: DeferredEntity; hash: string }>();\n      const toRemove = options.removed.map(e => e.entityRef);\n\n      for (const chunk of lodash.chunk(options.added, 1000)) {\n        const entityRefs = chunk.map(e => stringifyEntityRef(e.entity));\n        const rows = await tx<DbRefreshStateRow>('refresh_state')\n          .select(['entity_ref', 'unprocessed_hash', 'location_key'])\n          .whereIn('entity_ref', entityRefs);\n        const oldStates = new Map(\n          rows.map(row => [\n            row.entity_ref,\n            {\n              unprocessed_hash: row.unprocessed_hash,\n              location_key: row.location_key,\n            },\n          ]),\n        );\n\n        chunk.forEach((deferred, i) => {\n          const entityRef = entityRefs[i];\n          const newHash = generateStableHash(deferred.entity);\n          const oldState = oldStates.get(entityRef);\n          if (oldState === undefined) {\n            // Add any entity that does not exist in the database\n            toAdd.push({ deferred, hash: newHash });\n          } else if (\n            (deferred.locationKey ?? null) !== (oldState.location_key ?? null)\n          ) {\n            // Remove and then re-add any entity that exists, but with a different location key\n            toRemove.push(entityRef);\n            toAdd.push({ deferred, hash: newHash });\n          } else if (newHash !== oldState.unprocessed_hash) {\n            // Entities with modifications should be pushed through too\n            toUpsert.push({ deferred, hash: newHash });\n          }\n        });\n      }\n\n      return { toAdd, toUpsert, toRemove };\n    }\n\n    // Grab all of the existing references from the same source, and their locationKeys as well\n    const oldRefs = await tx<DbRefreshStateReferencesRow>(\n      'refresh_state_references',\n    )\n      .leftJoin<DbRefreshStateRow>('refresh_state', {\n        target_entity_ref: 'entity_ref',\n      })\n      .where({ source_key: options.sourceKey })\n      .select({\n        target_entity_ref: 'refresh_state_references.target_entity_ref',\n        location_key: 'refresh_state.location_key',\n        unprocessed_hash: 'refresh_state.unprocessed_hash',\n      });\n\n    const items = options.items.map(deferred => ({\n      deferred,\n      ref: stringifyEntityRef(deferred.entity),\n      hash: generateStableHash(deferred.entity),\n    }));\n\n    const oldRefsSet = new Map(\n      oldRefs.map(r => [\n        r.target_entity_ref,\n        {\n          locationKey: r.location_key,\n          oldEntityHash: r.unprocessed_hash,\n        },\n      ]),\n    );\n    const newRefsSet = new Set(items.map(item => item.ref));\n\n    const toAdd = new Array<{ deferred: DeferredEntity; hash: string }>();\n    const toUpsert = new Array<{ deferred: DeferredEntity; hash: string }>();\n    const toRemove = oldRefs\n      .map(row => row.target_entity_ref)\n      .filter(ref => !newRefsSet.has(ref));\n\n    for (const item of items) {\n      const oldRef = oldRefsSet.get(item.ref);\n      const upsertItem = { deferred: item.deferred, hash: item.hash };\n      if (!oldRef) {\n        // Add any entity that does not exist in the database\n        toAdd.push(upsertItem);\n      } else if (\n        (oldRef.locationKey ?? undefined) !==\n        (item.deferred.locationKey ?? undefined)\n      ) {\n        // Remove and then re-add any entity that exists, but with a different location key\n        toRemove.push(item.ref);\n        toAdd.push(upsertItem);\n      } else if (oldRef.oldEntityHash !== item.hash) {\n        // Entities with modifications should be pushed through too\n        toUpsert.push(upsertItem);\n      }\n    }\n\n    return { toAdd, toUpsert, toRemove };\n  }\n}\n"],"names":["retryOnDeadlock","rethrowError","deleteWithEagerPruningOfChildren","lodash","uuid","stringifyEntityRef","isDatabaseConflictError","updateUnprocessedEntity","insertUnprocessedEntity","checkLocationKeyConflict","isDeadlockError","refreshByRefreshKeys","toAdd","toUpsert","toRemove","generateStableHash"],"mappings":";;;;;;;;;;;;;;;;;;AA4CA,MAAM,UAAA,GAAa,EAAA;AAEZ,MAAM,uBAAA,CAAoD;AAAA,EAC9C,OAAA;AAAA,EAKjB,YAAY,OAAA,EAAoD;AAC9D,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA;AAAA,EACjB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,YAAe,EAAA,EAAiD;AACpE,IAAA,OAAOA,qBAAgB,YAAY;AACjC,MAAA,IAAI;AACF,QAAA,IAAI,MAAA,GAAwB,KAAA,CAAA;AAC5B,QAAA,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,WAAA;AAAA,UAC1B,OAAM,EAAA,KAAM;AAIV,YAAA,MAAA,GAAS,MAAM,GAAG,EAAE,CAAA;AAAA,UACtB,CAAA;AAAA,UACA;AAAA;AAAA,YAEE,qBAAA,EAAuB;AAAA;AACzB,SACF;AACA,QAAA,OAAO,MAAA;AAAA,MACT,SAAS,CAAA,EAAG;AACV,QAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,0BAAA,EAA6B,CAAC,CAAA,CAAE,CAAA;AAC1D,QAAA,MAAMC,wBAAa,CAAC,CAAA;AAAA,MACtB;AAAA,IACF,CAAA,EAAG,IAAA,CAAK,OAAA,CAAQ,QAAQ,CAAA;AAAA,EAC1B;AAAA,EAEA,MAAM,0BAAA,CACJ,QAAA,EACA,OAAA,EACe;AACf,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAM,EAAE,OAAO,QAAA,EAAU,QAAA,KAAa,MAAM,IAAA,CAAK,WAAA,CAAY,EAAA,EAAI,OAAO,CAAA;AAExE,IAAA,IAAI,SAAS,MAAA,EAAQ;AACnB,MAAA,MAAM,YAAA,GAAe,MAAMC,iEAAA,CAAiC;AAAA,QAC1D,IAAA,EAAM,EAAA;AAAA,QACN,UAAA,EAAY,QAAA;AAAA,QACZ,WAAW,OAAA,CAAQ;AAAA,OACpB,CAAA;AACD,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,QAClB,YAAY,YAAY,CAAA,WAAA,EAAc,IAAA,CAAK,SAAA,CAAU,QAAQ,CAAC,CAAA;AAAA,OAChE;AAAA,IACF;AAEA,IAAA,IAAI,MAAM,MAAA,EAAQ;AAUhB,MAAA,KAAA,MAAW,KAAA,IAASC,uBAAA,CAAO,KAAA,CAAM,KAAA,EAAO,EAAE,CAAA,EAAG;AAC3C,QAAA,IAAI;AACF,UAAA,MAAM,EAAA,CAAG,WAAA;AAAA,YACP,eAAA;AAAA,YACA,KAAA,CAAM,IAAI,CAAA,IAAA,MAAS;AAAA,cACjB,WAAWC,sBAAA,EAAK;AAAA,cAChB,UAAA,EAAYC,+BAAA,CAAmB,IAAA,CAAK,QAAA,CAAS,MAAM,CAAA;AAAA,cACnD,kBAAA,EAAoB,IAAA,CAAK,SAAA,CAAU,IAAA,CAAK,SAAS,MAAM,CAAA;AAAA,cACvD,kBAAkB,IAAA,CAAK,IAAA;AAAA,cACvB,MAAA,EAAQ,EAAA;AAAA,cACR,YAAA,EAAc,KAAK,QAAA,CAAS,WAAA;AAAA,cAC5B,cAAA,EAAgB,EAAA,CAAG,EAAA,CAAG,GAAA,EAAI;AAAA,cAC1B,iBAAA,EAAmB,EAAA,CAAG,EAAA,CAAG,GAAA;AAAI,aAC/B,CAAE,CAAA;AAAA,YACF;AAAA,WACF;AACA,UAAA,MAAM,EAAA,CAAG,WAAA;AAAA,YACP,0BAAA;AAAA,YACA,KAAA,CAAM,IAAI,CAAA,IAAA,MAAS;AAAA,cACjB,YAAY,OAAA,CAAQ,SAAA;AAAA,cACpB,iBAAA,EAAmBA,+BAAA,CAAmB,IAAA,CAAK,QAAA,CAAS,MAAM;AAAA,aAC5D,CAAE,CAAA;AAAA,YACF;AAAA,WACF;AAAA,QACF,SAAS,KAAA,EAAO;AACd,UAAA,IAAI,CAACC,wCAAA,CAAwB,KAAK,CAAA,EAAG;AACnC,YAAA,MAAM,KAAA;AAAA,UACR,CAAA,MAAO;AACL,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,cAClB,uDAAuD,KAAK,CAAA;AAAA,aAC9D;AACA,YAAA,QAAA,CAAS,IAAA,CAAK,GAAG,KAAK,CAAA;AAAA,UACxB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,IAAA,IAAI,SAAS,MAAA,EAAQ;AACnB,MAAA,KAAA,MAAW;AAAA,QACT,QAAA,EAAU,EAAE,MAAA,EAAQ,WAAA,EAAY;AAAA,QAChC;AAAA,WACG,QAAA,EAAU;AACb,QAAA,MAAM,SAAA,GAAYD,gCAAmB,MAAM,CAAA;AAE3C,QAAA,IAAI;AACF,UAAA,IAAI,EAAA,GAAK,MAAME,+CAAA,CAAwB;AAAA,YACrC,EAAA;AAAA,YACA,MAAA;AAAA,YACA,IAAA;AAAA,YACA;AAAA,WACD,CAAA;AACD,UAAA,IAAI,CAAC,EAAA,EAAI;AACP,YAAA,EAAA,GAAK,MAAMC,+CAAA,CAAwB;AAAA,cACjC,EAAA;AAAA,cACA,MAAA;AAAA,cACA,IAAA;AAAA,cACA,WAAA;AAAA,cACA,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,aACtB,CAAA;AAAA,UACH;AACA,UAAA,IAAI,EAAA,EAAI;AACN,YAAA,MAAM,GAAgC,0BAA0B,CAAA,CAC7D,MAAM,mBAAA,EAAqB,SAAS,EACpC,MAAA,EAAO;AAEV,YAAA,MAAM,EAAA;AAAA,cACJ;AAAA,cACA,MAAA,CAAO;AAAA,cACP,YAAY,OAAA,CAAQ,SAAA;AAAA,cACpB,iBAAA,EAAmB;AAAA,aACpB,CAAA;AAAA,UACH,CAAA,MAAO;AACL,YAAA,MAAM,EAAA,CAAgC,0BAA0B,CAAA,CAC7D,KAAA,CAAM,qBAAqB,SAAS,CAAA,CACpC,QAAA,CAAS,EAAE,UAAA,EAAY,OAAA,CAAQ,SAAA,EAAW,EAC1C,MAAA,EAAO;AAEV,YAAA,MAAM,cAAA,GAAiB,MAAMC,iDAAA,CAAyB;AAAA,cACpD,EAAA;AAAA,cACA,SAAA;AAAA,cACA;AAAA,aACD,CAAA;AACD,YAAA,IAAI,cAAA,EAAgB;AAClB,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,gBAClB,CAAA,OAAA,EAAU,QAAQ,SAAS,CAAA,gCAAA,EAAmC,SAAS,CAAA,uBAAA,EAA0B,cAAc,iBAAiB,WAAW,CAAA;AAAA,eAC7I;AAAA,YACF;AAAA,UACF;AAAA,QACF,SAAS,KAAA,EAAO;AACd,UAAA,IAAIC,oBAAA,CAAgB,EAAA,EAAI,KAAK,CAAA,EAAG;AAC9B,YAAA,MAAM,KAAA;AAAA,UACR;AACA,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,YAClB,kBAAkB,SAAS,CAAA,eAAA,EAAkB,OAAA,CAAQ,SAAS,MAAM,KAAK,CAAA;AAAA,WAC3E;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,wBAAwB,QAAA,EAA0C;AACtE,IAAA,MAAM,EAAA,GAAK,QAAA;AAEX,IAAA,MAAM,OAAO,MAAM,EAAA;AAAA,MACjB;AAAA,KACF,CACG,QAAA,CAAS,YAAY,CAAA,CACrB,aAAa,YAAY,CAAA;AAE5B,IAAA,OAAO,IAAA,CACJ,GAAA,CAAI,CAAA,GAAA,KAAO,GAAA,CAAI,UAAU,CAAA,CACzB,MAAA,CAAO,CAAC,GAAA,KAAuB,CAAC,CAAC,GAAG,CAAA;AAAA,EACzC;AAAA,EAEA,MAAM,oBAAA,CACJ,QAAA,EACA,OAAA,EACA;AACA,IAAA,MAAM,EAAA,GAAK,QAAA;AACX,IAAA,MAAMC,0CAAqB,EAAE,EAAA,EAAI,IAAA,EAAM,OAAA,CAAQ,MAAM,CAAA;AAAA,EACvD;AAAA,EAEA,MAAc,WAAA,CACZ,EAAA,EACA,OAAA,EAKC;AACD,IAAA,IAAI,OAAA,CAAQ,SAAS,OAAA,EAAS;AAC5B,MAAA,MAAMC,MAAAA,GAAQ,IAAI,KAAA,EAAkD;AACpE,MAAA,MAAMC,SAAAA,GAAW,IAAI,KAAA,EAAkD;AACvE,MAAA,MAAMC,YAAW,OAAA,CAAQ,OAAA,CAAQ,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,SAAS,CAAA;AAErD,MAAA,KAAA,MAAW,SAASX,uBAAA,CAAO,KAAA,CAAM,OAAA,CAAQ,KAAA,EAAO,GAAI,CAAA,EAAG;AACrD,QAAA,MAAM,aAAa,KAAA,CAAM,GAAA,CAAI,OAAKE,+BAAA,CAAmB,CAAA,CAAE,MAAM,CAAC,CAAA;AAC9D,QAAA,MAAM,IAAA,GAAO,MAAM,EAAA,CAAsB,eAAe,EACrD,MAAA,CAAO,CAAC,YAAA,EAAc,kBAAA,EAAoB,cAAc,CAAC,CAAA,CACzD,OAAA,CAAQ,cAAc,UAAU,CAAA;AACnC,QAAA,MAAM,YAAY,IAAI,GAAA;AAAA,UACpB,IAAA,CAAK,IAAI,CAAA,GAAA,KAAO;AAAA,YACd,GAAA,CAAI,UAAA;AAAA,YACJ;AAAA,cACE,kBAAkB,GAAA,CAAI,gBAAA;AAAA,cACtB,cAAc,GAAA,CAAI;AAAA;AACpB,WACD;AAAA,SACH;AAEA,QAAA,KAAA,CAAM,OAAA,CAAQ,CAAC,QAAA,EAAU,CAAA,KAAM;AAC7B,UAAA,MAAM,SAAA,GAAY,WAAW,CAAC,CAAA;AAC9B,UAAA,MAAM,OAAA,GAAUU,uBAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAClD,UAAA,MAAM,QAAA,GAAW,SAAA,CAAU,GAAA,CAAI,SAAS,CAAA;AACxC,UAAA,IAAI,aAAa,MAAA,EAAW;AAE1B,YAAAH,OAAM,IAAA,CAAK,EAAE,QAAA,EAAU,IAAA,EAAM,SAAS,CAAA;AAAA,UACxC,YACG,QAAA,CAAS,WAAA,IAAe,IAAA,OAAW,QAAA,CAAS,gBAAgB,IAAA,CAAA,EAC7D;AAEA,YAAAE,SAAAA,CAAS,KAAK,SAAS,CAAA;AACvB,YAAAF,OAAM,IAAA,CAAK,EAAE,QAAA,EAAU,IAAA,EAAM,SAAS,CAAA;AAAA,UACxC,CAAA,MAAA,IAAW,OAAA,KAAY,QAAA,CAAS,gBAAA,EAAkB;AAEhD,YAAAC,UAAS,IAAA,CAAK,EAAE,QAAA,EAAU,IAAA,EAAM,SAAS,CAAA;AAAA,UAC3C;AAAA,QACF,CAAC,CAAA;AAAA,MACH;AAEA,MAAA,OAAO,EAAE,KAAA,EAAAD,MAAAA,EAAO,QAAA,EAAAC,SAAAA,EAAU,UAAAC,SAAAA,EAAS;AAAA,IACrC;AAGA,IAAA,MAAM,UAAU,MAAM,EAAA;AAAA,MACpB;AAAA,KACF,CACG,SAA4B,eAAA,EAAiB;AAAA,MAC5C,iBAAA,EAAmB;AAAA,KACpB,EACA,KAAA,CAAM,EAAE,YAAY,OAAA,CAAQ,SAAA,EAAW,CAAA,CACvC,MAAA,CAAO;AAAA,MACN,iBAAA,EAAmB,4CAAA;AAAA,MACnB,YAAA,EAAc,4BAAA;AAAA,MACd,gBAAA,EAAkB;AAAA,KACnB,CAAA;AAEH,IAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,KAAA,CAAM,GAAA,CAAI,CAAA,QAAA,MAAa;AAAA,MAC3C,QAAA;AAAA,MACA,GAAA,EAAKT,+BAAA,CAAmB,QAAA,CAAS,MAAM,CAAA;AAAA,MACvC,IAAA,EAAMU,uBAAA,CAAmB,QAAA,CAAS,MAAM;AAAA,KAC1C,CAAE,CAAA;AAEF,IAAA,MAAM,aAAa,IAAI,GAAA;AAAA,MACrB,OAAA,CAAQ,IAAI,CAAA,CAAA,KAAK;AAAA,QACf,CAAA,CAAE,iBAAA;AAAA,QACF;AAAA,UACE,aAAa,CAAA,CAAE,YAAA;AAAA,UACf,eAAe,CAAA,CAAE;AAAA;AACnB,OACD;AAAA,KACH;AACA,IAAA,MAAM,UAAA,GAAa,IAAI,GAAA,CAAI,KAAA,CAAM,IAAI,CAAA,IAAA,KAAQ,IAAA,CAAK,GAAG,CAAC,CAAA;AAEtD,IAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAkD;AACpE,IAAA,MAAM,QAAA,GAAW,IAAI,KAAA,EAAkD;AACvE,IAAA,MAAM,QAAA,GAAW,OAAA,CACd,GAAA,CAAI,CAAA,GAAA,KAAO,GAAA,CAAI,iBAAiB,CAAA,CAChC,MAAA,CAAO,CAAA,GAAA,KAAO,CAAC,UAAA,CAAW,GAAA,CAAI,GAAG,CAAC,CAAA;AAErC,IAAA,KAAA,MAAW,QAAQ,KAAA,EAAO;AACxB,MAAA,MAAM,MAAA,GAAS,UAAA,CAAW,GAAA,CAAI,IAAA,CAAK,GAAG,CAAA;AACtC,MAAA,MAAM,aAAa,EAAE,QAAA,EAAU,KAAK,QAAA,EAAU,IAAA,EAAM,KAAK,IAAA,EAAK;AAC9D,MAAA,IAAI,CAAC,MAAA,EAAQ;AAEX,QAAA,KAAA,CAAM,KAAK,UAAU,CAAA;AAAA,MACvB,YACG,MAAA,CAAO,WAAA,IAAe,aACtB,IAAA,CAAK,QAAA,CAAS,eAAe,MAAA,CAAA,EAC9B;AAEA,QAAA,QAAA,CAAS,IAAA,CAAK,KAAK,GAAG,CAAA;AACtB,QAAA,KAAA,CAAM,KAAK,UAAU,CAAA;AAAA,MACvB,CAAA,MAAA,IAAW,MAAA,CAAO,aAAA,KAAkB,IAAA,CAAK,IAAA,EAAM;AAE7C,QAAA,QAAA,CAAS,KAAK,UAAU,CAAA;AAAA,MAC1B;AAAA,IACF;AAEA,IAAA,OAAO,EAAE,KAAA,EAAO,QAAA,EAAU,QAAA,EAAS;AAAA,EACrC;AACF;;;;"}

package/dist/database/operations/relations/syncRelations.cjs.js

@@ -0,0 +1,148 @@
+'use strict';
+
+var lodash = require('lodash');
+
+function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e : { default: e }; }
+
+var lodash__default = /*#__PURE__*/_interopDefaultCompat(lodash);
+
+const BATCH_SIZE = 50;
+function tripletKey(r) {
+  return `${r.source_entity_ref}\0${r.type}\0${r.target_entity_ref}`;
+}
+async function syncRelations(knex, originatingEntityId, desired) {
+  const client = knex.client.config.client;
+  const deduped = lodash__default.default.uniqBy(desired, tripletKey);
+  if (client === "pg") {
+    return syncPostgres(knex, originatingEntityId, deduped);
+  }
+  return syncSimple(knex, originatingEntityId, deduped);
+}
+async function syncPostgres(knex, originatingEntityId, desired) {
+  if (desired.length === 0) {
+    const { rows: deleted2 } = await knex.raw(
+      `DELETE FROM relations
+       WHERE originating_entity_id = ?
+       RETURNING source_entity_ref, type, target_entity_ref`,
+      [originatingEntityId]
+    );
+    return { deleted: deleted2, inserted: [] };
+  }
+  const sources = [];
+  const types = [];
+  const targets = [];
+  for (const r of desired) {
+    sources.push(r.source_entity_ref);
+    types.push(r.type);
+    targets.push(r.target_entity_ref);
+  }
+  const { rows } = await knex.raw(
+    `
+    WITH desired AS (
+      SELECT * FROM unnest(?::text[], ?::text[], ?::text[])
+        AS t(source_entity_ref, type, target_entity_ref)
+    ),
+    deleted AS (
+      DELETE FROM relations r
+      WHERE r.originating_entity_id = ?
+        AND NOT EXISTS (
+          SELECT 1 FROM desired d
+          WHERE d.source_entity_ref = r.source_entity_ref
+            AND d.type = r.type
+            AND d.target_entity_ref = r.target_entity_ref
+        )
+      RETURNING r.source_entity_ref, r.type, r.target_entity_ref
+    ),
+    inserted AS (
+      INSERT INTO relations (originating_entity_id, source_entity_ref, type, target_entity_ref)
+      SELECT ?, d.source_entity_ref, d.type, d.target_entity_ref
+      FROM desired d
+      WHERE NOT EXISTS (
+        SELECT 1 FROM relations r
+        WHERE r.originating_entity_id = ?
+          AND r.source_entity_ref = d.source_entity_ref
+          AND r.type = d.type
+          AND r.target_entity_ref = d.target_entity_ref
+      )
+      RETURNING source_entity_ref, type, target_entity_ref
+    )
+    SELECT 'd' AS op, source_entity_ref, type, target_entity_ref FROM deleted
+    UNION ALL
+    SELECT 'i' AS op, source_entity_ref, type, target_entity_ref FROM inserted
+    `,
+    [
+      sources,
+      types,
+      targets,
+      originatingEntityId,
+      originatingEntityId,
+      originatingEntityId
+    ]
+  );
+  const deleted = [];
+  const inserted = [];
+  for (const row of rows) {
+    const triplet = {
+      source_entity_ref: row.source_entity_ref,
+      type: row.type,
+      target_entity_ref: row.target_entity_ref
+    };
+    if (row.op === "d") {
+      deleted.push(triplet);
+    } else {
+      inserted.push(triplet);
+    }
+  }
+  return { deleted, inserted };
+}
+async function syncSimple(knex, originatingEntityId, desired) {
+  const existing = await knex("relations").where({ originating_entity_id: originatingEntityId }).select(
+    "source_entity_ref",
+    "type",
+    "target_entity_ref"
+  );
+  const existingSet = new Map(existing.map((r) => [tripletKey(r), r]));
+  const desiredSet = new Map(desired.map((r) => [tripletKey(r), r]));
+  const toDelete = [...existingSet.entries()].filter(([key]) => !desiredSet.has(key)).map(([, row]) => row);
+  const toInsert = [...desiredSet.entries()].filter(([key]) => !existingSet.has(key)).map(([, row]) => row);
+  if (toDelete.length > 0) {
+    for (const chunk of lodash__default.default(toDelete).chunk(BATCH_SIZE).value()) {
+      await knex("relations").where({ originating_entity_id: originatingEntityId }).andWhere(function matchChunk() {
+        for (const r of chunk) {
+          this.orWhere({
+            source_entity_ref: r.source_entity_ref,
+            type: r.type,
+            target_entity_ref: r.target_entity_ref
+          });
+        }
+      }).delete();
+    }
+  }
+  if (toInsert.length > 0) {
+    await knex.batchInsert(
+      "relations",
+      toInsert.map((r) => ({
+        originating_entity_id: originatingEntityId,
+        source_entity_ref: r.source_entity_ref,
+        type: r.type,
+        target_entity_ref: r.target_entity_ref
+      })),
+      BATCH_SIZE
+    );
+  }
+  return {
+    deleted: toDelete.map((r) => ({
+      source_entity_ref: r.source_entity_ref,
+      type: r.type,
+      target_entity_ref: r.target_entity_ref
+    })),
+    inserted: toInsert.map((r) => ({
+      source_entity_ref: r.source_entity_ref,
+      type: r.type,
+      target_entity_ref: r.target_entity_ref
+    }))
+  };
+}
+
+exports.syncRelations = syncRelations;
+//# sourceMappingURL=syncRelations.cjs.js.map