@backstage/plugin-catalog-backend 1.9.0-next.3 → 1.9.0

package/CHANGELOG.md

@@ -1,5 +1,74 @@
 # @backstage/plugin-catalog-backend
 
+## 1.9.0
+
+### Minor Changes
+
+- 329b63f4dab: The catalog now has a new, optional `catalog.orphanStrategy` app-config parameter, which can have the string values `'keep'` (default) or `'delete'`.
+
+  If set to `'keep'` or left unset, the old behavior is maintained of keeping orphaned entities around until manually deleted.
+
+  If set to `'delete'`, the catalog will attempt to automatically clean out orphaned entities without manual intervention. Note that there are no guarantees that this process is instantaneous, so there may be some delay before orphaned items disappear.
+
+  For context, the [Life of an Entity](https://backstage.io/docs/features/software-catalog/life-of-an-entity/#orphaning) article goes into some more details on how the nature of orphaning works.
+
+  To enable the new behavior, you will need to pass the plugin task scheduler to your catalog backend builder. If your code already looks like this, you don't need to change it:
+
+  ```ts
+  // in packages/backend/src/plugins/catalog.ts
+  export default async function createPlugin(
+    env: PluginEnvironment,
+  ): Promise<Router> {
+    const builder = await CatalogBuilder.create(env);
+  ```
+
+  But if you pass things into the catalog builder one by one, you'll need to add the new field:
+
+  ```diff
+   // in packages/backend/src/plugins/catalog.ts
+     const builder = await CatalogBuilder.create({
+       // ... other dependencies
+  +    scheduler: env.scheduler,
+     });
+  ```
+
+  Finally adjust your app-config:
+
+  ```yaml
+  catalog:
+    orphanStrategy: delete
+  ```
+
+- 92a4590fc3a: Add monorepo support to CodeOwnersProccesor.
+
+### Patch Changes
+
+- 62a725e3a94: Use the `LocationSpec` type from the `catalog-common` package in place of the deprecated `LocationSpec` from the `catalog-node` package.
+- be5aca50114: Updates and moves OpenAPI spec to `src/schema/openapi.yaml` and uses `ApiRouter` type from `@backstage/backend-openapi-utils` to handle automatic types from the OpenAPI spec file.
+- c9a0fdcd2c8: Fix deprecated types.
+- 899ebfd8e02: Add full text search support to the `by-query` endpoint.
+- 1e4f5e91b8e: Bump `zod` and `zod-to-json-schema` dependencies.
+- c4b846359c0: Allow replacement of the BuiltinKindsEntityProcessor which enables customization of schema validation and connections emitted.
+- c36b89f2af3: Fixed bug in the `DefaultCatalogProcessingEngine` where entities that contained multiple different types of relations for the same source entity would not properly trigger stitching for that source entity.
+- 01ae205352e: Collator factories instantiated in new backend system modules and now marked as deprecated. Will be continued to be exported publicly until the new backend system is fully rolled out.
+- Updated dependencies
+  - @backstage/backend-common@0.18.4
+  - @backstage/plugin-scaffolder-common@1.2.7
+  - @backstage/catalog-client@1.4.1
+  - @backstage/plugin-permission-node@0.7.7
+  - @backstage/plugin-permission-common@0.7.5
+  - @backstage/backend-tasks@0.5.1
+  - @backstage/catalog-model@1.3.0
+  - @backstage/plugin-search-backend-module-catalog@0.1.0
+  - @backstage/integration@1.4.4
+  - @backstage/plugin-catalog-node@1.3.5
+  - @backstage/backend-plugin-api@0.5.1
+  - @backstage/config@1.0.7
+  - @backstage/errors@1.1.5
+  - @backstage/types@1.0.2
+  - @backstage/plugin-catalog-common@1.0.13
+  - @backstage/plugin-search-common@1.2.3
+
 ## 1.9.0-next.3
 
 ### Minor Changes

package/alpha/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend",
-  "version": "1.9.0-next.3",
+  "version": "1.9.0",
   "main": "../dist/alpha.cjs.js",
   "types": "../dist/alpha.d.ts"
 }

package/config.d.ts

@@ -81,7 +81,6 @@ export interface Config {
      * be used in combination with static locations to only serve operator
      * provided locations. Effectively this removes the ability to register new
      * components to a running backstage instance.
-     *
      */
     readonly?: boolean;
 
@@ -136,5 +135,12 @@ export interface Config {
         allow: Array<string>;
       }>;
     }>;
+
+    /**
+     * The strategy to use for entities that are orphaned, i.e. no longer have
+     * any other entities or providers referencing them. The default value is
+     * "keep".
+     */
+    orphanStrategy?: 'keep' | 'delete';
   };
 }

package/dist/alpha.cjs.js

@@ -4,7 +4,7 @@ Object.defineProperty(exports, '__esModule', { value: true });
 
 var alpha = require('@backstage/plugin-catalog-common/alpha');
 var pluginPermissionNode = require('@backstage/plugin-permission-node');
-var CatalogBuilder = require('./cjs/CatalogBuilder-c295413f.cjs.js');
+var CatalogBuilder = require('./cjs/CatalogBuilder-ecfd2db6.cjs.js');
 var backendPluginApi = require('@backstage/backend-plugin-api');
 var alpha$1 = require('@backstage/plugin-catalog-node/alpha');
 var backendCommon = require('@backstage/backend-common');
@@ -26,6 +26,7 @@ require('p-limit');
 require('uuid');
 require('luxon');
 require('prom-client');
+require('lodash/uniq');
 require('fast-json-stable-stringify');
 require('@opentelemetry/api');
 require('zod');
@@ -94,7 +95,8 @@ const catalogPlugin = backendPluginApi.createBackendPlugin({
         permissions: backendPluginApi.coreServices.permissions,
         database: backendPluginApi.coreServices.database,
         httpRouter: backendPluginApi.coreServices.httpRouter,
-        lifecycle: backendPluginApi.coreServices.lifecycle
+        lifecycle: backendPluginApi.coreServices.lifecycle,
+        scheduler: backendPluginApi.coreServices.scheduler
       },
       async init({
         logger,
@@ -103,7 +105,8 @@ const catalogPlugin = backendPluginApi.createBackendPlugin({
         database,
         permissions,
         httpRouter,
-        lifecycle
+        lifecycle,
+        scheduler
       }) {
         const winstonLogger = backendCommon.loggerToWinstonLogger(logger);
         const builder = await CatalogBuilder.CatalogBuilder.create({
@@ -111,6 +114,7 @@ const catalogPlugin = backendPluginApi.createBackendPlugin({
           reader,
           permissions,
           database,
+          scheduler,
           logger: winstonLogger
         });
         builder.addProcessor(...processingExtensions.processors);

package/dist/alpha.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"alpha.cjs.js","sources":["../src/permissions/conditionExports.ts","../src/service/CatalogPlugin.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { RESOURCE_TYPE_CATALOG_ENTITY } from '@backstage/plugin-catalog-common/alpha';\nimport { createConditionExports } from '@backstage/plugin-permission-node';\nimport { permissionRules } from './rules';\n\nconst { conditions, createConditionalDecision } = createConditionExports({\n  pluginId: 'catalog',\n  resourceType: RESOURCE_TYPE_CATALOG_ENTITY,\n  rules: permissionRules,\n});\n\n/**\n * These conditions are used when creating conditional decisions for catalog\n * entities that are returned by authorization policies.\n *\n * @alpha\n */\nexport const catalogConditions = conditions;\n\n/**\n * `createCatalogConditionalDecision` can be used when authoring policies to\n * create conditional decisions. It requires a permission of type\n * `ResourcePermission<'catalog-entity'>` to be passed as the first parameter.\n * It's recommended that you use the provided `isResourcePermission` and\n * `isPermission` helper methods to narrow the type of the permission passed to\n * the handle method as shown below.\n *\n * ```\n * // MyAuthorizationPolicy.ts\n * ...\n * import { createCatalogPolicyDecision } from '@backstage/plugin-catalog-backend';\n * import { RESOURCE_TYPE_CATALOG_ENTITY } from '@backstage/plugin-catalog-common';\n *\n * class MyAuthorizationPolicy implements PermissionPolicy {\n *   async handle(request, user) {\n *     ...\n *\n *     if (isResourcePermission(request.permission, RESOURCE_TYPE_CATALOG_ENTITY)) {\n *       return createCatalogConditionalDecision(\n *         request.permission,\n *         { anyOf: [...insert conditions here...] }\n *       );\n *     }\n *\n *     ...\n * }\n * ```\n *\n * @alpha\n */\nexport const createCatalogConditionalDecision = createConditionalDecision;\n","/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nimport {\n  createBackendPlugin,\n  coreServices,\n} from '@backstage/backend-plugin-api';\nimport { CatalogBuilder } from './CatalogBuilder';\nimport {\n  CatalogProcessingExtensionPoint,\n  catalogProcessingExtensionPoint,\n} from '@backstage/plugin-catalog-node/alpha';\nimport {\n  CatalogProcessor,\n  EntityProvider,\n} from '@backstage/plugin-catalog-node';\nimport { loggerToWinstonLogger } from '@backstage/backend-common';\n\nclass CatalogExtensionPointImpl implements CatalogProcessingExtensionPoint {\n  #processors = new Array<CatalogProcessor>();\n  #entityProviders = new Array<EntityProvider>();\n\n  addProcessor(\n    ...processors: Array<CatalogProcessor | Array<CatalogProcessor>>\n  ): void {\n    this.#processors.push(...processors.flat());\n  }\n\n  addEntityProvider(\n    ...providers: Array<EntityProvider | Array<EntityProvider>>\n  ): void {\n    this.#entityProviders.push(...providers.flat());\n  }\n\n  get processors() {\n    return this.#processors;\n  }\n\n  get entityProviders() {\n    return this.#entityProviders;\n  }\n}\n\n/**\n * Catalog plugin\n * @alpha\n */\nexport const catalogPlugin = createBackendPlugin({\n  pluginId: 'catalog',\n  register(env) {\n    const processingExtensions = new CatalogExtensionPointImpl();\n    // plugins depending on this API will be initialized before this plugins init method is executed.\n    env.registerExtensionPoint(\n      catalogProcessingExtensionPoint,\n      processingExtensions,\n    );\n\n    env.registerInit({\n      deps: {\n        logger: coreServices.logger,\n        config: coreServices.config,\n        reader: coreServices.urlReader,\n        permissions: coreServices.permissions,\n        database: coreServices.database,\n        httpRouter: coreServices.httpRouter,\n        lifecycle: coreServices.lifecycle,\n      },\n      async init({\n        logger,\n        config,\n        reader,\n        database,\n        permissions,\n        httpRouter,\n        lifecycle,\n      }) {\n        const winstonLogger = loggerToWinstonLogger(logger);\n        const builder = await CatalogBuilder.create({\n          config,\n          reader,\n          permissions,\n          database,\n          logger: winstonLogger,\n        });\n        builder.addProcessor(...processingExtensions.processors);\n        builder.addEntityProvider(...processingExtensions.entityProviders);\n        const { processingEngine, router } = await builder.build();\n\n        await processingEngine.start();\n        lifecycle.addShutdownHook(() => processingEngine.stop());\n        httpRouter.use(router);\n      },\n    });\n  },\n});\n"],"names":["createConditionExports","RESOURCE_TYPE_CATALOG_ENTITY","permissionRules","createBackendPlugin","catalogProcessingExtensionPoint","coreServices","loggerToWinstonLogger","CatalogBuilder"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoBA,MAAM,EAAE,UAAA,EAAY,yBAA0B,EAAA,GAAIA,2CAAuB,CAAA;AAAA,EACvE,QAAU,EAAA,SAAA;AAAA,EACV,YAAc,EAAAC,kCAAA;AAAA,EACd,KAAO,EAAAC,8BAAA;AACT,CAAC,CAAA,CAAA;AAQM,MAAM,iBAAoB,GAAA,WAAA;AAiC1B,MAAM,gCAAmC,GAAA;;;;;;;;;;;;;;;ACjEhD,IAAA,WAAA,EAAA,gBAAA,CAAA;AA8BA,MAAM,yBAAqE,CAAA;AAAA,EAA3E,WAAA,GAAA;AACE,IAAA,YAAA,CAAA,IAAA,EAAA,WAAA,EAAc,IAAI,KAAwB,EAAA,CAAA,CAAA;AAC1C,IAAA,YAAA,CAAA,IAAA,EAAA,gBAAA,EAAmB,IAAI,KAAsB,EAAA,CAAA,CAAA;AAAA,GAAA;AAAA,EAE7C,gBACK,UACG,EAAA;AACN,IAAA,YAAA,CAAA,IAAA,EAAK,WAAY,CAAA,CAAA,IAAA,CAAK,GAAG,UAAA,CAAW,MAAM,CAAA,CAAA;AAAA,GAC5C;AAAA,EAEA,qBACK,SACG,EAAA;AACN,IAAA,YAAA,CAAA,IAAA,EAAK,gBAAiB,CAAA,CAAA,IAAA,CAAK,GAAG,SAAA,CAAU,MAAM,CAAA,CAAA;AAAA,GAChD;AAAA,EAEA,IAAI,UAAa,GAAA;AACf,IAAA,OAAO,YAAK,CAAA,IAAA,EAAA,WAAA,CAAA,CAAA;AAAA,GACd;AAAA,EAEA,IAAI,eAAkB,GAAA;AACpB,IAAA,OAAO,YAAK,CAAA,IAAA,EAAA,gBAAA,CAAA,CAAA;AAAA,GACd;AACF,CAAA;AAtBE,WAAA,GAAA,IAAA,OAAA,EAAA,CAAA;AACA,gBAAA,GAAA,IAAA,OAAA,EAAA,CAAA;AA2BK,MAAM,gBAAgBC,oCAAoB,CAAA;AAAA,EAC/C,QAAU,EAAA,SAAA;AAAA,EACV,SAAS,GAAK,EAAA;AACZ,IAAM,MAAA,oBAAA,GAAuB,IAAI,yBAA0B,EAAA,CAAA;AAE3D,IAAI,GAAA,CAAA,sBAAA;AAAA,MACFC,uCAAA;AAAA,MACA,oBAAA;AAAA,KACF,CAAA;AAEA,IAAA,GAAA,CAAI,YAAa,CAAA;AAAA,MACf,IAAM,EAAA;AAAA,QACJ,QAAQC,6BAAa,CAAA,MAAA;AAAA,QACrB,QAAQA,6BAAa,CAAA,MAAA;AAAA,QACrB,QAAQA,6BAAa,CAAA,SAAA;AAAA,QACrB,aAAaA,6BAAa,CAAA,WAAA;AAAA,QAC1B,UAAUA,6BAAa,CAAA,QAAA;AAAA,QACvB,YAAYA,6BAAa,CAAA,UAAA;AAAA,QACzB,WAAWA,6BAAa,CAAA,SAAA;AAAA,OAC1B;AAAA,MACA,MAAM,IAAK,CAAA;AAAA,QACT,MAAA;AAAA,QACA,MAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,WAAA;AAAA,QACA,UAAA;AAAA,QACA,SAAA;AAAA,OACC,EAAA;AACD,QAAM,MAAA,aAAA,GAAgBC,oCAAsB,MAAM,CAAA,CAAA;AAClD,QAAM,MAAA,OAAA,GAAU,MAAMC,6BAAA,CAAe,MAAO,CAAA;AAAA,UAC1C,MAAA;AAAA,UACA,MAAA;AAAA,UACA,WAAA;AAAA,UACA,QAAA;AAAA,UACA,MAAQ,EAAA,aAAA;AAAA,SACT,CAAA,CAAA;AACD,QAAQ,OAAA,CAAA,YAAA,CAAa,GAAG,oBAAA,CAAqB,UAAU,CAAA,CAAA;AACvD,QAAQ,OAAA,CAAA,iBAAA,CAAkB,GAAG,oBAAA,CAAqB,eAAe,CAAA,CAAA;AACjE,QAAA,MAAM,EAAE,gBAAkB,EAAA,MAAA,EAAW,GAAA,MAAM,QAAQ,KAAM,EAAA,CAAA;AAEzD,QAAA,MAAM,iBAAiB,KAAM,EAAA,CAAA;AAC7B,QAAA,SAAA,CAAU,eAAgB,CAAA,MAAM,gBAAiB,CAAA,IAAA,EAAM,CAAA,CAAA;AACvD,QAAA,UAAA,CAAW,IAAI,MAAM,CAAA,CAAA;AAAA,OACvB;AAAA,KACD,CAAA,CAAA;AAAA,GACH;AACF,CAAC;;;;;;;;"}
+{"version":3,"file":"alpha.cjs.js","sources":["../src/permissions/conditionExports.ts","../src/service/CatalogPlugin.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { RESOURCE_TYPE_CATALOG_ENTITY } from '@backstage/plugin-catalog-common/alpha';\nimport { createConditionExports } from '@backstage/plugin-permission-node';\nimport { permissionRules } from './rules';\n\nconst { conditions, createConditionalDecision } = createConditionExports({\n  pluginId: 'catalog',\n  resourceType: RESOURCE_TYPE_CATALOG_ENTITY,\n  rules: permissionRules,\n});\n\n/**\n * These conditions are used when creating conditional decisions for catalog\n * entities that are returned by authorization policies.\n *\n * @alpha\n */\nexport const catalogConditions = conditions;\n\n/**\n * `createCatalogConditionalDecision` can be used when authoring policies to\n * create conditional decisions. It requires a permission of type\n * `ResourcePermission<'catalog-entity'>` to be passed as the first parameter.\n * It's recommended that you use the provided `isResourcePermission` and\n * `isPermission` helper methods to narrow the type of the permission passed to\n * the handle method as shown below.\n *\n * ```\n * // MyAuthorizationPolicy.ts\n * ...\n * import { createCatalogPolicyDecision } from '@backstage/plugin-catalog-backend';\n * import { RESOURCE_TYPE_CATALOG_ENTITY } from '@backstage/plugin-catalog-common';\n *\n * class MyAuthorizationPolicy implements PermissionPolicy {\n *   async handle(request, user) {\n *     ...\n *\n *     if (isResourcePermission(request.permission, RESOURCE_TYPE_CATALOG_ENTITY)) {\n *       return createCatalogConditionalDecision(\n *         request.permission,\n *         { anyOf: [...insert conditions here...] }\n *       );\n *     }\n *\n *     ...\n * }\n * ```\n *\n * @alpha\n */\nexport const createCatalogConditionalDecision = createConditionalDecision;\n","/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nimport {\n  createBackendPlugin,\n  coreServices,\n} from '@backstage/backend-plugin-api';\nimport { CatalogBuilder } from './CatalogBuilder';\nimport {\n  CatalogProcessingExtensionPoint,\n  catalogProcessingExtensionPoint,\n} from '@backstage/plugin-catalog-node/alpha';\nimport {\n  CatalogProcessor,\n  EntityProvider,\n} from '@backstage/plugin-catalog-node';\nimport { loggerToWinstonLogger } from '@backstage/backend-common';\n\nclass CatalogExtensionPointImpl implements CatalogProcessingExtensionPoint {\n  #processors = new Array<CatalogProcessor>();\n  #entityProviders = new Array<EntityProvider>();\n\n  addProcessor(\n    ...processors: Array<CatalogProcessor | Array<CatalogProcessor>>\n  ): void {\n    this.#processors.push(...processors.flat());\n  }\n\n  addEntityProvider(\n    ...providers: Array<EntityProvider | Array<EntityProvider>>\n  ): void {\n    this.#entityProviders.push(...providers.flat());\n  }\n\n  get processors() {\n    return this.#processors;\n  }\n\n  get entityProviders() {\n    return this.#entityProviders;\n  }\n}\n\n/**\n * Catalog plugin\n * @alpha\n */\nexport const catalogPlugin = createBackendPlugin({\n  pluginId: 'catalog',\n  register(env) {\n    const processingExtensions = new CatalogExtensionPointImpl();\n    // plugins depending on this API will be initialized before this plugins init method is executed.\n    env.registerExtensionPoint(\n      catalogProcessingExtensionPoint,\n      processingExtensions,\n    );\n\n    env.registerInit({\n      deps: {\n        logger: coreServices.logger,\n        config: coreServices.config,\n        reader: coreServices.urlReader,\n        permissions: coreServices.permissions,\n        database: coreServices.database,\n        httpRouter: coreServices.httpRouter,\n        lifecycle: coreServices.lifecycle,\n        scheduler: coreServices.scheduler,\n      },\n      async init({\n        logger,\n        config,\n        reader,\n        database,\n        permissions,\n        httpRouter,\n        lifecycle,\n        scheduler,\n      }) {\n        const winstonLogger = loggerToWinstonLogger(logger);\n        const builder = await CatalogBuilder.create({\n          config,\n          reader,\n          permissions,\n          database,\n          scheduler,\n          logger: winstonLogger,\n        });\n        builder.addProcessor(...processingExtensions.processors);\n        builder.addEntityProvider(...processingExtensions.entityProviders);\n        const { processingEngine, router } = await builder.build();\n\n        await processingEngine.start();\n        lifecycle.addShutdownHook(() => processingEngine.stop());\n        httpRouter.use(router);\n      },\n    });\n  },\n});\n"],"names":["createConditionExports","RESOURCE_TYPE_CATALOG_ENTITY","permissionRules","createBackendPlugin","catalogProcessingExtensionPoint","coreServices","loggerToWinstonLogger","CatalogBuilder"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoBA,MAAM,EAAE,UAAA,EAAY,yBAA0B,EAAA,GAAIA,2CAAuB,CAAA;AAAA,EACvE,QAAU,EAAA,SAAA;AAAA,EACV,YAAc,EAAAC,kCAAA;AAAA,EACd,KAAO,EAAAC,8BAAA;AACT,CAAC,CAAA,CAAA;AAQM,MAAM,iBAAoB,GAAA,WAAA;AAiC1B,MAAM,gCAAmC,GAAA;;;;;;;;;;;;;;;ACjEhD,IAAA,WAAA,EAAA,gBAAA,CAAA;AA8BA,MAAM,yBAAqE,CAAA;AAAA,EAA3E,WAAA,GAAA;AACE,IAAA,YAAA,CAAA,IAAA,EAAA,WAAA,EAAc,IAAI,KAAwB,EAAA,CAAA,CAAA;AAC1C,IAAA,YAAA,CAAA,IAAA,EAAA,gBAAA,EAAmB,IAAI,KAAsB,EAAA,CAAA,CAAA;AAAA,GAAA;AAAA,EAE7C,gBACK,UACG,EAAA;AACN,IAAA,YAAA,CAAA,IAAA,EAAK,WAAY,CAAA,CAAA,IAAA,CAAK,GAAG,UAAA,CAAW,MAAM,CAAA,CAAA;AAAA,GAC5C;AAAA,EAEA,qBACK,SACG,EAAA;AACN,IAAA,YAAA,CAAA,IAAA,EAAK,gBAAiB,CAAA,CAAA,IAAA,CAAK,GAAG,SAAA,CAAU,MAAM,CAAA,CAAA;AAAA,GAChD;AAAA,EAEA,IAAI,UAAa,GAAA;AACf,IAAA,OAAO,YAAK,CAAA,IAAA,EAAA,WAAA,CAAA,CAAA;AAAA,GACd;AAAA,EAEA,IAAI,eAAkB,GAAA;AACpB,IAAA,OAAO,YAAK,CAAA,IAAA,EAAA,gBAAA,CAAA,CAAA;AAAA,GACd;AACF,CAAA;AAtBE,WAAA,GAAA,IAAA,OAAA,EAAA,CAAA;AACA,gBAAA,GAAA,IAAA,OAAA,EAAA,CAAA;AA2BK,MAAM,gBAAgBC,oCAAoB,CAAA;AAAA,EAC/C,QAAU,EAAA,SAAA;AAAA,EACV,SAAS,GAAK,EAAA;AACZ,IAAM,MAAA,oBAAA,GAAuB,IAAI,yBAA0B,EAAA,CAAA;AAE3D,IAAI,GAAA,CAAA,sBAAA;AAAA,MACFC,uCAAA;AAAA,MACA,oBAAA;AAAA,KACF,CAAA;AAEA,IAAA,GAAA,CAAI,YAAa,CAAA;AAAA,MACf,IAAM,EAAA;AAAA,QACJ,QAAQC,6BAAa,CAAA,MAAA;AAAA,QACrB,QAAQA,6BAAa,CAAA,MAAA;AAAA,QACrB,QAAQA,6BAAa,CAAA,SAAA;AAAA,QACrB,aAAaA,6BAAa,CAAA,WAAA;AAAA,QAC1B,UAAUA,6BAAa,CAAA,QAAA;AAAA,QACvB,YAAYA,6BAAa,CAAA,UAAA;AAAA,QACzB,WAAWA,6BAAa,CAAA,SAAA;AAAA,QACxB,WAAWA,6BAAa,CAAA,SAAA;AAAA,OAC1B;AAAA,MACA,MAAM,IAAK,CAAA;AAAA,QACT,MAAA;AAAA,QACA,MAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,WAAA;AAAA,QACA,UAAA;AAAA,QACA,SAAA;AAAA,QACA,SAAA;AAAA,OACC,EAAA;AACD,QAAM,MAAA,aAAA,GAAgBC,oCAAsB,MAAM,CAAA,CAAA;AAClD,QAAM,MAAA,OAAA,GAAU,MAAMC,6BAAA,CAAe,MAAO,CAAA;AAAA,UAC1C,MAAA;AAAA,UACA,MAAA;AAAA,UACA,WAAA;AAAA,UACA,QAAA;AAAA,UACA,SAAA;AAAA,UACA,MAAQ,EAAA,aAAA;AAAA,SACT,CAAA,CAAA;AACD,QAAQ,OAAA,CAAA,YAAA,CAAa,GAAG,oBAAA,CAAqB,UAAU,CAAA,CAAA;AACvD,QAAQ,OAAA,CAAA,iBAAA,CAAkB,GAAG,oBAAA,CAAqB,eAAe,CAAA,CAAA;AACjE,QAAA,MAAM,EAAE,gBAAkB,EAAA,MAAA,EAAW,GAAA,MAAM,QAAQ,KAAM,EAAA,CAAA;AAEzD,QAAA,MAAM,iBAAiB,KAAM,EAAA,CAAA;AAC7B,QAAA,SAAA,CAAU,eAAgB,CAAA,MAAM,gBAAiB,CAAA,IAAA,EAAM,CAAA,CAAA;AACvD,QAAA,UAAA,CAAW,IAAI,MAAM,CAAA,CAAA;AAAA,OACvB;AAAA,KACD,CAAA,CAAA;AAAA,GACH;AACF,CAAC;;;;;;;;"}

package/dist/cjs/{CatalogBuilder-c295413f.cjs.js → CatalogBuilder-ecfd2db6.cjs.js}

@@ -19,6 +19,7 @@ var uuid = require('uuid');
 var backendCommon = require('@backstage/backend-common');
 var luxon = require('luxon');
 var promClient = require('prom-client');
+var uniq = require('lodash/uniq');
 var stableStringify = require('fast-json-stable-stringify');
 var api = require('@opentelemetry/api');
 var zod = require('zod');
@@ -59,6 +60,7 @@ var g__default = /*#__PURE__*/_interopDefaultLegacy(g);
 var path__default = /*#__PURE__*/_interopDefaultLegacy(path);
 var yaml__default = /*#__PURE__*/_interopDefaultLegacy(yaml);
 var limiterFactory__default = /*#__PURE__*/_interopDefaultLegacy(limiterFactory);
+var uniq__default = /*#__PURE__*/_interopDefaultLegacy(uniq);
 var stableStringify__default = /*#__PURE__*/_interopDefaultLegacy(stableStringify);
 var express__default = /*#__PURE__*/_interopDefaultLegacy(express);
 var Router__default = /*#__PURE__*/_interopDefaultLegacy(Router);
@@ -1169,6 +1171,48 @@ async function updateUnprocessedEntity(options) {
   return refreshResult === 1;
 }
 
+async function deleteOrphanedEntities(options) {
+  const { tx } = options;
+  let total = 0;
+  for (let i = 0; i < 100; ++i) {
+    const candidates = await tx.with(
+      "orphans",
+      (orphans) => orphans.from("refresh_state").select("entity_id", "entity_ref").whereNotIn(
+        "entity_ref",
+        (keep) => keep.distinct("target_entity_ref").from("refresh_state_references")
+      )
+    ).select({
+      entityId: "orphans.entity_id",
+      relationSourceId: "refresh_state.entity_id"
+    }).from("orphans").leftOuterJoin(
+      "relations",
+      "relations.target_entity_ref",
+      "orphans.entity_ref"
+    ).leftOuterJoin(
+      "refresh_state",
+      "refresh_state.entity_ref",
+      "relations.source_entity_ref"
+    );
+    if (!candidates.length) {
+      break;
+    }
+    const orphanIds = uniq__default["default"](candidates.map((r) => r.entityId));
+    const orphanRelationIds = uniq__default["default"](
+      candidates.map((r) => r.relationSourceId).filter(Boolean)
+    );
+    total += orphanIds.length;
+    await tx.table("refresh_state").delete().whereIn("entity_id", orphanIds);
+    await tx.table("final_entities").update({
+      hash: "orphan-relation-deleted"
+    }).whereIn("entity_id", orphanRelationIds);
+    await tx.table("refresh_state").update({
+      result_hash: "orphan-relation-deleted",
+      next_update_at: tx.fn.now()
+    }).whereIn("entity_id", orphanRelationIds);
+  }
+  return total;
+}
+
 function generateStableHash$1(entity) {
   return crypto.createHash("sha1").update(stableStringify__default["default"]({ ...entity })).digest("hex");
 }
@@ -1309,6 +1353,10 @@ class DefaultProcessingDatabase {
     const entityRefs = rows.map((r) => r.source_entity_ref).filter(Boolean);
     return { entityRefs };
   }
+  async deleteOrphanedEntities(txOpaque) {
+    const tx = txOpaque;
+    return await deleteOrphanedEntities({ tx });
+  }
   async transaction(fn) {
     try {
       let result = void 0;
@@ -1445,21 +1493,40 @@ function startTaskPipeline(options) {
 
 const CACHE_TTL = 5;
 class DefaultCatalogProcessingEngine {
-  constructor(logger, processingDatabase, orchestrator, stitcher, createHash, pollingIntervalMs = 1e3, onProcessingError, tracker = progressTracker()) {
-    this.logger = logger;
-    this.processingDatabase = processingDatabase;
-    this.orchestrator = orchestrator;
-    this.stitcher = stitcher;
-    this.createHash = createHash;
-    this.pollingIntervalMs = pollingIntervalMs;
-    this.onProcessingError = onProcessingError;
-    this.tracker = tracker;
+  constructor(options) {
+    var _a, _b, _c;
+    this.config = options.config;
+    this.scheduler = options.scheduler;
+    this.logger = options.logger;
+    this.processingDatabase = options.processingDatabase;
+    this.orchestrator = options.orchestrator;
+    this.stitcher = options.stitcher;
+    this.createHash = options.createHash;
+    this.pollingIntervalMs = (_a = options.pollingIntervalMs) != null ? _a : 1e3;
+    this.orphanCleanupIntervalMs = (_b = options.orphanCleanupIntervalMs) != null ? _b : 3e4;
+    this.onProcessingError = options.onProcessingError;
+    this.tracker = (_c = options.tracker) != null ? _c : progressTracker();
+    this.stopFunc = void 0;
   }
   async start() {
     if (this.stopFunc) {
       throw new Error("Processing engine is already started");
     }
-    this.stopFunc = startTaskPipeline({
+    const stopPipeline = this.startPipeline();
+    const stopCleanup = this.startOrphanCleanup();
+    this.stopFunc = () => {
+      stopPipeline();
+      stopCleanup();
+    };
+  }
+  async stop() {
+    if (this.stopFunc) {
+      this.stopFunc();
+      this.stopFunc = void 0;
+    }
+  }
+  startPipeline() {
+    return startTaskPipeline({
       lowWatermark: 5,
       highWatermark: 10,
       pollingIntervalMs: this.pollingIntervalMs,
@@ -1618,11 +1685,40 @@ class DefaultCatalogProcessingEngine {
       }
     });
   }
-  async stop() {
-    if (this.stopFunc) {
-      this.stopFunc();
-      this.stopFunc = void 0;
+  startOrphanCleanup() {
+    var _a;
+    const strategy = (_a = this.config.getOptionalString("catalog.orphanStrategy")) != null ? _a : "keep";
+    if (strategy !== "delete") {
+      return () => {
+      };
+    }
+    const runOnce = async () => {
+      try {
+        await this.processingDatabase.transaction(async (tx) => {
+          const n = await this.processingDatabase.deleteOrphanedEntities(tx);
+          this.logger.info(`Deleted ${n} orphaned entities`);
+        });
+      } catch (error) {
+        this.logger.warn(`Failed to delete orphaned entities`, error);
+      }
+    };
+    if (this.scheduler) {
+      const abortController = new AbortController();
+      this.scheduler.scheduleTask({
+        id: "catalog_orphan_cleanup",
+        frequency: { milliseconds: this.orphanCleanupIntervalMs },
+        timeout: { milliseconds: this.orphanCleanupIntervalMs * 0.8 },
+        fn: runOnce,
+        signal: abortController.signal
+      });
+      return () => {
+        abortController.abort();
+      };
     }
+    const intervalKey = setInterval(runOnce, this.orphanCleanupIntervalMs);
+    return () => {
+      clearInterval(intervalKey);
+    };
   }
 }
 function progressTracker() {
@@ -1652,9 +1748,7 @@ function progressTracker() {
   const meter = api.metrics.getMeter("default");
   const stitchedEntities = meter.createCounter(
     "catalog.stitched.entities.count",
-    {
-      description: "Amount of entities stitched"
-    }
+    { description: "Amount of entities stitched" }
   );
   const processedEntities = meter.createCounter(
     "catalog.processed.entities.count",
@@ -4684,7 +4778,7 @@ class CatalogBuilder {
    */
   async build() {
     var _a, _b;
-    const { config, database, logger, permissions } = this.env;
+    const { config, database, logger, permissions, scheduler } = this.env;
     const policy = this.buildEntityPolicy();
     const processors = this.buildProcessors();
     const parser = this.parser || defaultEntityDataParser;
@@ -4766,18 +4860,20 @@ class CatalogBuilder {
       [...this.entityProviders, locationStore, configLocationProvider],
       (provider) => provider.getProviderName()
     );
-    const processingEngine = new DefaultCatalogProcessingEngine(
+    const processingEngine = new DefaultCatalogProcessingEngine({
+      config,
+      scheduler,
       logger,
       processingDatabase,
       orchestrator,
       stitcher,
-      () => crypto.createHash("sha1"),
-      1e3,
-      (event) => {
+      createHash: () => crypto.createHash("sha1"),
+      pollingIntervalMs: 1e3,
+      onProcessingError: (event) => {
         var _a2;
         (_a2 = this.onProcessingError) == null ? void 0 : _a2.call(this, event);
       }
-    );
+    });
     const locationAnalyzer = (_b = this.locationAnalyzer) != null ? _b : new RepoLocationAnalyzer(logger, integrations, this.locationAnalyzers);
     const locationService = new AuthorizedLocationService(
       new DefaultLocationService(locationStore, orchestrator, {
@@ -4960,4 +5056,4 @@ exports.createCatalogPermissionRule = createCatalogPermissionRule;
 exports.createRandomProcessingInterval = createRandomProcessingInterval;
 exports.parseEntityYaml = parseEntityYaml;
 exports.permissionRules = permissionRules;
-//# sourceMappingURL=CatalogBuilder-c295413f.cjs.js.map
+//# sourceMappingURL=CatalogBuilder-ecfd2db6.cjs.js.map