@backstage/plugin-catalog-backend 1.2.1-next.1 → 1.3.0

package/CHANGELOG.md

@@ -1,5 +1,142 @@
 # @backstage/plugin-catalog-backend
 
+## 1.3.0
+
+### Minor Changes
+
+- 1dd6c22cc8: Added an option to be able to trigger refreshes on entities based on a prestored arbitrary key.
+
+  The UrlReaderProcessor, FileReaderProcessor got updated to store the absolute URL of the catalog file as a refresh key. In the format of `<type>:<target>`
+  The PlaceholderProcessor got updated to store the resolverValues as refreshKeys for the entities.
+
+  The custom resolvers will need to be updated to pass in a `CatalogProcessorEmit` function as parameter and they should be updated to emit their refresh processingResults. You can see the updated resolvers in the `PlaceholderProcessor.ts`
+
+  ```ts
+    // yamlPlaceholderResolver
+    ...
+    const { content, url } = await readTextLocation(params);
+
+    params.emit(processingResult.refresh(`url:${url}`));
+    ...
+  ```
+
+- 91c1d12123: Export experimental `catalogPlugin` for the new backend system. This export is not considered stable and should not be used in production.
+
+### Patch Changes
+
+- 1e02fe46d6: Fixed bug where catalog metrics weren't being tracked.
+- 5f6b847c15: Fix Error Code in Register Component DryRun
+- a70869e775: Updated dependency `msw` to `^0.43.0`.
+- 4e9a90e307: Updated dependency `luxon` to `^3.0.0`.
+- 72622d9143: Updated dependency `yaml` to `^2.0.0`.
+- 8006d0f9bf: Updated dependency `msw` to `^0.44.0`.
+- 679b32172e: Updated dependency `knex` to `^2.0.0`.
+- fa0533e604: CatalogBuilder supports now subscription to processing engine errors.
+
+  ```ts
+  subscribe(options: {
+    onProcessingError: (event: { unprocessedEntity: Entity, error: Error }) => Promise<void> | void;
+  });
+  ```
+
+  If you want to get notified on errors while processing the entities, you call CatalogBuilder.subscribe
+  to get notifications with the parameters defined as above.
+
+- e2d7b76f43: Upgrade git-url-parse to 12.0.0.
+
+  Motivation for upgrade is transitively upgrading parse-url which is vulnerable
+  to several CVEs detected by Snyk.
+
+  - SNYK-JS-PARSEURL-2935944
+  - SNYK-JS-PARSEURL-2935947
+  - SNYK-JS-PARSEURL-2936249
+
+- f1dcc6f3c6: Use entity type predicates from catalog-model
+- 9a6aba1d85: Many symbol declarations have been moved to `@backstage/plugin-catalog-node`. This has no affect on users of this package as they are all re-exported. Modules that build on top of the catalog backend plugin should switch all of their imports to the `@backstage/plugin-catalog-node` package and remove the dependency on `@backstage/plugin-catalog-backend`.
+- Updated dependencies
+  - @backstage/backend-plugin-api@0.1.0
+  - @backstage/backend-common@0.14.1
+  - @backstage/catalog-model@1.1.0
+  - @backstage/plugin-search-common@1.0.0
+  - @backstage/plugin-catalog-node@1.0.0
+  - @backstage/integration@1.2.2
+  - @backstage/catalog-client@1.0.4
+  - @backstage/plugin-permission-common@0.6.3
+  - @backstage/plugin-permission-node@0.6.3
+  - @backstage/errors@1.1.0
+  - @backstage/plugin-catalog-common@1.0.4
+  - @backstage/plugin-scaffolder-common@1.1.2
+
+## 1.3.0-next.3
+
+### Minor Changes
+
+- 1dd6c22cc8: Added an option to be able to trigger refreshes on entities based on a prestored arbitrary key.
+
+  The UrlReaderProcessor, FileReaderProcessor got updated to store the absolute URL of the catalog file as a refresh key. In the format of `<type>:<target>`
+  The PlaceholderProcessor got updated to store the resolverValues as refreshKeys for the entities.
+
+  The custom resolvers will need to be updated to pass in a `CatalogProcessorEmit` function as parameter and they should be updated to emit their refresh processingResults. You can see the updated resolvers in the `PlaceholderProcessor.ts`
+
+  ```ts
+    // yamlPlaceholderResolver
+    ...
+    const { content, url } = await readTextLocation(params);
+
+    params.emit(processingResult.refresh(`url:${url}`));
+    ...
+  ```
+
+- 91c1d12123: Export experimental `catalogPlugin` for the new backend system. This export is not considered stable and should not be used in production.
+
+### Patch Changes
+
+- 1e02fe46d6: Fixed bug where catalog metrics weren't being tracked.
+- 5f6b847c15: Fix Error Code in Register Component DryRun
+- a70869e775: Updated dependency `msw` to `^0.43.0`.
+- 4e9a90e307: Updated dependency `luxon` to `^3.0.0`.
+- 72622d9143: Updated dependency `yaml` to `^2.0.0`.
+- fa0533e604: CatalogBuilder supports now subscription to processing engine errors.
+
+  ```ts
+  subscribe(options: {
+    onProcessingError: (event: { unprocessedEntity: Entity, error: Error }) => Promise<void> | void;
+  });
+  ```
+
+  If you want to get notified on errors while processing the entities, you call CatalogBuilder.subscribe
+  to get notifications with the parameters defined as above.
+
+- 9a6aba1d85: Many symbol declarations have been moved to `@backstage/plugin-catalog-node`. This has no affect on users of this package as they are all re-exported. Modules that build on top of the catalog backend plugin should switch all of their imports to the `@backstage/plugin-catalog-node` package and remove the dependency on `@backstage/plugin-catalog-backend`.
+- Updated dependencies
+  - @backstage/backend-plugin-api@0.1.0-next.0
+  - @backstage/plugin-catalog-node@1.0.0-next.0
+  - @backstage/backend-common@0.14.1-next.3
+  - @backstage/catalog-client@1.0.4-next.2
+  - @backstage/integration@1.2.2-next.3
+  - @backstage/plugin-permission-common@0.6.3-next.1
+  - @backstage/plugin-permission-node@0.6.3-next.2
+  - @backstage/catalog-model@1.1.0-next.3
+
+## 1.2.1-next.2
+
+### Patch Changes
+
+- 679b32172e: Updated dependency `knex` to `^2.0.0`.
+- e2d7b76f43: Upgrade git-url-parse to 12.0.0.
+
+  Motivation for upgrade is transitively upgrading parse-url which is vulnerable
+  to several CVEs detected by Snyk.
+
+  - SNYK-JS-PARSEURL-2935944
+  - SNYK-JS-PARSEURL-2935947
+  - SNYK-JS-PARSEURL-2936249
+
+- Updated dependencies
+  - @backstage/catalog-model@1.1.0-next.2
+  - @backstage/backend-common@0.14.1-next.2
+  - @backstage/integration@1.2.2-next.2
+
 ## 1.2.1-next.1
 
 ### Patch Changes

package/alpha/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend",
-  "version": "1.2.1-next.1",
+  "version": "1.3.0",
   "main": "../dist/index.cjs.js",
   "types": "../dist/index.alpha.d.ts"
 }

package/dist/index.alpha.d.ts

@@ -6,18 +6,34 @@
 
 /// <reference types="node" />
 
+import { BackendRegistrable } from '@backstage/backend-plugin-api';
 import { CatalogApi } from '@backstage/catalog-client';
 import { CatalogEntityDocument } from '@backstage/plugin-catalog-common';
-import { CompoundEntityRef } from '@backstage/catalog-model';
+import { CatalogProcessor } from '@backstage/plugin-catalog-node';
+import { CatalogProcessorCache } from '@backstage/plugin-catalog-node';
+import { CatalogProcessorEmit } from '@backstage/plugin-catalog-node';
+import { CatalogProcessorEntityResult } from '@backstage/plugin-catalog-node';
+import { CatalogProcessorErrorResult } from '@backstage/plugin-catalog-node';
+import { CatalogProcessorLocationResult } from '@backstage/plugin-catalog-node';
+import { CatalogProcessorParser } from '@backstage/plugin-catalog-node';
+import { CatalogProcessorRefreshKeysResult } from '@backstage/plugin-catalog-node';
+import { CatalogProcessorRelationResult } from '@backstage/plugin-catalog-node';
+import { CatalogProcessorResult } from '@backstage/plugin-catalog-node';
 import { ConditionalPolicyDecision } from '@backstage/plugin-permission-common';
 import { Conditions } from '@backstage/plugin-permission-node';
 import { Config } from '@backstage/config';
+import { DeferredEntity } from '@backstage/plugin-catalog-node';
 import { DocumentCollatorFactory } from '@backstage/plugin-search-common';
 import { Entity } from '@backstage/catalog-model';
 import { EntityPolicy } from '@backstage/catalog-model';
+import { EntityProvider } from '@backstage/plugin-catalog-node';
+import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
+import { EntityProviderMutation } from '@backstage/plugin-catalog-node';
+import { EntityRelationSpec } from '@backstage/plugin-catalog-node';
 import { GetEntitiesRequest } from '@backstage/catalog-client';
 import { JsonValue } from '@backstage/types';
 import { LocationEntityV1alpha1 } from '@backstage/catalog-model';
+import { LocationSpec } from '@backstage/plugin-catalog-node';
 import { Logger } from 'winston';
 import { Permission } from '@backstage/plugin-permission-common';
 import { PermissionAuthorizer } from '@backstage/plugin-permission-common';
@@ -27,6 +43,7 @@ import { PermissionEvaluator } from '@backstage/plugin-permission-common';
 import { PermissionRule } from '@backstage/plugin-permission-node';
 import { PluginDatabaseManager } from '@backstage/backend-common';
 import { PluginEndpointDiscovery } from '@backstage/backend-common';
+import { processingResult } from '@backstage/plugin-catalog-node';
 import { Readable } from 'stream';
 import { ResourcePermission } from '@backstage/plugin-permission-common';
 import { Router } from 'express';
@@ -151,6 +168,7 @@ export declare class CatalogBuilder {
     private processors;
     private processorsReplace;
     private parser;
+    private onProcessingError?;
     private processingInterval;
     private locationAnalyzer;
     private permissionRules;
@@ -282,6 +300,12 @@ export declare class CatalogBuilder {
         processingEngine: CatalogProcessingEngine;
         router: Router;
     }>;
+    subscribe(options: {
+        onProcessingError: (event: {
+            unprocessedEntity: Entity;
+            errors: Error[];
+        }) => Promise<void> | void;
+    }): void;
     private buildEntityPolicy;
     private buildProcessors;
     private checkDeprecatedReaderProcessors;
@@ -321,143 +345,37 @@ export declare type CatalogEnvironment = {
  */
 export declare type CatalogPermissionRule<TParams extends unknown[] = unknown[]> = PermissionRule<Entity, EntitiesSearchFilter, 'catalog-entity', TParams>;
 
+/**
+ * Catalog plugin
+ * @alpha
+ */
+export declare const catalogPlugin: (option: unknown) => BackendRegistrable;
+
 /** @public */
 export declare interface CatalogProcessingEngine {
     start(): Promise<void>;
     stop(): Promise<void>;
 }
 
-/**
- * @public
- */
-export declare type CatalogProcessor = {
-    /**
-     * A unique identifier for the Catalog Processor.
-     */
-    getProcessorName(): string;
-    /**
-     * Reads the contents of a location.
-     *
-     * @param location - The location to read
-     * @param optional - Whether a missing target should trigger an error
-     * @param emit - A sink for items resulting from the read
-     * @param parser - A parser, that is able to take the raw catalog descriptor
-     *               data and turn it into the actual result pieces.
-     * @param cache - A cache for storing values local to this processor and the current entity.
-     * @returns True if handled by this processor, false otherwise
-     */
-    readLocation?(location: LocationSpec, optional: boolean, emit: CatalogProcessorEmit, parser: CatalogProcessorParser, cache: CatalogProcessorCache): Promise<boolean>;
-    /**
-     * Pre-processes an emitted entity, after it has been emitted but before it
-     * has been validated.
-     *
-     * This type of processing usually involves enriching the entity with
-     * additional data, and the input entity may actually still be incomplete
-     * when the processor is invoked.
-     *
-     * @param entity - The (possibly partial) entity to process
-     * @param location - The location that the entity came from
-     * @param emit - A sink for auxiliary items resulting from the processing
-     * @param originLocation - The location that the entity originally came from.
-     *   While location resolves to the direct parent location, originLocation
-     *   tells which location was used to start the ingestion loop.
-     * @param cache - A cache for storing values local to this processor and the current entity.
-     * @returns The same entity or a modified version of it
-     */
-    preProcessEntity?(entity: Entity, location: LocationSpec, emit: CatalogProcessorEmit, originLocation: LocationSpec, cache: CatalogProcessorCache): Promise<Entity>;
-    /**
-     * Validates the entity as a known entity kind, after it has been pre-
-     * processed and has passed through basic overall validation.
-     *
-     * @param entity - The entity to validate
-     * @returns Resolves to true, if the entity was of a kind that was known and
-     *   handled by this processor, and was found to be valid. Resolves to false,
-     *   if the entity was not of a kind that was known by this processor.
-     *   Rejects to an Error describing the problem, if the entity was of a kind
-     *   that was known by this processor and was not valid.
-     */
-    validateEntityKind?(entity: Entity): Promise<boolean>;
-    /**
-     * Post-processes an emitted entity, after it has been validated.
-     *
-     * @param entity - The entity to process
-     * @param location - The location that the entity came from
-     * @param emit - A sink for auxiliary items resulting from the processing
-     * @param cache - A cache for storing values local to this processor and the current entity.
-     * @returns The same entity or a modified version of it
-     */
-    postProcessEntity?(entity: Entity, location: LocationSpec, emit: CatalogProcessorEmit, cache: CatalogProcessorCache): Promise<Entity>;
-};
+export { CatalogProcessor }
 
-/**
- * A cache for storing data during processing.
- *
- * The values stored in the cache are always local to each processor, meaning
- * no processor can see cache values from other processors.
- *
- * The cache instance provided to the CatalogProcessor is also scoped to the
- * entity being processed, meaning that each processor run can't see cache
- * values from processing runs for other entities.
- *
- * Values that are set during a processing run will only be visible in the directly
- * following run. The cache will be overwritten every run unless no new cache items
- * are written, in which case the existing values remain in the cache.
- *
- * @public
- */
-export declare interface CatalogProcessorCache {
-    /**
-     * Retrieve a value from the cache.
-     */
-    get<ItemType extends JsonValue>(key: string): Promise<ItemType | undefined>;
-    /**
-     * Store a value in the cache.
-     */
-    set<ItemType extends JsonValue>(key: string, value: ItemType): Promise<void>;
-}
+export { CatalogProcessorCache }
 
-/** @public */
-export declare type CatalogProcessorEmit = (generated: CatalogProcessorResult) => void;
+export { CatalogProcessorEmit }
 
-/** @public */
-export declare type CatalogProcessorEntityResult = {
-    type: 'entity';
-    entity: Entity;
-    location: LocationSpec;
-};
+export { CatalogProcessorEntityResult }
 
-/** @public */
-export declare type CatalogProcessorErrorResult = {
-    type: 'error';
-    error: Error;
-    location: LocationSpec;
-};
+export { CatalogProcessorErrorResult }
 
-/** @public */
-export declare type CatalogProcessorLocationResult = {
-    type: 'location';
-    location: LocationSpec;
-};
+export { CatalogProcessorLocationResult }
 
-/**
- * A parser, that is able to take the raw catalog descriptor data and turn it
- * into the actual result pieces. The default implementation performs a YAML
- * document parsing.
- * @public
- */
-export declare type CatalogProcessorParser = (options: {
-    data: Buffer;
-    location: LocationSpec;
-}) => AsyncIterable<CatalogProcessorResult>;
+export { CatalogProcessorParser }
 
-/** @public */
-export declare type CatalogProcessorRelationResult = {
-    type: 'relation';
-    relation: EntityRelationSpec;
-};
+export { CatalogProcessorRefreshKeysResult }
 
-/** @public */
-export declare type CatalogProcessorResult = CatalogProcessorLocationResult | CatalogProcessorEntityResult | CatalogProcessorRelationResult | CatalogProcessorErrorResult;
+export { CatalogProcessorRelationResult }
+
+export { CatalogProcessorResult }
 
 /** @public */
 export declare class CodeOwnersProcessor implements CatalogProcessor {
@@ -585,14 +503,7 @@ export declare type DefaultCatalogCollatorFactoryOptions = {
     catalogClient?: CatalogApi;
 };
 
-/**
- * Entities that are not yet processed.
- * @public
- */
-export declare type DeferredEntity = {
-    entity: Entity;
-    locationKey?: string;
-};
+export { DeferredEntity }
 
 /**
  * Matches rows in the search table.
@@ -629,64 +540,13 @@ export declare type EntityFilter = {
     not: EntityFilter;
 } | EntitiesSearchFilter;
 
-/**
- * An EntityProvider is able to provide entities to the catalog.
- * See https://backstage.io/docs/features/software-catalog/life-of-an-entity for more details.
- * @public
- */
-export declare interface EntityProvider {
-    /** Unique provider name used internally for caching. */
-    getProviderName(): string;
-    /** Connect is called upon initialization by the catalog engine. */
-    connect(connection: EntityProviderConnection): Promise<void>;
-}
+export { EntityProvider }
 
-/**
- * The EntityProviderConnection is the connection between the catalog and the entity provider.
- * The EntityProvider use this connection to add and remove entities from the catalog.
- * @public
- */
-export declare interface EntityProviderConnection {
-    /**
-     * Applies either a full or delta update to the catalog engine.
-     */
-    applyMutation(mutation: EntityProviderMutation): Promise<void>;
-}
+export { EntityProviderConnection }
 
-/**
- * @public
- * A 'full' mutation replaces all existing entities created by this entity provider with new ones.
- * A 'delta' mutation can both add and remove entities provided by this provider. Previously provided
- * entities from a 'full' mutation are not removed.
- */
-export declare type EntityProviderMutation = {
-    type: 'full';
-    entities: DeferredEntity[];
-} | {
-    type: 'delta';
-    added: DeferredEntity[];
-    removed: DeferredEntity[];
-};
+export { EntityProviderMutation }
 
-/**
- * Holds the relation data for entities.
- *
- * @public
- */
-export declare type EntityRelationSpec = {
-    /**
-     * The source entity of this relation.
-     */
-    source: CompoundEntityRef;
-    /**
-     * The type of the relation.
-     */
-    type: string;
-    /**
-     * The target entity of this relation.
-     */
-    target: CompoundEntityRef;
-};
+export { EntityRelationSpec }
 
 /** @public */
 export declare class FileReaderProcessor implements CatalogProcessor {
@@ -718,22 +578,7 @@ export declare type LocationEntityProcessorOptions = {
     integrations: ScmIntegrationRegistry;
 };
 
-/**
- * Holds the entity location information.
- *
- * @remarks
- *
- *  `presence` flag: when using repo importer plugin, location is being created before the component yaml file is merged to the main branch.
- *  This flag is then set to indicate that the file can be not present.
- *  default value: 'required'.
- *
- * @public
- */
-export declare type LocationSpec = {
-    type: string;
-    target: string;
-    presence?: 'optional' | 'required';
-};
+export { LocationSpec }
 
 /** @public */
 export declare function locationSpecToLocationEntity(opts: {
@@ -768,7 +613,7 @@ export declare class PlaceholderProcessor implements CatalogProcessor {
     private readonly options;
     constructor(options: PlaceholderProcessorOptions);
     getProcessorName(): string;
-    preProcessEntity(entity: Entity, location: LocationSpec): Promise<Entity>;
+    preProcessEntity(entity: Entity, location: LocationSpec, emit: CatalogProcessorEmit): Promise<Entity>;
 }
 
 /** @public */
@@ -788,6 +633,7 @@ export declare type PlaceholderResolverParams = {
     baseUrl: string;
     read: PlaceholderResolverRead;
     resolveUrl: PlaceholderResolverResolveUrl;
+    emit: CatalogProcessorEmit;
 };
 
 /** @public */
@@ -802,19 +648,7 @@ export declare type PlaceholderResolverResolveUrl = (url: string, base: string)
  */
 export declare type ProcessingIntervalFunction = () => number;
 
-/**
- * Factory functions for the standard processing result types.
- *
- * @public
- */
-export declare const processingResult: Readonly<{
-    readonly notFoundError: (atLocation: LocationSpec, message: string) => CatalogProcessorResult;
-    readonly inputError: (atLocation: LocationSpec, message: string) => CatalogProcessorResult;
-    readonly generalError: (atLocation: LocationSpec, message: string) => CatalogProcessorResult;
-    readonly location: (newLocation: LocationSpec) => CatalogProcessorResult;
-    readonly entity: (atLocation: LocationSpec, newEntity: Entity) => CatalogProcessorResult;
-    readonly relation: (spec: EntityRelationSpec) => CatalogProcessorResult;
-}>;
+export { processingResult }
 
 /**
  * Makes all keys of an entire hierarchy optional.