@backstage/plugin-search-backend-node 0.4.6 → 0.5.1-next.0

package/CHANGELOG.md

@@ -1,5 +1,44 @@
 # @backstage/plugin-search-backend-node
 
+## 0.5.1-next.0
+
+### Patch Changes
+
+- 3e54f6c436: Use `@backstage/plugin-search-common` package instead of `@backstage/search-common`.
+- Updated dependencies
+  - @backstage/plugin-search-common@0.3.1-next.0
+
+## 0.5.0
+
+### Minor Changes
+
+- 022507c860: **BREAKING**
+
+  The Backstage Search Platform's indexing process has been rewritten as a stream
+  pipeline in order to improve efficiency and performance on large document sets.
+
+  The concepts of `Collator` and `Decorator` have been replaced with readable and
+  transform object streams (respectively), as well as factory classes to
+  instantiate them. Accordingly, the `SearchEngine.index()` method has also been
+  replaced with a `getIndexer()` factory method that resolves to a writable
+  object stream.
+
+  Check [this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta)
+  for further details.
+
+### Patch Changes
+
+- Updated dependencies
+  - @backstage/search-common@0.3.0
+
+## 0.4.7
+
+### Patch Changes
+
+- Fix for the previous release with missing type declarations.
+- Updated dependencies
+  - @backstage/search-common@0.2.4
+
 ## 0.4.6
 
 ### Patch Changes

package/dist/index.cjs.js

@@ -2,7 +2,9 @@
 
 Object.defineProperty(exports, '__esModule', { value: true });
 
+var stream = require('stream');
 var lunr = require('lunr');
+var errors = require('@backstage/errors');
 
 function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
 
@@ -23,26 +25,26 @@ class IndexBuilder {
     return this.documentTypes;
   }
   addCollator({
-    collator,
+    factory,
     defaultRefreshIntervalSeconds
   }) {
-    this.logger.info(`Added ${collator.constructor.name} collator for type ${collator.type}`);
-    this.collators[collator.type] = {
+    this.logger.info(`Added ${factory.constructor.name} collator factory for type ${factory.type}`);
+    this.collators[factory.type] = {
       refreshInterval: defaultRefreshIntervalSeconds,
-      collate: collator
+      factory
     };
-    this.documentTypes[collator.type] = {
-      visibilityPermission: collator.visibilityPermission
+    this.documentTypes[factory.type] = {
+      visibilityPermission: factory.visibilityPermission
     };
   }
-  addDecorator({ decorator }) {
-    const types = decorator.types || ["*"];
-    this.logger.info(`Added decorator ${decorator.constructor.name} to types ${types.join(", ")}`);
+  addDecorator({ factory }) {
+    const types = factory.types || ["*"];
+    this.logger.info(`Added decorator ${factory.constructor.name} to types ${types.join(", ")}`);
     types.forEach((type) => {
       if (this.decorators.hasOwnProperty(type)) {
-        this.decorators[type].push(decorator);
+        this.decorators[type].push(factory);
       } else {
-        this.decorators[type] = [decorator];
+        this.decorators[type] = [factory];
       }
     });
   }
@@ -50,29 +52,24 @@ class IndexBuilder {
     const scheduler = new Scheduler({ logger: this.logger });
     Object.keys(this.collators).forEach((type) => {
       scheduler.addToSchedule(async () => {
-        const decorators = (this.decorators["*"] || []).concat(this.decorators[type] || []);
-        this.logger.debug(`Collating documents for ${type} via ${this.collators[type].collate.constructor.name}`);
-        let documents;
-        try {
-          documents = await this.collators[type].collate.execute();
-        } catch (e) {
-          this.logger.error(`Collating documents for ${type} via ${this.collators[type].collate.constructor.name} failed: ${e}`);
-          return;
-        }
-        for (let i = 0; i < decorators.length; i++) {
-          this.logger.debug(`Decorating ${type} documents via ${decorators[i].constructor.name}`);
-          try {
-            documents = await decorators[i].execute(documents);
-          } catch (e) {
-            this.logger.error(`Decorating ${type} documents via ${decorators[i].constructor.name} failed: ${e}`);
-            return;
-          }
-        }
-        if (!documents || documents.length === 0) {
-          this.logger.debug(`No documents for type "${type}" to index`);
-          return;
-        }
-        await this.searchEngine.index(type, documents);
+        const collator = await this.collators[type].factory.getCollator();
+        this.logger.info(`Collating documents for ${type} via ${this.collators[type].factory.constructor.name}`);
+        const decorators = await Promise.all((this.decorators["*"] || []).concat(this.decorators[type] || []).map(async (factory) => {
+          const decorator = await factory.getDecorator();
+          this.logger.info(`Attached decorator via ${factory.constructor.name} to ${type} index pipeline.`);
+          return decorator;
+        }));
+        const indexer = await this.searchEngine.getIndexer(type);
+        return new Promise((done) => {
+          stream.pipeline([collator, ...decorators, indexer], (error) => {
+            if (error) {
+              this.logger.error(`Collating documents for ${type} failed: ${error}`);
+            } else {
+              this.logger.info(`Collating documents for ${type} succeeded`);
+            }
+            done();
+          });
+        });
       }, this.collators[type].refreshInterval * 1e3);
     });
     return {
@@ -133,6 +130,144 @@ class Scheduler {
   }
 }
 
+class BatchSearchEngineIndexer extends stream.Writable {
+  constructor(options) {
+    super({ objectMode: true });
+    this.currentBatch = [];
+    this.batchSize = options.batchSize;
+    this.initialized = new Promise((done) => {
+      setImmediate(async () => {
+        try {
+          await this.initialize();
+          done(void 0);
+        } catch (e) {
+          errors.assertError(e);
+          done(e);
+        }
+      });
+    });
+  }
+  async _write(doc, _e, done) {
+    const maybeError = await this.initialized;
+    if (maybeError) {
+      done(maybeError);
+      return;
+    }
+    this.currentBatch.push(doc);
+    if (this.currentBatch.length < this.batchSize) {
+      done();
+      return;
+    }
+    try {
+      await this.index(this.currentBatch);
+      this.currentBatch = [];
+      done();
+    } catch (e) {
+      errors.assertError(e);
+      done(e);
+    }
+  }
+  async _final(done) {
+    try {
+      if (this.currentBatch.length) {
+        await this.index(this.currentBatch);
+        this.currentBatch = [];
+      }
+      await this.finalize();
+      done();
+    } catch (e) {
+      errors.assertError(e);
+      done(e);
+    }
+  }
+}
+
+class DecoratorBase extends stream.Transform {
+  constructor() {
+    super({ objectMode: true });
+    this.initialized = new Promise((done) => {
+      setImmediate(async () => {
+        try {
+          await this.initialize();
+          done(void 0);
+        } catch (e) {
+          errors.assertError(e);
+          done(e);
+        }
+      });
+    });
+  }
+  async _transform(document, _, done) {
+    const maybeError = await this.initialized;
+    if (maybeError) {
+      done(maybeError);
+      return;
+    }
+    try {
+      const decorated = await this.decorate(document);
+      if (decorated === void 0) {
+        done();
+        return;
+      }
+      if (Array.isArray(decorated)) {
+        decorated.forEach((doc) => {
+          this.push(doc);
+        });
+        done();
+        return;
+      }
+      this.push(decorated);
+      done();
+    } catch (e) {
+      errors.assertError(e);
+      done(e);
+    }
+  }
+  async _final(done) {
+    try {
+      await this.finalize();
+      done();
+    } catch (e) {
+      errors.assertError(e);
+      done(e);
+    }
+  }
+}
+
+class LunrSearchEngineIndexer extends BatchSearchEngineIndexer {
+  constructor() {
+    super({ batchSize: 100 });
+    this.schemaInitialized = false;
+    this.docStore = {};
+    this.builder = new lunr__default["default"].Builder();
+    this.builder.pipeline.add(lunr__default["default"].trimmer, lunr__default["default"].stopWordFilter, lunr__default["default"].stemmer);
+    this.builder.searchPipeline.add(lunr__default["default"].stemmer);
+  }
+  async initialize() {
+  }
+  async finalize() {
+  }
+  async index(documents) {
+    if (!this.schemaInitialized) {
+      Object.keys(documents[0]).forEach((field) => {
+        this.builder.field(field);
+      });
+      this.builder.ref("location");
+      this.schemaInitialized = true;
+    }
+    documents.forEach((document) => {
+      this.builder.add(document);
+      this.docStore[document.location] = document;
+    });
+  }
+  buildIndex() {
+    return this.builder.build();
+  }
+  getDocumentStore() {
+    return this.docStore;
+  }
+}
+
 class LunrSearchEngine {
   constructor({ logger }) {
     this.lunrIndices = {};
@@ -192,19 +327,13 @@ class LunrSearchEngine {
   setTranslator(translator) {
     this.translator = translator;
   }
-  async index(type, documents) {
-    const lunrBuilder = new lunr__default["default"].Builder();
-    lunrBuilder.pipeline.add(lunr__default["default"].trimmer, lunr__default["default"].stopWordFilter, lunr__default["default"].stemmer);
-    lunrBuilder.searchPipeline.add(lunr__default["default"].stemmer);
-    Object.keys(documents[0]).forEach((field) => {
-      lunrBuilder.field(field);
+  async getIndexer(type) {
+    const indexer = new LunrSearchEngineIndexer();
+    indexer.on("close", () => {
+      this.lunrIndices[type] = indexer.buildIndex();
+      this.docStore = { ...this.docStore, ...indexer.getDocumentStore() };
     });
-    lunrBuilder.ref("location");
-    documents.forEach((document) => {
-      lunrBuilder.add(document);
-      this.docStore[document.location] = document;
-    });
-    this.lunrIndices[type] = lunrBuilder.build();
+    return indexer;
   }
   async query(query) {
     const { lunrQueryBuilder, documentTypes, pageSize } = this.translator(query);
@@ -255,7 +384,75 @@ function encodePageCursor({ page }) {
   return Buffer.from(`${page}`, "utf-8").toString("base64");
 }
 
+class TestPipeline {
+  constructor({
+    collator,
+    decorator,
+    indexer
+  }) {
+    this.collator = collator;
+    this.decorator = decorator;
+    this.indexer = indexer;
+  }
+  static withSubject(subject) {
+    if (subject instanceof stream.Transform) {
+      return new TestPipeline({ decorator: subject });
+    }
+    if (subject instanceof stream.Readable) {
+      return new TestPipeline({ collator: subject });
+    }
+    if (subject instanceof stream.Writable) {
+      return new TestPipeline({ indexer: subject });
+    }
+    throw new Error("Unknown test subject: are you passing a readable, writable, or transform stream?");
+  }
+  withDocuments(documents) {
+    if (this.collator) {
+      throw new Error("Cannot provide documents when testing a collator.");
+    }
+    this.collator = new stream.Readable({ objectMode: true });
+    this.collator._read = () => {
+    };
+    process.nextTick(() => {
+      documents.forEach((document) => {
+        this.collator.push(document);
+      });
+      this.collator.push(null);
+    });
+    return this;
+  }
+  async execute() {
+    const documents = [];
+    if (!this.collator) {
+      throw new Error("Cannot execute pipeline without a collator or documents");
+    }
+    if (!this.indexer) {
+      this.indexer = new stream.Writable({ objectMode: true });
+      this.indexer._write = (document, _, done) => {
+        documents.push(document);
+        done();
+      };
+    }
+    return new Promise((done) => {
+      const pipes = [this.collator];
+      if (this.decorator) {
+        pipes.push(this.decorator);
+      }
+      pipes.push(this.indexer);
+      stream.pipeline(pipes, (error) => {
+        done({
+          error,
+          documents
+        });
+      });
+    });
+  }
+}
+
+exports.BatchSearchEngineIndexer = BatchSearchEngineIndexer;
+exports.DecoratorBase = DecoratorBase;
 exports.IndexBuilder = IndexBuilder;
 exports.LunrSearchEngine = LunrSearchEngine;
 exports.Scheduler = Scheduler;
+exports.TestPipeline = TestPipeline;
 //# sourceMappingURL=index.cjs.js.map

package/dist/index.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.cjs.js","sources":["../src/IndexBuilder.ts","../src/runPeriodically.ts","../src/Scheduler.ts","../src/engines/LunrSearchEngine.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  DocumentCollator,\n  DocumentDecorator,\n  DocumentTypeInfo,\n  IndexableDocument,\n  SearchEngine,\n} from '@backstage/search-common';\nimport { Logger } from 'winston';\nimport { Scheduler } from './index';\nimport {\n  RegisterCollatorParameters,\n  RegisterDecoratorParameters,\n} from './types';\n\ninterface CollatorEnvelope {\n  collate: DocumentCollator;\n  refreshInterval: number;\n}\n\ntype IndexBuilderOptions = {\n  searchEngine: SearchEngine;\n  logger: Logger;\n};\n\nexport class IndexBuilder {\n  private collators: Record<string, CollatorEnvelope>;\n  private decorators: Record<string, DocumentDecorator[]>;\n  private documentTypes: Record<string, DocumentTypeInfo>;\n  private searchEngine: SearchEngine;\n  private logger: Logger;\n\n  constructor({ logger, searchEngine }: IndexBuilderOptions) {\n    this.collators = {};\n    this.decorators = {};\n    this.documentTypes = {};\n    this.logger = logger;\n    this.searchEngine = searchEngine;\n  }\n\n  getSearchEngine(): SearchEngine {\n    return this.searchEngine;\n  }\n\n  getDocumentTypes(): Record<string, DocumentTypeInfo> {\n    return this.documentTypes;\n  }\n\n  /**\n   * Makes the index builder aware of a collator that should be executed at the\n   * given refresh interval.\n   */\n  addCollator({\n    collator,\n    defaultRefreshIntervalSeconds,\n  }: RegisterCollatorParameters): void {\n    this.logger.info(\n      `Added ${collator.constructor.name} collator for type ${collator.type}`,\n    );\n    this.collators[collator.type] = {\n      refreshInterval: defaultRefreshIntervalSeconds,\n      collate: collator,\n    };\n    this.documentTypes[collator.type] = {\n      visibilityPermission: collator.visibilityPermission,\n    };\n  }\n\n  /**\n   * Makes the index builder aware of a decorator. If no types are provided on\n   * the decorator, it will be applied to documents from all known collators,\n   * otherwise it will only be applied to documents of the given types.\n   */\n  addDecorator({ decorator }: RegisterDecoratorParameters): void {\n    const types = decorator.types || ['*'];\n    this.logger.info(\n      `Added decorator ${decorator.constructor.name} to types ${types.join(\n        ', ',\n      )}`,\n    );\n    types.forEach(type => {\n      if (this.decorators.hasOwnProperty(type)) {\n        this.decorators[type].push(decorator);\n      } else {\n        this.decorators[type] = [decorator];\n      }\n    });\n  }\n\n  /**\n   * Compiles collators and decorators into tasks, which are added to a\n   * scheduler returned to the caller.\n   */\n  async build(): Promise<{ scheduler: Scheduler }> {\n    const scheduler = new Scheduler({ logger: this.logger });\n\n    Object.keys(this.collators).forEach(type => {\n      scheduler.addToSchedule(async () => {\n        // Collate, Decorate, Index.\n        const decorators: DocumentDecorator[] = (\n          this.decorators['*'] || []\n        ).concat(this.decorators[type] || []);\n\n        this.logger.debug(\n          `Collating documents for ${type} via ${this.collators[type].collate.constructor.name}`,\n        );\n        let documents: IndexableDocument[];\n\n        try {\n          documents = await this.collators[type].collate.execute();\n        } catch (e) {\n          this.logger.error(\n            `Collating documents for ${type} via ${this.collators[type].collate.constructor.name} failed: ${e}`,\n          );\n          return;\n        }\n\n        for (let i = 0; i < decorators.length; i++) {\n          this.logger.debug(\n            `Decorating ${type} documents via ${decorators[i].constructor.name}`,\n          );\n          try {\n            documents = await decorators[i].execute(documents);\n          } catch (e) {\n            this.logger.error(\n              `Decorating ${type} documents via ${decorators[i].constructor.name} failed: ${e}`,\n            );\n            return;\n          }\n        }\n\n        if (!documents || documents.length === 0) {\n          this.logger.debug(`No documents for type \"${type}\" to index`);\n          return;\n        }\n\n        // pushing documents to index to a configured search engine.\n        await this.searchEngine.index(type, documents);\n      }, this.collators[type].refreshInterval * 1000);\n    });\n\n    return {\n      scheduler,\n    };\n  }\n}\n","/*\n * Copyright 2020 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Runs a function repeatedly, with a fixed wait between invocations.\n *\n * Supports async functions, and silently ignores exceptions and rejections.\n *\n * @param fn - The function to run. May return a Promise.\n * @param delayMs - The delay between a completed function invocation and the\n *                next.\n * @returns A function that, when called, stops the invocation loop.\n */\nexport function runPeriodically(fn: () => any, delayMs: number): () => void {\n  let cancel: () => void;\n  let cancelled = false;\n  const cancellationPromise = new Promise<void>(resolve => {\n    cancel = () => {\n      resolve();\n      cancelled = true;\n    };\n  });\n\n  const startRefresh = async () => {\n    while (!cancelled) {\n      try {\n        await fn();\n      } catch {\n        // ignore intentionally\n      }\n\n      await Promise.race([\n        new Promise(resolve => setTimeout(resolve, delayMs)),\n        cancellationPromise,\n      ]);\n    }\n  };\n  startRefresh();\n\n  return cancel!;\n}\n","/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Logger } from 'winston';\nimport { runPeriodically } from './runPeriodically';\n\ntype TaskEnvelope = {\n  task: Function;\n  interval: number;\n};\n\n/**\n * TODO: coordination, error handling\n */\n\nexport class Scheduler {\n  private logger: Logger;\n  private schedule: TaskEnvelope[];\n  private runningTasks: Function[] = [];\n\n  constructor({ logger }: { logger: Logger }) {\n    this.logger = logger;\n    this.schedule = [];\n  }\n\n  /**\n   * Adds each task and interval to the schedule.\n   * When running the tasks, the scheduler waits at least for the time specified\n   * in the interval once the task was completed, before running it again.\n   */\n  addToSchedule(task: Function, interval: number) {\n    if (this.runningTasks.length) {\n      throw new Error(\n        'Cannot add task to schedule that has already been started.',\n      );\n    }\n    this.schedule.push({ task, interval });\n  }\n\n  /**\n   * Starts the scheduling process for each task\n   */\n  start() {\n    this.logger.info('Starting all scheduled search tasks.');\n    this.schedule.forEach(({ task, interval }) => {\n      this.runningTasks.push(runPeriodically(() => task(), interval));\n    });\n  }\n\n  /**\n   * Stop all scheduled tasks.\n   */\n  stop() {\n    this.logger.info('Stopping all scheduled search tasks.');\n    this.runningTasks.forEach(cancel => {\n      cancel();\n    });\n    this.runningTasks = [];\n  }\n}\n","/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  IndexableDocument,\n  SearchQuery,\n  SearchResultSet,\n  QueryTranslator,\n  SearchEngine,\n} from '@backstage/search-common';\nimport lunr from 'lunr';\nimport { Logger } from 'winston';\n\nexport type ConcreteLunrQuery = {\n  lunrQueryBuilder: lunr.Index.QueryBuilder;\n  documentTypes?: string[];\n  pageSize: number;\n};\n\ntype LunrResultEnvelope = {\n  result: lunr.Index.Result;\n  type: string;\n};\n\ntype LunrQueryTranslator = (query: SearchQuery) => ConcreteLunrQuery;\n\nexport class LunrSearchEngine implements SearchEngine {\n  protected lunrIndices: Record<string, lunr.Index> = {};\n  protected docStore: Record<string, IndexableDocument>;\n  protected logger: Logger;\n\n  constructor({ logger }: { logger: Logger }) {\n    this.logger = logger;\n    this.docStore = {};\n  }\n\n  protected translator: QueryTranslator = ({\n    term,\n    filters,\n    types,\n  }: SearchQuery): ConcreteLunrQuery => {\n    const pageSize = 25;\n\n    return {\n      lunrQueryBuilder: q => {\n        const termToken = lunr.tokenizer(term);\n\n        // Support for typeahead seach is based on https://github.com/olivernn/lunr.js/issues/256#issuecomment-295407852\n        // look for an exact match and apply a large positive boost\n        q.term(termToken, {\n          usePipeline: true,\n          boost: 100,\n        });\n        // look for terms that match the beginning of this term and apply a\n        // medium boost\n        q.term(termToken, {\n          usePipeline: false,\n          boost: 10,\n          wildcard: lunr.Query.wildcard.TRAILING,\n        });\n        // look for terms that match with an edit distance of 2 and apply a\n        // small boost\n        q.term(termToken, {\n          usePipeline: false,\n          editDistance: 2,\n          boost: 1,\n        });\n\n        if (filters) {\n          Object.entries(filters).forEach(([field, fieldValue]) => {\n            if (!q.allFields.includes(field)) {\n              // Throw for unknown field, as this will be a non match\n              throw new Error(`unrecognised field ${field}`);\n            }\n            // Arrays are poorly supported, but we can make it better for single-item arrays,\n            // which should be a common case\n            const value =\n              Array.isArray(fieldValue) && fieldValue.length === 1\n                ? fieldValue[0]\n                : fieldValue;\n\n            // Require that the given field has the given value\n            if (['string', 'number', 'boolean'].includes(typeof value)) {\n              q.term(lunr.tokenizer(value?.toString()), {\n                presence: lunr.Query.presence.REQUIRED,\n                fields: [field],\n              });\n            } else if (Array.isArray(value)) {\n              // Illustrate how multi-value filters could work.\n              // But warn that Lurn supports this poorly.\n              this.logger.warn(\n                `Non-scalar filter value used for field ${field}. Consider using a different Search Engine for better results.`,\n              );\n              q.term(lunr.tokenizer(value), {\n                presence: lunr.Query.presence.OPTIONAL,\n                fields: [field],\n              });\n            } else {\n              // Log a warning or something about unknown filter value\n              this.logger.warn(`Unknown filter type used on field ${field}`);\n            }\n          });\n        }\n      },\n      documentTypes: types,\n      pageSize,\n    };\n  };\n\n  setTranslator(translator: LunrQueryTranslator) {\n    this.translator = translator;\n  }\n\n  async index(type: string, documents: IndexableDocument[]): Promise<void> {\n    const lunrBuilder = new lunr.Builder();\n\n    lunrBuilder.pipeline.add(lunr.trimmer, lunr.stopWordFilter, lunr.stemmer);\n    lunrBuilder.searchPipeline.add(lunr.stemmer);\n\n    // Make this lunr index aware of all relevant fields.\n    Object.keys(documents[0]).forEach(field => {\n      lunrBuilder.field(field);\n    });\n\n    // Set \"location\" field as reference field\n    lunrBuilder.ref('location');\n\n    documents.forEach((document: IndexableDocument) => {\n      // Add document to Lunar index\n      lunrBuilder.add(document);\n      // Store documents in memory to be able to look up document using the ref during query time\n      // This is not how you should implement your SearchEngine implementation! Do not copy!\n      this.docStore[document.location] = document;\n    });\n\n    // \"Rotate\" the index by simply overwriting any existing index of the same name.\n    this.lunrIndices[type] = lunrBuilder.build();\n  }\n\n  async query(query: SearchQuery): Promise<SearchResultSet> {\n    const { lunrQueryBuilder, documentTypes, pageSize } = this.translator(\n      query,\n    ) as ConcreteLunrQuery;\n\n    const results: LunrResultEnvelope[] = [];\n\n    // Iterate over the filtered list of this.lunrIndex keys.\n    Object.keys(this.lunrIndices)\n      .filter(type => !documentTypes || documentTypes.includes(type))\n      .forEach(type => {\n        try {\n          results.push(\n            ...this.lunrIndices[type].query(lunrQueryBuilder).map(result => {\n              return {\n                result: result,\n                type: type,\n              };\n            }),\n          );\n        } catch (err) {\n          // if a field does not exist on a index, we can see that as a no-match\n          if (\n            err instanceof Error &&\n            err.message.startsWith('unrecognised field')\n          ) {\n            return;\n          }\n          throw err;\n        }\n      });\n\n    // Sort results.\n    results.sort((doc1, doc2) => {\n      return doc2.result.score - doc1.result.score;\n    });\n\n    // Perform paging\n    const { page } = decodePageCursor(query.pageCursor);\n    const offset = page * pageSize;\n    const hasPreviousPage = page > 0;\n    const hasNextPage = results.length > offset + pageSize;\n    const nextPageCursor = hasNextPage\n      ? encodePageCursor({ page: page + 1 })\n      : undefined;\n    const previousPageCursor = hasPreviousPage\n      ? encodePageCursor({ page: page - 1 })\n      : undefined;\n\n    // Translate results into SearchResultSet\n    const realResultSet: SearchResultSet = {\n      results: results.slice(offset, offset + pageSize).map(d => {\n        return { type: d.type, document: this.docStore[d.result.ref] };\n      }),\n      nextPageCursor,\n      previousPageCursor,\n    };\n\n    return realResultSet;\n  }\n}\n\nexport function decodePageCursor(pageCursor?: string): { page: number } {\n  if (!pageCursor) {\n    return { page: 0 };\n  }\n\n  return {\n    page: Number(Buffer.from(pageCursor, 'base64').toString('utf-8')),\n  };\n}\n\nexport function encodePageCursor({ page }: { page: number }): string {\n  return Buffer.from(`${page}`, 'utf-8').toString('base64');\n}\n"],"names":["lunr"],"mappings":";;;;;;;;;;mBAwC0B;AAAA,EAOxB,YAAY,EAAE,QAAQ,gBAAqC;AACzD,SAAK,YAAY;AACjB,SAAK,aAAa;AAClB,SAAK,gBAAgB;AACrB,SAAK,SAAS;AACd,SAAK,eAAe;AAAA;AAAA,EAGtB,kBAAgC;AAC9B,WAAO,KAAK;AAAA;AAAA,EAGd,mBAAqD;AACnD,WAAO,KAAK;AAAA;AAAA,EAOd,YAAY;AAAA,IACV;AAAA,IACA;AAAA,KACmC;AACnC,SAAK,OAAO,KACV,SAAS,SAAS,YAAY,0BAA0B,SAAS;AAEnE,SAAK,UAAU,SAAS,QAAQ;AAAA,MAC9B,iBAAiB;AAAA,MACjB,SAAS;AAAA;AAEX,SAAK,cAAc,SAAS,QAAQ;AAAA,MAClC,sBAAsB,SAAS;AAAA;AAAA;AAAA,EASnC,aAAa,EAAE,aAAgD;AAC7D,UAAM,QAAQ,UAAU,SAAS,CAAC;AAClC,SAAK,OAAO,KACV,mBAAmB,UAAU,YAAY,iBAAiB,MAAM,KAC9D;AAGJ,UAAM,QAAQ,UAAQ;AACpB,UAAI,KAAK,WAAW,eAAe,OAAO;AACxC,aAAK,WAAW,MAAM,KAAK;AAAA,aACtB;AACL,aAAK,WAAW,QAAQ,CAAC;AAAA;AAAA;AAAA;AAAA,QASzB,QAA2C;AAC/C,UAAM,YAAY,IAAI,UAAU,EAAE,QAAQ,KAAK;AAE/C,WAAO,KAAK,KAAK,WAAW,QAAQ,UAAQ;AAC1C,gBAAU,cAAc,YAAY;AAElC,cAAM,aACJ,MAAK,WAAW,QAAQ,IACxB,OAAO,KAAK,WAAW,SAAS;AAElC,aAAK,OAAO,MACV,2BAA2B,YAAY,KAAK,UAAU,MAAM,QAAQ,YAAY;AAElF,YAAI;AAEJ,YAAI;AACF,sBAAY,MAAM,KAAK,UAAU,MAAM,QAAQ;AAAA,iBACxC,GAAP;AACA,eAAK,OAAO,MACV,2BAA2B,YAAY,KAAK,UAAU,MAAM,QAAQ,YAAY,gBAAgB;AAElG;AAAA;AAGF,iBAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,eAAK,OAAO,MACV,cAAc,sBAAsB,WAAW,GAAG,YAAY;AAEhE,cAAI;AACF,wBAAY,MAAM,WAAW,GAAG,QAAQ;AAAA,mBACjC,GAAP;AACA,iBAAK,OAAO,MACV,cAAc,sBAAsB,WAAW,GAAG,YAAY,gBAAgB;AAEhF;AAAA;AAAA;AAIJ,YAAI,CAAC,aAAa,UAAU,WAAW,GAAG;AACxC,eAAK,OAAO,MAAM,0BAA0B;AAC5C;AAAA;AAIF,cAAM,KAAK,aAAa,MAAM,MAAM;AAAA,SACnC,KAAK,UAAU,MAAM,kBAAkB;AAAA;AAG5C,WAAO;AAAA,MACL;AAAA;AAAA;AAAA;;yBCnI0B,IAAe,SAA6B;AAC1E,MAAI;AACJ,MAAI,YAAY;AAChB,QAAM,sBAAsB,IAAI,QAAc,aAAW;AACvD,aAAS,MAAM;AACb;AACA,kBAAY;AAAA;AAAA;AAIhB,QAAM,eAAe,YAAY;AAC/B,WAAO,CAAC,WAAW;AACjB,UAAI;AACF,cAAM;AAAA,cACN;AAAA;AAIF,YAAM,QAAQ,KAAK;AAAA,QACjB,IAAI,QAAQ,aAAW,WAAW,SAAS;AAAA,QAC3C;AAAA;AAAA;AAAA;AAIN;AAEA,SAAO;AAAA;;gBCxBc;AAAA,EAKrB,YAAY,EAAE,UAA8B;AAFpC,wBAA2B;AAGjC,SAAK,SAAS;AACd,SAAK,WAAW;AAAA;AAAA,EAQlB,cAAc,MAAgB,UAAkB;AAC9C,QAAI,KAAK,aAAa,QAAQ;AAC5B,YAAM,IAAI,MACR;AAAA;AAGJ,SAAK,SAAS,KAAK,EAAE,MAAM;AAAA;AAAA,EAM7B,QAAQ;AACN,SAAK,OAAO,KAAK;AACjB,SAAK,SAAS,QAAQ,CAAC,EAAE,MAAM,eAAe;AAC5C,WAAK,aAAa,KAAK,gBAAgB,MAAM,QAAQ;AAAA;AAAA;AAAA,EAOzD,OAAO;AACL,SAAK,OAAO,KAAK;AACjB,SAAK,aAAa,QAAQ,YAAU;AAClC;AAAA;AAEF,SAAK,eAAe;AAAA;AAAA;;uBC/B8B;AAAA,EAKpD,YAAY,EAAE,UAA8B;AAJlC,uBAA0C;AAS1C,sBAA8B,CAAC;AAAA,MACvC;AAAA,MACA;AAAA,MACA;AAAA,UACoC;AACpC,YAAM,WAAW;AAEjB,aAAO;AAAA,QACL,kBAAkB,OAAK;AACrB,gBAAM,YAAYA,yBAAK,UAAU;AAIjC,YAAE,KAAK,WAAW;AAAA,YAChB,aAAa;AAAA,YACb,OAAO;AAAA;AAIT,YAAE,KAAK,WAAW;AAAA,YAChB,aAAa;AAAA,YACb,OAAO;AAAA,YACP,UAAUA,yBAAK,MAAM,SAAS;AAAA;AAIhC,YAAE,KAAK,WAAW;AAAA,YAChB,aAAa;AAAA,YACb,cAAc;AAAA,YACd,OAAO;AAAA;AAGT,cAAI,SAAS;AACX,mBAAO,QAAQ,SAAS,QAAQ,CAAC,CAAC,OAAO,gBAAgB;AACvD,kBAAI,CAAC,EAAE,UAAU,SAAS,QAAQ;AAEhC,sBAAM,IAAI,MAAM,sBAAsB;AAAA;AAIxC,oBAAM,QACJ,MAAM,QAAQ,eAAe,WAAW,WAAW,IAC/C,WAAW,KACX;AAGN,kBAAI,CAAC,UAAU,UAAU,WAAW,SAAS,OAAO,QAAQ;AAC1D,kBAAE,KAAKA,yBAAK,UAAU,+BAAO,aAAa;AAAA,kBACxC,UAAUA,yBAAK,MAAM,SAAS;AAAA,kBAC9B,QAAQ,CAAC;AAAA;AAAA,yBAEF,MAAM,QAAQ,QAAQ;AAG/B,qBAAK,OAAO,KACV,0CAA0C;AAE5C,kBAAE,KAAKA,yBAAK,UAAU,QAAQ;AAAA,kBAC5B,UAAUA,yBAAK,MAAM,SAAS;AAAA,kBAC9B,QAAQ,CAAC;AAAA;AAAA,qBAEN;AAEL,qBAAK,OAAO,KAAK,qCAAqC;AAAA;AAAA;AAAA;AAAA;AAAA,QAK9D,eAAe;AAAA,QACf;AAAA;AAAA;AAzEF,SAAK,SAAS;AACd,SAAK,WAAW;AAAA;AAAA,EA4ElB,cAAc,YAAiC;AAC7C,SAAK,aAAa;AAAA;AAAA,QAGd,MAAM,MAAc,WAA+C;AACvE,UAAM,cAAc,IAAIA,yBAAK;AAE7B,gBAAY,SAAS,IAAIA,yBAAK,SAASA,yBAAK,gBAAgBA,yBAAK;AACjE,gBAAY,eAAe,IAAIA,yBAAK;AAGpC,WAAO,KAAK,UAAU,IAAI,QAAQ,WAAS;AACzC,kBAAY,MAAM;AAAA;AAIpB,gBAAY,IAAI;AAEhB,cAAU,QAAQ,CAAC,aAAgC;AAEjD,kBAAY,IAAI;AAGhB,WAAK,SAAS,SAAS,YAAY;AAAA;AAIrC,SAAK,YAAY,QAAQ,YAAY;AAAA;AAAA,QAGjC,MAAM,OAA8C;AACxD,UAAM,EAAE,kBAAkB,eAAe,aAAa,KAAK,WACzD;AAGF,UAAM,UAAgC;AAGtC,WAAO,KAAK,KAAK,aACd,OAAO,UAAQ,CAAC,iBAAiB,cAAc,SAAS,OACxD,QAAQ,UAAQ;AACf,UAAI;AACF,gBAAQ,KACN,GAAG,KAAK,YAAY,MAAM,MAAM,kBAAkB,IAAI,YAAU;AAC9D,iBAAO;AAAA,YACL;AAAA,YACA;AAAA;AAAA;AAAA,eAIC,KAAP;AAEA,YACE,eAAe,SACf,IAAI,QAAQ,WAAW,uBACvB;AACA;AAAA;AAEF,cAAM;AAAA;AAAA;AAKZ,YAAQ,KAAK,CAAC,MAAM,SAAS;AAC3B,aAAO,KAAK,OAAO,QAAQ,KAAK,OAAO;AAAA;AAIzC,UAAM,EAAE,SAAS,iBAAiB,MAAM;AACxC,UAAM,SAAS,OAAO;AACtB,UAAM,kBAAkB,OAAO;AAC/B,UAAM,cAAc,QAAQ,SAAS,SAAS;AAC9C,UAAM,iBAAiB,cACnB,iBAAiB,EAAE,MAAM,OAAO,OAChC;AACJ,UAAM,qBAAqB,kBACvB,iBAAiB,EAAE,MAAM,OAAO,OAChC;AAGJ,UAAM,gBAAiC;AAAA,MACrC,SAAS,QAAQ,MAAM,QAAQ,SAAS,UAAU,IAAI,OAAK;AACzD,eAAO,EAAE,MAAM,EAAE,MAAM,UAAU,KAAK,SAAS,EAAE,OAAO;AAAA;AAAA,MAE1D;AAAA,MACA;AAAA;AAGF,WAAO;AAAA;AAAA;0BAIsB,YAAuC;AACtE,MAAI,CAAC,YAAY;AACf,WAAO,EAAE,MAAM;AAAA;AAGjB,SAAO;AAAA,IACL,MAAM,OAAO,OAAO,KAAK,YAAY,UAAU,SAAS;AAAA;AAAA;0BAI3B,EAAE,QAAkC;AACnE,SAAO,OAAO,KAAK,GAAG,QAAQ,SAAS,SAAS;AAAA;;;;;;"}
+{"version":3,"file":"index.cjs.js","sources":["../src/IndexBuilder.ts","../src/runPeriodically.ts","../src/Scheduler.ts","../src/indexing/BatchSearchEngineIndexer.ts","../src/indexing/DecoratorBase.ts","../src/engines/LunrSearchEngineIndexer.ts","../src/engines/LunrSearchEngine.ts","../src/test-utils/TestPipeline.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  DocumentCollatorFactory,\n  DocumentDecoratorFactory,\n  DocumentTypeInfo,\n  SearchEngine,\n} from '@backstage/plugin-search-common';\nimport { Transform, pipeline } from 'stream';\nimport { Logger } from 'winston';\nimport { Scheduler } from './index';\nimport {\n  IndexBuilderOptions,\n  RegisterCollatorParameters,\n  RegisterDecoratorParameters,\n} from './types';\n\ninterface CollatorEnvelope {\n  factory: DocumentCollatorFactory;\n  refreshInterval: number;\n}\n\n/**\n * @beta\n */\nexport class IndexBuilder {\n  private collators: Record<string, CollatorEnvelope>;\n  private decorators: Record<string, DocumentDecoratorFactory[]>;\n  private documentTypes: Record<string, DocumentTypeInfo>;\n  private searchEngine: SearchEngine;\n  private logger: Logger;\n\n  constructor({ logger, searchEngine }: IndexBuilderOptions) {\n    this.collators = {};\n    this.decorators = {};\n    this.documentTypes = {};\n    this.logger = logger;\n    this.searchEngine = searchEngine;\n  }\n\n  getSearchEngine(): SearchEngine {\n    return this.searchEngine;\n  }\n\n  getDocumentTypes(): Record<string, DocumentTypeInfo> {\n    return this.documentTypes;\n  }\n\n  /**\n   * Makes the index builder aware of a collator that should be executed at the\n   * given refresh interval.\n   */\n  addCollator({\n    factory,\n    defaultRefreshIntervalSeconds,\n  }: RegisterCollatorParameters): void {\n    this.logger.info(\n      `Added ${factory.constructor.name} collator factory for type ${factory.type}`,\n    );\n    this.collators[factory.type] = {\n      refreshInterval: defaultRefreshIntervalSeconds,\n      factory,\n    };\n    this.documentTypes[factory.type] = {\n      visibilityPermission: factory.visibilityPermission,\n    };\n  }\n\n  /**\n   * Makes the index builder aware of a decorator. If no types are provided on\n   * the decorator, it will be applied to documents from all known collators,\n   * otherwise it will only be applied to documents of the given types.\n   */\n  addDecorator({ factory }: RegisterDecoratorParameters): void {\n    const types = factory.types || ['*'];\n    this.logger.info(\n      `Added decorator ${factory.constructor.name} to types ${types.join(\n        ', ',\n      )}`,\n    );\n    types.forEach(type => {\n      if (this.decorators.hasOwnProperty(type)) {\n        this.decorators[type].push(factory);\n      } else {\n        this.decorators[type] = [factory];\n      }\n    });\n  }\n\n  /**\n   * Compiles collators and decorators into tasks, which are added to a\n   * scheduler returned to the caller.\n   */\n  async build(): Promise<{ scheduler: Scheduler }> {\n    const scheduler = new Scheduler({ logger: this.logger });\n\n    Object.keys(this.collators).forEach(type => {\n      scheduler.addToSchedule(async () => {\n        // Instantiate the collator.\n        const collator = await this.collators[type].factory.getCollator();\n        this.logger.info(\n          `Collating documents for ${type} via ${this.collators[type].factory.constructor.name}`,\n        );\n\n        // Instantiate all relevant decorators.\n        const decorators: Transform[] = await Promise.all(\n          (this.decorators['*'] || [])\n            .concat(this.decorators[type] || [])\n            .map(async factory => {\n              const decorator = await factory.getDecorator();\n              this.logger.info(\n                `Attached decorator via ${factory.constructor.name} to ${type} index pipeline.`,\n              );\n              return decorator;\n            }),\n        );\n\n        // Instantiate the indexer.\n        const indexer = await this.searchEngine.getIndexer(type);\n\n        // Compose collator/decorators/indexer into a pipeline\n        return new Promise<void>(done => {\n          pipeline([collator, ...decorators, indexer], error => {\n            if (error) {\n              this.logger.error(\n                `Collating documents for ${type} failed: ${error}`,\n              );\n            } else {\n              this.logger.info(`Collating documents for ${type} succeeded`);\n            }\n\n            // Signal index pipeline completion!\n            done();\n          });\n        });\n      }, this.collators[type].refreshInterval * 1000);\n    });\n\n    return {\n      scheduler,\n    };\n  }\n}\n","/*\n * Copyright 2020 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Runs a function repeatedly, with a fixed wait between invocations.\n *\n * Supports async functions, and silently ignores exceptions and rejections.\n *\n * @param fn - The function to run. May return a Promise.\n * @param delayMs - The delay between a completed function invocation and the\n *                next.\n * @returns A function that, when called, stops the invocation loop.\n */\nexport function runPeriodically(fn: () => any, delayMs: number): () => void {\n  let cancel: () => void;\n  let cancelled = false;\n  const cancellationPromise = new Promise<void>(resolve => {\n    cancel = () => {\n      resolve();\n      cancelled = true;\n    };\n  });\n\n  const startRefresh = async () => {\n    while (!cancelled) {\n      try {\n        await fn();\n      } catch {\n        // ignore intentionally\n      }\n\n      await Promise.race([\n        new Promise(resolve => setTimeout(resolve, delayMs)),\n        cancellationPromise,\n      ]);\n    }\n  };\n  startRefresh();\n\n  return cancel!;\n}\n","/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Logger } from 'winston';\nimport { runPeriodically } from './runPeriodically';\n\ntype TaskEnvelope = {\n  task: Function;\n  interval: number;\n};\n\n/**\n * TODO: coordination, error handling\n */\n\n/**\n * @beta\n */\nexport class Scheduler {\n  private logger: Logger;\n  private schedule: TaskEnvelope[];\n  private runningTasks: Function[] = [];\n\n  constructor({ logger }: { logger: Logger }) {\n    this.logger = logger;\n    this.schedule = [];\n  }\n\n  /**\n   * Adds each task and interval to the schedule.\n   * When running the tasks, the scheduler waits at least for the time specified\n   * in the interval once the task was completed, before running it again.\n   */\n  addToSchedule(task: Function, interval: number) {\n    if (this.runningTasks.length) {\n      throw new Error(\n        'Cannot add task to schedule that has already been started.',\n      );\n    }\n    this.schedule.push({ task, interval });\n  }\n\n  /**\n   * Starts the scheduling process for each task\n   */\n  start() {\n    this.logger.info('Starting all scheduled search tasks.');\n    this.schedule.forEach(({ task, interval }) => {\n      this.runningTasks.push(runPeriodically(() => task(), interval));\n    });\n  }\n\n  /**\n   * Stop all scheduled tasks.\n   */\n  stop() {\n    this.logger.info('Stopping all scheduled search tasks.');\n    this.runningTasks.forEach(cancel => {\n      cancel();\n    });\n    this.runningTasks = [];\n  }\n}\n","/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { assertError } from '@backstage/errors';\nimport { IndexableDocument } from '@backstage/plugin-search-common';\nimport { Writable } from 'stream';\n\n/**\n * @beta\n */\nexport type BatchSearchEngineOptions = {\n  batchSize: number;\n};\n\n/**\n * Base class encapsulating batch-based stream processing. Useful as a base\n * class for search engine indexers.\n * @beta\n */\nexport abstract class BatchSearchEngineIndexer extends Writable {\n  private batchSize: number;\n  private currentBatch: IndexableDocument[] = [];\n  private initialized: Promise<undefined | Error>;\n\n  constructor(options: BatchSearchEngineOptions) {\n    super({ objectMode: true });\n    this.batchSize = options.batchSize;\n\n    // @todo Once node v15 is minimum, convert to _construct implementation.\n    this.initialized = new Promise(done => {\n      // Necessary to allow concrete implementation classes to construct\n      // themselves before calling their initialize() methods.\n      setImmediate(async () => {\n        try {\n          await this.initialize();\n          done(undefined);\n        } catch (e) {\n          assertError(e);\n          done(e);\n        }\n      });\n    });\n  }\n\n  /**\n   * Receives an array of indexable documents (of size this.batchSize) which\n   * should be written to the search engine. This method won't be called again\n   * at least until it resolves.\n   */\n  public abstract index(documents: IndexableDocument[]): Promise<void>;\n\n  /**\n   * Any asynchronous setup tasks can be performed here.\n   */\n  public abstract initialize(): Promise<void>;\n\n  /**\n   * Any asynchronous teardown tasks can be performed here.\n   */\n  public abstract finalize(): Promise<void>;\n\n  /**\n   * Encapsulates batch stream write logic.\n   * @internal\n   */\n  async _write(\n    doc: IndexableDocument,\n    _e: any,\n    done: (error?: Error | null) => void,\n  ) {\n    // Wait for init before proceeding. Throw error if initialization failed.\n    const maybeError = await this.initialized;\n    if (maybeError) {\n      done(maybeError);\n      return;\n    }\n\n    this.currentBatch.push(doc);\n    if (this.currentBatch.length < this.batchSize) {\n      done();\n      return;\n    }\n\n    try {\n      await this.index(this.currentBatch);\n      this.currentBatch = [];\n      done();\n    } catch (e) {\n      assertError(e);\n      done(e);\n    }\n  }\n\n  /**\n   * Encapsulates finalization and final error handling logic.\n   * @internal\n   */\n  async _final(done: (error?: Error | null) => void) {\n    try {\n      // Index any remaining documents.\n      if (this.currentBatch.length) {\n        await this.index(this.currentBatch);\n        this.currentBatch = [];\n      }\n      await this.finalize();\n      done();\n    } catch (e) {\n      assertError(e);\n      done(e);\n    }\n  }\n}\n","/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { assertError } from '@backstage/errors';\nimport { IndexableDocument } from '@backstage/plugin-search-common';\nimport { Transform } from 'stream';\n\n/**\n * Base class encapsulating simple async transformations. Useful as a base\n * class for Backstage search decorators.\n * @beta\n */\nexport abstract class DecoratorBase extends Transform {\n  private initialized: Promise<undefined | Error>;\n\n  constructor() {\n    super({ objectMode: true });\n\n    // @todo Once node v15 is minimum, convert to _construct implementation.\n    this.initialized = new Promise(done => {\n      // Necessary to allow concrete implementation classes to construct\n      // themselves before calling their initialize() methods.\n      setImmediate(async () => {\n        try {\n          await this.initialize();\n          done(undefined);\n        } catch (e) {\n          assertError(e);\n          done(e);\n        }\n      });\n    });\n  }\n\n  /**\n   * Any asynchronous setup tasks can be performed here.\n   */\n  public abstract initialize(): Promise<void>;\n\n  /**\n   * Receives a single indexable document. In your decorate method, you can:\n   *\n   * - Resolve `undefined` to indicate the record should be omitted.\n   * - Resolve a single modified document, which could contain new fields,\n   *   edited fields, or removed fields.\n   * - Resolve an array of indexable documents, if the purpose if the decorator\n   *   is to convert one document into multiple derivative documents.\n   */\n  public abstract decorate(\n    document: IndexableDocument,\n  ): Promise<IndexableDocument | IndexableDocument[] | undefined>;\n\n  /**\n   * Any asynchronous teardown tasks can be performed here.\n   */\n  public abstract finalize(): Promise<void>;\n\n  /**\n   * Encapsulates simple transform stream logic.\n   * @internal\n   */\n  async _transform(\n    document: IndexableDocument,\n    _: any,\n    done: (error?: Error | null) => void,\n  ) {\n    // Wait for init before proceeding. Throw error if initialization failed.\n    const maybeError = await this.initialized;\n    if (maybeError) {\n      done(maybeError);\n      return;\n    }\n\n    try {\n      const decorated = await this.decorate(document);\n\n      // If undefined was returned, omit the record and move on.\n      if (decorated === undefined) {\n        done();\n        return;\n      }\n\n      // If an array of documents was given, push them all.\n      if (Array.isArray(decorated)) {\n        decorated.forEach(doc => {\n          this.push(doc);\n        });\n        done();\n        return;\n      }\n\n      // Otherwise, just push the decorated document.\n      this.push(decorated);\n      done();\n    } catch (e) {\n      assertError(e);\n      done(e);\n    }\n  }\n\n  /**\n   * Encapsulates finalization and final error handling logic.\n   * @internal\n   */\n  async _final(done: (error?: Error | null) => void) {\n    try {\n      await this.finalize();\n      done();\n    } catch (e) {\n      assertError(e);\n      done(e);\n    }\n  }\n}\n","/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { IndexableDocument } from '@backstage/plugin-search-common';\nimport lunr from 'lunr';\nimport { BatchSearchEngineIndexer } from '../indexing';\n\n/**\n * @beta\n */\nexport class LunrSearchEngineIndexer extends BatchSearchEngineIndexer {\n  private schemaInitialized = false;\n  private builder: lunr.Builder;\n  private docStore: Record<string, IndexableDocument> = {};\n\n  constructor() {\n    super({ batchSize: 100 });\n\n    this.builder = new lunr.Builder();\n    this.builder.pipeline.add(lunr.trimmer, lunr.stopWordFilter, lunr.stemmer);\n    this.builder.searchPipeline.add(lunr.stemmer);\n  }\n\n  // No async initialization required.\n  async initialize(): Promise<void> {}\n  async finalize(): Promise<void> {}\n\n  async index(documents: IndexableDocument[]): Promise<void> {\n    if (!this.schemaInitialized) {\n      // Make this lunr index aware of all relevant fields.\n      Object.keys(documents[0]).forEach(field => {\n        this.builder.field(field);\n      });\n\n      // Set \"location\" field as reference field\n      this.builder.ref('location');\n\n      this.schemaInitialized = true;\n    }\n\n    documents.forEach(document => {\n      // Add document to Lunar index\n      this.builder.add(document);\n\n      // Store documents in memory to be able to look up document using the ref during query time\n      // This is not how you should implement your SearchEngine implementation! Do not copy!\n      this.docStore[document.location] = document;\n    });\n  }\n\n  buildIndex() {\n    return this.builder.build();\n  }\n\n  getDocumentStore() {\n    return this.docStore;\n  }\n}\n","/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  IndexableDocument,\n  SearchQuery,\n  SearchResultSet,\n  QueryTranslator,\n  SearchEngine,\n} from '@backstage/plugin-search-common';\nimport lunr from 'lunr';\nimport { Logger } from 'winston';\nimport { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer';\n\n/**\n * @beta\n */\nexport type ConcreteLunrQuery = {\n  lunrQueryBuilder: lunr.Index.QueryBuilder;\n  documentTypes?: string[];\n  pageSize: number;\n};\n\ntype LunrResultEnvelope = {\n  result: lunr.Index.Result;\n  type: string;\n};\n\n/**\n * @beta\n */\nexport type LunrQueryTranslator = (query: SearchQuery) => ConcreteLunrQuery;\n\n/**\n * @beta\n */\nexport class LunrSearchEngine implements SearchEngine {\n  protected lunrIndices: Record<string, lunr.Index> = {};\n  protected docStore: Record<string, IndexableDocument>;\n  protected logger: Logger;\n\n  constructor({ logger }: { logger: Logger }) {\n    this.logger = logger;\n    this.docStore = {};\n  }\n\n  protected translator: QueryTranslator = ({\n    term,\n    filters,\n    types,\n  }: SearchQuery): ConcreteLunrQuery => {\n    const pageSize = 25;\n\n    return {\n      lunrQueryBuilder: q => {\n        const termToken = lunr.tokenizer(term);\n\n        // Support for typeahead seach is based on https://github.com/olivernn/lunr.js/issues/256#issuecomment-295407852\n        // look for an exact match and apply a large positive boost\n        q.term(termToken, {\n          usePipeline: true,\n          boost: 100,\n        });\n        // look for terms that match the beginning of this term and apply a\n        // medium boost\n        q.term(termToken, {\n          usePipeline: false,\n          boost: 10,\n          wildcard: lunr.Query.wildcard.TRAILING,\n        });\n        // look for terms that match with an edit distance of 2 and apply a\n        // small boost\n        q.term(termToken, {\n          usePipeline: false,\n          editDistance: 2,\n          boost: 1,\n        });\n\n        if (filters) {\n          Object.entries(filters).forEach(([field, fieldValue]) => {\n            if (!q.allFields.includes(field)) {\n              // Throw for unknown field, as this will be a non match\n              throw new Error(`unrecognised field ${field}`);\n            }\n            // Arrays are poorly supported, but we can make it better for single-item arrays,\n            // which should be a common case\n            const value =\n              Array.isArray(fieldValue) && fieldValue.length === 1\n                ? fieldValue[0]\n                : fieldValue;\n\n            // Require that the given field has the given value\n            if (['string', 'number', 'boolean'].includes(typeof value)) {\n              q.term(lunr.tokenizer(value?.toString()), {\n                presence: lunr.Query.presence.REQUIRED,\n                fields: [field],\n              });\n            } else if (Array.isArray(value)) {\n              // Illustrate how multi-value filters could work.\n              // But warn that Lurn supports this poorly.\n              this.logger.warn(\n                `Non-scalar filter value used for field ${field}. Consider using a different Search Engine for better results.`,\n              );\n              q.term(lunr.tokenizer(value), {\n                presence: lunr.Query.presence.OPTIONAL,\n                fields: [field],\n              });\n            } else {\n              // Log a warning or something about unknown filter value\n              this.logger.warn(`Unknown filter type used on field ${field}`);\n            }\n          });\n        }\n      },\n      documentTypes: types,\n      pageSize,\n    };\n  };\n\n  setTranslator(translator: LunrQueryTranslator) {\n    this.translator = translator;\n  }\n\n  async getIndexer(type: string) {\n    const indexer = new LunrSearchEngineIndexer();\n\n    indexer.on('close', () => {\n      // Once the stream is closed, build the index and store the documents in\n      // memory for later retrieval.\n      this.lunrIndices[type] = indexer.buildIndex();\n      this.docStore = { ...this.docStore, ...indexer.getDocumentStore() };\n    });\n\n    return indexer;\n  }\n\n  async query(query: SearchQuery): Promise<SearchResultSet> {\n    const { lunrQueryBuilder, documentTypes, pageSize } = this.translator(\n      query,\n    ) as ConcreteLunrQuery;\n\n    const results: LunrResultEnvelope[] = [];\n\n    // Iterate over the filtered list of this.lunrIndex keys.\n    Object.keys(this.lunrIndices)\n      .filter(type => !documentTypes || documentTypes.includes(type))\n      .forEach(type => {\n        try {\n          results.push(\n            ...this.lunrIndices[type].query(lunrQueryBuilder).map(result => {\n              return {\n                result: result,\n                type: type,\n              };\n            }),\n          );\n        } catch (err) {\n          // if a field does not exist on a index, we can see that as a no-match\n          if (\n            err instanceof Error &&\n            err.message.startsWith('unrecognised field')\n          ) {\n            return;\n          }\n          throw err;\n        }\n      });\n\n    // Sort results.\n    results.sort((doc1, doc2) => {\n      return doc2.result.score - doc1.result.score;\n    });\n\n    // Perform paging\n    const { page } = decodePageCursor(query.pageCursor);\n    const offset = page * pageSize;\n    const hasPreviousPage = page > 0;\n    const hasNextPage = results.length > offset + pageSize;\n    const nextPageCursor = hasNextPage\n      ? encodePageCursor({ page: page + 1 })\n      : undefined;\n    const previousPageCursor = hasPreviousPage\n      ? encodePageCursor({ page: page - 1 })\n      : undefined;\n\n    // Translate results into SearchResultSet\n    const realResultSet: SearchResultSet = {\n      results: results.slice(offset, offset + pageSize).map(d => {\n        return { type: d.type, document: this.docStore[d.result.ref] };\n      }),\n      nextPageCursor,\n      previousPageCursor,\n    };\n\n    return realResultSet;\n  }\n}\n\nexport function decodePageCursor(pageCursor?: string): { page: number } {\n  if (!pageCursor) {\n    return { page: 0 };\n  }\n\n  return {\n    page: Number(Buffer.from(pageCursor, 'base64').toString('utf-8')),\n  };\n}\n\nexport function encodePageCursor({ page }: { page: number }): string {\n  return Buffer.from(`${page}`, 'utf-8').toString('base64');\n}\n","/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { IndexableDocument } from '@backstage/plugin-search-common';\nimport { pipeline, Readable, Transform, Writable } from 'stream';\n\n/**\n * Object resolved after a test pipeline is executed.\n * @beta\n */\nexport type TestPipelineResult = {\n  /**\n   * If an error was emitted by the pipeline, it will be set here.\n   */\n  error: unknown;\n\n  /**\n   * A list of documents collected at the end of the pipeline. If the subject\n   * under test is an indexer, this will be an empty array (because your\n   * indexer should have received the documents instead).\n   */\n  documents: IndexableDocument[];\n};\n\n/**\n * Test utility for Backstage Search collators, decorators, and indexers.\n * @beta\n */\nexport class TestPipeline {\n  private collator?: Readable;\n  private decorator?: Transform;\n  private indexer?: Writable;\n\n  private constructor({\n    collator,\n    decorator,\n    indexer,\n  }: {\n    collator?: Readable;\n    decorator?: Transform;\n    indexer?: Writable;\n  }) {\n    this.collator = collator;\n    this.decorator = decorator;\n    this.indexer = indexer;\n  }\n\n  /**\n   * Provide the collator, decorator, or indexer to be tested.\n   */\n  static withSubject(subject: Readable | Transform | Writable) {\n    if (subject instanceof Transform) {\n      return new TestPipeline({ decorator: subject });\n    }\n\n    if (subject instanceof Readable) {\n      return new TestPipeline({ collator: subject });\n    }\n\n    if (subject instanceof Writable) {\n      return new TestPipeline({ indexer: subject });\n    }\n\n    throw new Error(\n      'Unknown test subject: are you passing a readable, writable, or transform stream?',\n    );\n  }\n\n  /**\n   * Provide documents for testing decorators and indexers.\n   */\n  withDocuments(documents: IndexableDocument[]): TestPipeline {\n    if (this.collator) {\n      throw new Error('Cannot provide documents when testing a collator.');\n    }\n\n    // Set a naive readable stream that just pushes all given documents.\n    this.collator = new Readable({ objectMode: true });\n    this.collator._read = () => {};\n    process.nextTick(() => {\n      documents.forEach(document => {\n        this.collator!.push(document);\n      });\n      this.collator!.push(null);\n    });\n\n    return this;\n  }\n\n  /**\n   * Execute the test pipeline so that you can make assertions about the result\n   * or behavior of the given test subject.\n   */\n  async execute(): Promise<TestPipelineResult> {\n    const documents: IndexableDocument[] = [];\n    if (!this.collator) {\n      throw new Error(\n        'Cannot execute pipeline without a collator or documents',\n      );\n    }\n\n    // If we are here and there is no indexer, we are testing a collator or a\n    // decorator. Set up a naive writable that captures documents in memory.\n    if (!this.indexer) {\n      this.indexer = new Writable({ objectMode: true });\n      this.indexer._write = (document: IndexableDocument, _, done) => {\n        documents.push(document);\n        done();\n      };\n    }\n\n    return new Promise<TestPipelineResult>(done => {\n      const pipes: (Readable | Transform | Writable)[] = [this.collator!];\n      if (this.decorator) {\n        pipes.push(this.decorator);\n      }\n      pipes.push(this.indexer!);\n\n      pipeline(pipes, error => {\n        done({\n          error,\n          documents,\n        });\n      });\n    });\n  }\n}\n"],"names":["Writable","Transform","lunr","Readable"],"mappings":";;;;;;;;;;;;mBAuC0B;AAAA,EAOxB,YAAY,EAAE,QAAQ,gBAAqC;AACzD,SAAK,YAAY;AACjB,SAAK,aAAa;AAClB,SAAK,gBAAgB;AACrB,SAAK,SAAS;AACd,SAAK,eAAe;AAAA;AAAA,EAGtB,kBAAgC;AAC9B,WAAO,KAAK;AAAA;AAAA,EAGd,mBAAqD;AACnD,WAAO,KAAK;AAAA;AAAA,EAOd,YAAY;AAAA,IACV;AAAA,IACA;AAAA,KACmC;AACnC,SAAK,OAAO,KACV,SAAS,QAAQ,YAAY,kCAAkC,QAAQ;AAEzE,SAAK,UAAU,QAAQ,QAAQ;AAAA,MAC7B,iBAAiB;AAAA,MACjB;AAAA;AAEF,SAAK,cAAc,QAAQ,QAAQ;AAAA,MACjC,sBAAsB,QAAQ;AAAA;AAAA;AAAA,EASlC,aAAa,EAAE,WAA8C;AAC3D,UAAM,QAAQ,QAAQ,SAAS,CAAC;AAChC,SAAK,OAAO,KACV,mBAAmB,QAAQ,YAAY,iBAAiB,MAAM,KAC5D;AAGJ,UAAM,QAAQ,UAAQ;AACpB,UAAI,KAAK,WAAW,eAAe,OAAO;AACxC,aAAK,WAAW,MAAM,KAAK;AAAA,aACtB;AACL,aAAK,WAAW,QAAQ,CAAC;AAAA;AAAA;AAAA;AAAA,QASzB,QAA2C;AAC/C,UAAM,YAAY,IAAI,UAAU,EAAE,QAAQ,KAAK;AAE/C,WAAO,KAAK,KAAK,WAAW,QAAQ,UAAQ;AAC1C,gBAAU,cAAc,YAAY;AAElC,cAAM,WAAW,MAAM,KAAK,UAAU,MAAM,QAAQ;AACpD,aAAK,OAAO,KACV,2BAA2B,YAAY,KAAK,UAAU,MAAM,QAAQ,YAAY;AAIlF,cAAM,aAA0B,MAAM,QAAQ,IAC3C,MAAK,WAAW,QAAQ,IACtB,OAAO,KAAK,WAAW,SAAS,IAChC,IAAI,OAAM,YAAW;AACpB,gBAAM,YAAY,MAAM,QAAQ;AAChC,eAAK,OAAO,KACV,0BAA0B,QAAQ,YAAY,WAAW;AAE3D,iBAAO;AAAA;AAKb,cAAM,UAAU,MAAM,KAAK,aAAa,WAAW;AAGnD,eAAO,IAAI,QAAc,UAAQ;AAC/B,0BAAS,CAAC,UAAU,GAAG,YAAY,UAAU,WAAS;AACpD,gBAAI,OAAO;AACT,mBAAK,OAAO,MACV,2BAA2B,gBAAgB;AAAA,mBAExC;AACL,mBAAK,OAAO,KAAK,2BAA2B;AAAA;AAI9C;AAAA;AAAA;AAAA,SAGH,KAAK,UAAU,MAAM,kBAAkB;AAAA;AAG5C,WAAO;AAAA,MACL;AAAA;AAAA;AAAA;;yBC/H0B,IAAe,SAA6B;AAC1E,MAAI;AACJ,MAAI,YAAY;AAChB,QAAM,sBAAsB,IAAI,QAAc,aAAW;AACvD,aAAS,MAAM;AACb;AACA,kBAAY;AAAA;AAAA;AAIhB,QAAM,eAAe,YAAY;AAC/B,WAAO,CAAC,WAAW;AACjB,UAAI;AACF,cAAM;AAAA,cACN;AAAA;AAIF,YAAM,QAAQ,KAAK;AAAA,QACjB,IAAI,QAAQ,aAAW,WAAW,SAAS;AAAA,QAC3C;AAAA;AAAA;AAAA;AAIN;AAEA,SAAO;AAAA;;gBCrBc;AAAA,EAKrB,YAAY,EAAE,UAA8B;AAFpC,wBAA2B;AAGjC,SAAK,SAAS;AACd,SAAK,WAAW;AAAA;AAAA,EAQlB,cAAc,MAAgB,UAAkB;AAC9C,QAAI,KAAK,aAAa,QAAQ;AAC5B,YAAM,IAAI,MACR;AAAA;AAGJ,SAAK,SAAS,KAAK,EAAE,MAAM;AAAA;AAAA,EAM7B,QAAQ;AACN,SAAK,OAAO,KAAK;AACjB,SAAK,SAAS,QAAQ,CAAC,EAAE,MAAM,eAAe;AAC5C,WAAK,aAAa,KAAK,gBAAgB,MAAM,QAAQ;AAAA;AAAA;AAAA,EAOzD,OAAO;AACL,SAAK,OAAO,KAAK;AACjB,SAAK,aAAa,QAAQ,YAAU;AAClC;AAAA;AAEF,SAAK,eAAe;AAAA;AAAA;;uCCzC+BA,gBAAS;AAAA,EAK9D,YAAY,SAAmC;AAC7C,UAAM,EAAE,YAAY;AAJd,wBAAoC;AAK1C,SAAK,YAAY,QAAQ;AAGzB,SAAK,cAAc,IAAI,QAAQ,UAAQ;AAGrC,mBAAa,YAAY;AACvB,YAAI;AACF,gBAAM,KAAK;AACX,eAAK;AAAA,iBACE,GAAP;AACA,6BAAY;AACZ,eAAK;AAAA;AAAA;AAAA;AAAA;AAAA,QA2BP,OACJ,KACA,IACA,MACA;AAEA,UAAM,aAAa,MAAM,KAAK;AAC9B,QAAI,YAAY;AACd,WAAK;AACL;AAAA;AAGF,SAAK,aAAa,KAAK;AACvB,QAAI,KAAK,aAAa,SAAS,KAAK,WAAW;AAC7C;AACA;AAAA;AAGF,QAAI;AACF,YAAM,KAAK,MAAM,KAAK;AACtB,WAAK,eAAe;AACpB;AAAA,aACO,GAAP;AACA,yBAAY;AACZ,WAAK;AAAA;AAAA;AAAA,QAQH,OAAO,MAAsC;AACjD,QAAI;AAEF,UAAI,KAAK,aAAa,QAAQ;AAC5B,cAAM,KAAK,MAAM,KAAK;AACtB,aAAK,eAAe;AAAA;AAEtB,YAAM,KAAK;AACX;AAAA,aACO,GAAP;AACA,yBAAY;AACZ,WAAK;AAAA;AAAA;AAAA;;4BChGiCC,iBAAU;AAAA,EAGpD,cAAc;AACZ,UAAM,EAAE,YAAY;AAGpB,SAAK,cAAc,IAAI,QAAQ,UAAQ;AAGrC,mBAAa,YAAY;AACvB,YAAI;AACF,gBAAM,KAAK;AACX,eAAK;AAAA,iBACE,GAAP;AACA,6BAAY;AACZ,eAAK;AAAA;AAAA;AAAA;AAAA;AAAA,QAiCP,WACJ,UACA,GACA,MACA;AAEA,UAAM,aAAa,MAAM,KAAK;AAC9B,QAAI,YAAY;AACd,WAAK;AACL;AAAA;AAGF,QAAI;AACF,YAAM,YAAY,MAAM,KAAK,SAAS;AAGtC,UAAI,cAAc,QAAW;AAC3B;AACA;AAAA;AAIF,UAAI,MAAM,QAAQ,YAAY;AAC5B,kBAAU,QAAQ,SAAO;AACvB,eAAK,KAAK;AAAA;AAEZ;AACA;AAAA;AAIF,WAAK,KAAK;AACV;AAAA,aACO,GAAP;AACA,yBAAY;AACZ,WAAK;AAAA;AAAA;AAAA,QAQH,OAAO,MAAsC;AACjD,QAAI;AACF,YAAM,KAAK;AACX;AAAA,aACO,GAAP;AACA,yBAAY;AACZ,WAAK;AAAA;AAAA;AAAA;;sCCpGkC,yBAAyB;AAAA,EAKpE,cAAc;AACZ,UAAM,EAAE,WAAW;AALb,6BAAoB;AAEpB,oBAA8C;AAKpD,SAAK,UAAU,IAAIC,yBAAK;AACxB,SAAK,QAAQ,SAAS,IAAIA,yBAAK,SAASA,yBAAK,gBAAgBA,yBAAK;AAClE,SAAK,QAAQ,eAAe,IAAIA,yBAAK;AAAA;AAAA,QAIjC,aAA4B;AAAA;AAAA,QAC5B,WAA0B;AAAA;AAAA,QAE1B,MAAM,WAA+C;AACzD,QAAI,CAAC,KAAK,mBAAmB;AAE3B,aAAO,KAAK,UAAU,IAAI,QAAQ,WAAS;AACzC,aAAK,QAAQ,MAAM;AAAA;AAIrB,WAAK,QAAQ,IAAI;AAEjB,WAAK,oBAAoB;AAAA;AAG3B,cAAU,QAAQ,cAAY;AAE5B,WAAK,QAAQ,IAAI;AAIjB,WAAK,SAAS,SAAS,YAAY;AAAA;AAAA;AAAA,EAIvC,aAAa;AACX,WAAO,KAAK,QAAQ;AAAA;AAAA,EAGtB,mBAAmB;AACjB,WAAO,KAAK;AAAA;AAAA;;uBCnBsC;AAAA,EAKpD,YAAY,EAAE,UAA8B;AAJlC,uBAA0C;AAS1C,sBAA8B,CAAC;AAAA,MACvC;AAAA,MACA;AAAA,MACA;AAAA,UACoC;AACpC,YAAM,WAAW;AAEjB,aAAO;AAAA,QACL,kBAAkB,OAAK;AACrB,gBAAM,YAAYA,yBAAK,UAAU;AAIjC,YAAE,KAAK,WAAW;AAAA,YAChB,aAAa;AAAA,YACb,OAAO;AAAA;AAIT,YAAE,KAAK,WAAW;AAAA,YAChB,aAAa;AAAA,YACb,OAAO;AAAA,YACP,UAAUA,yBAAK,MAAM,SAAS;AAAA;AAIhC,YAAE,KAAK,WAAW;AAAA,YAChB,aAAa;AAAA,YACb,cAAc;AAAA,YACd,OAAO;AAAA;AAGT,cAAI,SAAS;AACX,mBAAO,QAAQ,SAAS,QAAQ,CAAC,CAAC,OAAO,gBAAgB;AACvD,kBAAI,CAAC,EAAE,UAAU,SAAS,QAAQ;AAEhC,sBAAM,IAAI,MAAM,sBAAsB;AAAA;AAIxC,oBAAM,QACJ,MAAM,QAAQ,eAAe,WAAW,WAAW,IAC/C,WAAW,KACX;AAGN,kBAAI,CAAC,UAAU,UAAU,WAAW,SAAS,OAAO,QAAQ;AAC1D,kBAAE,KAAKA,yBAAK,UAAU,+BAAO,aAAa;AAAA,kBACxC,UAAUA,yBAAK,MAAM,SAAS;AAAA,kBAC9B,QAAQ,CAAC;AAAA;AAAA,yBAEF,MAAM,QAAQ,QAAQ;AAG/B,qBAAK,OAAO,KACV,0CAA0C;AAE5C,kBAAE,KAAKA,yBAAK,UAAU,QAAQ;AAAA,kBAC5B,UAAUA,yBAAK,MAAM,SAAS;AAAA,kBAC9B,QAAQ,CAAC;AAAA;AAAA,qBAEN;AAEL,qBAAK,OAAO,KAAK,qCAAqC;AAAA;AAAA;AAAA;AAAA;AAAA,QAK9D,eAAe;AAAA,QACf;AAAA;AAAA;AAzEF,SAAK,SAAS;AACd,SAAK,WAAW;AAAA;AAAA,EA4ElB,cAAc,YAAiC;AAC7C,SAAK,aAAa;AAAA;AAAA,QAGd,WAAW,MAAc;AAC7B,UAAM,UAAU,IAAI;AAEpB,YAAQ,GAAG,SAAS,MAAM;AAGxB,WAAK,YAAY,QAAQ,QAAQ;AACjC,WAAK,WAAW,KAAK,KAAK,aAAa,QAAQ;AAAA;AAGjD,WAAO;AAAA;AAAA,QAGH,MAAM,OAA8C;AACxD,UAAM,EAAE,kBAAkB,eAAe,aAAa,KAAK,WACzD;AAGF,UAAM,UAAgC;AAGtC,WAAO,KAAK,KAAK,aACd,OAAO,UAAQ,CAAC,iBAAiB,cAAc,SAAS,OACxD,QAAQ,UAAQ;AACf,UAAI;AACF,gBAAQ,KACN,GAAG,KAAK,YAAY,MAAM,MAAM,kBAAkB,IAAI,YAAU;AAC9D,iBAAO;AAAA,YACL;AAAA,YACA;AAAA;AAAA;AAAA,eAIC,KAAP;AAEA,YACE,eAAe,SACf,IAAI,QAAQ,WAAW,uBACvB;AACA;AAAA;AAEF,cAAM;AAAA;AAAA;AAKZ,YAAQ,KAAK,CAAC,MAAM,SAAS;AAC3B,aAAO,KAAK,OAAO,QAAQ,KAAK,OAAO;AAAA;AAIzC,UAAM,EAAE,SAAS,iBAAiB,MAAM;AACxC,UAAM,SAAS,OAAO;AACtB,UAAM,kBAAkB,OAAO;AAC/B,UAAM,cAAc,QAAQ,SAAS,SAAS;AAC9C,UAAM,iBAAiB,cACnB,iBAAiB,EAAE,MAAM,OAAO,OAChC;AACJ,UAAM,qBAAqB,kBACvB,iBAAiB,EAAE,MAAM,OAAO,OAChC;AAGJ,UAAM,gBAAiC;AAAA,MACrC,SAAS,QAAQ,MAAM,QAAQ,SAAS,UAAU,IAAI,OAAK;AACzD,eAAO,EAAE,MAAM,EAAE,MAAM,UAAU,KAAK,SAAS,EAAE,OAAO;AAAA;AAAA,MAE1D;AAAA,MACA;AAAA;AAGF,WAAO;AAAA;AAAA;0BAIsB,YAAuC;AACtE,MAAI,CAAC,YAAY;AACf,WAAO,EAAE,MAAM;AAAA;AAGjB,SAAO;AAAA,IACL,MAAM,OAAO,OAAO,KAAK,YAAY,UAAU,SAAS;AAAA;AAAA;0BAI3B,EAAE,QAAkC;AACnE,SAAO,OAAO,KAAK,GAAG,QAAQ,SAAS,SAAS;AAAA;;mBCrLxB;AAAA,EAKhB,YAAY;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,KAKC;AACD,SAAK,WAAW;AAChB,SAAK,YAAY;AACjB,SAAK,UAAU;AAAA;AAAA,SAMV,YAAY,SAA0C;AAC3D,QAAI,mBAAmBD,kBAAW;AAChC,aAAO,IAAI,aAAa,EAAE,WAAW;AAAA;AAGvC,QAAI,mBAAmBE,iBAAU;AAC/B,aAAO,IAAI,aAAa,EAAE,UAAU;AAAA;AAGtC,QAAI,mBAAmBH,iBAAU;AAC/B,aAAO,IAAI,aAAa,EAAE,SAAS;AAAA;AAGrC,UAAM,IAAI,MACR;AAAA;AAAA,EAOJ,cAAc,WAA8C;AAC1D,QAAI,KAAK,UAAU;AACjB,YAAM,IAAI,MAAM;AAAA;AAIlB,SAAK,WAAW,IAAIG,gBAAS,EAAE,YAAY;AAC3C,SAAK,SAAS,QAAQ,MAAM;AAAA;AAC5B,YAAQ,SAAS,MAAM;AACrB,gBAAU,QAAQ,cAAY;AAC5B,aAAK,SAAU,KAAK;AAAA;AAEtB,WAAK,SAAU,KAAK;AAAA;AAGtB,WAAO;AAAA;AAAA,QAOH,UAAuC;AAC3C,UAAM,YAAiC;AACvC,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MACR;AAAA;AAMJ,QAAI,CAAC,KAAK,SAAS;AACjB,WAAK,UAAU,IAAIH,gBAAS,EAAE,YAAY;AAC1C,WAAK,QAAQ,SAAS,CAAC,UAA6B,GAAG,SAAS;AAC9D,kBAAU,KAAK;AACf;AAAA;AAAA;AAIJ,WAAO,IAAI,QAA4B,UAAQ;AAC7C,YAAM,QAA6C,CAAC,KAAK;AACzD,UAAI,KAAK,WAAW;AAClB,cAAM,KAAK,KAAK;AAAA;AAElB,YAAM,KAAK,KAAK;AAEhB,sBAAS,OAAO,WAAS;AACvB,aAAK;AAAA,UACH;AAAA,UACA;AAAA;AAAA;AAAA;AAAA;AAAA;;;;;;;;;"}

package/dist/index.d.ts

@@ -0,0 +1,244 @@
+/// <reference types="node" />
+import { SearchEngine, DocumentCollatorFactory, DocumentDecoratorFactory, DocumentTypeInfo, IndexableDocument, SearchQuery, QueryTranslator, SearchResultSet } from '@backstage/plugin-search-common';
+export { SearchEngine } from '@backstage/plugin-search-common';
+import { Logger } from 'winston';
+import lunr from 'lunr';
+import { Writable, Transform, Readable } from 'stream';
+
+/**
+ * @beta
+ */
+declare type IndexBuilderOptions = {
+    searchEngine: SearchEngine;
+    logger: Logger;
+};
+/**
+ * Parameters required to register a collator.
+ * @beta
+ */
+interface RegisterCollatorParameters {
+    /**
+     * The default interval (in seconds) that the provided collator will be called (can be overridden in config).
+     */
+    defaultRefreshIntervalSeconds: number;
+    /**
+     * The class responsible for returning the document collator of the given type.
+     */
+    factory: DocumentCollatorFactory;
+}
+/**
+ * Parameters required to register a decorator
+ * @beta
+ */
+interface RegisterDecoratorParameters {
+    /**
+     * The class responsible for returning the decorator which appends, modifies, or filters documents.
+     */
+    factory: DocumentDecoratorFactory;
+}
+
+/**
+ * @beta
+ */
+declare class IndexBuilder {
+    private collators;
+    private decorators;
+    private documentTypes;
+    private searchEngine;
+    private logger;
+    constructor({ logger, searchEngine }: IndexBuilderOptions);
+    getSearchEngine(): SearchEngine;
+    getDocumentTypes(): Record<string, DocumentTypeInfo>;
+    /**
+     * Makes the index builder aware of a collator that should be executed at the
+     * given refresh interval.
+     */
+    addCollator({ factory, defaultRefreshIntervalSeconds, }: RegisterCollatorParameters): void;
+    /**
+     * Makes the index builder aware of a decorator. If no types are provided on
+     * the decorator, it will be applied to documents from all known collators,
+     * otherwise it will only be applied to documents of the given types.
+     */
+    addDecorator({ factory }: RegisterDecoratorParameters): void;
+    /**
+     * Compiles collators and decorators into tasks, which are added to a
+     * scheduler returned to the caller.
+     */
+    build(): Promise<{
+        scheduler: Scheduler;
+    }>;
+}
+
+/**
+ * TODO: coordination, error handling
+ */
+/**
+ * @beta
+ */
+declare class Scheduler {
+    private logger;
+    private schedule;
+    private runningTasks;
+    constructor({ logger }: {
+        logger: Logger;
+    });
+    /**
+     * Adds each task and interval to the schedule.
+     * When running the tasks, the scheduler waits at least for the time specified
+     * in the interval once the task was completed, before running it again.
+     */
+    addToSchedule(task: Function, interval: number): void;
+    /**
+     * Starts the scheduling process for each task
+     */
+    start(): void;
+    /**
+     * Stop all scheduled tasks.
+     */
+    stop(): void;
+}
+
+/**
+ * @beta
+ */
+declare type BatchSearchEngineOptions = {
+    batchSize: number;
+};
+/**
+ * Base class encapsulating batch-based stream processing. Useful as a base
+ * class for search engine indexers.
+ * @beta
+ */
+declare abstract class BatchSearchEngineIndexer extends Writable {
+    private batchSize;
+    private currentBatch;
+    private initialized;
+    constructor(options: BatchSearchEngineOptions);
+    /**
+     * Receives an array of indexable documents (of size this.batchSize) which
+     * should be written to the search engine. This method won't be called again
+     * at least until it resolves.
+     */
+    abstract index(documents: IndexableDocument[]): Promise<void>;
+    /**
+     * Any asynchronous setup tasks can be performed here.
+     */
+    abstract initialize(): Promise<void>;
+    /**
+     * Any asynchronous teardown tasks can be performed here.
+     */
+    abstract finalize(): Promise<void>;
+}
+
+/**
+ * Base class encapsulating simple async transformations. Useful as a base
+ * class for Backstage search decorators.
+ * @beta
+ */
+declare abstract class DecoratorBase extends Transform {
+    private initialized;
+    constructor();
+    /**
+     * Any asynchronous setup tasks can be performed here.
+     */
+    abstract initialize(): Promise<void>;
+    /**
+     * Receives a single indexable document. In your decorate method, you can:
+     *
+     * - Resolve `undefined` to indicate the record should be omitted.
+     * - Resolve a single modified document, which could contain new fields,
+     *   edited fields, or removed fields.
+     * - Resolve an array of indexable documents, if the purpose if the decorator
+     *   is to convert one document into multiple derivative documents.
+     */
+    abstract decorate(document: IndexableDocument): Promise<IndexableDocument | IndexableDocument[] | undefined>;
+    /**
+     * Any asynchronous teardown tasks can be performed here.
+     */
+    abstract finalize(): Promise<void>;
+}
+
+/**
+ * @beta
+ */
+declare class LunrSearchEngineIndexer extends BatchSearchEngineIndexer {
+    private schemaInitialized;
+    private builder;
+    private docStore;
+    constructor();
+    initialize(): Promise<void>;
+    finalize(): Promise<void>;
+    index(documents: IndexableDocument[]): Promise<void>;
+    buildIndex(): lunr.Index;
+    getDocumentStore(): Record<string, IndexableDocument>;
+}
+
+/**
+ * @beta
+ */
+declare type ConcreteLunrQuery = {
+    lunrQueryBuilder: lunr.Index.QueryBuilder;
+    documentTypes?: string[];
+    pageSize: number;
+};
+/**
+ * @beta
+ */
+declare type LunrQueryTranslator = (query: SearchQuery) => ConcreteLunrQuery;
+/**
+ * @beta
+ */
+declare class LunrSearchEngine implements SearchEngine {
+    protected lunrIndices: Record<string, lunr.Index>;
+    protected docStore: Record<string, IndexableDocument>;
+    protected logger: Logger;
+    constructor({ logger }: {
+        logger: Logger;
+    });
+    protected translator: QueryTranslator;
+    setTranslator(translator: LunrQueryTranslator): void;
+    getIndexer(type: string): Promise<LunrSearchEngineIndexer>;
+    query(query: SearchQuery): Promise<SearchResultSet>;
+}
+
+/**
+ * Object resolved after a test pipeline is executed.
+ * @beta
+ */
+declare type TestPipelineResult = {
+    /**
+     * If an error was emitted by the pipeline, it will be set here.
+     */
+    error: unknown;
+    /**
+     * A list of documents collected at the end of the pipeline. If the subject
+     * under test is an indexer, this will be an empty array (because your
+     * indexer should have received the documents instead).
+     */
+    documents: IndexableDocument[];
+};
+/**
+ * Test utility for Backstage Search collators, decorators, and indexers.
+ * @beta
+ */
+declare class TestPipeline {
+    private collator?;
+    private decorator?;
+    private indexer?;
+    private constructor();
+    /**
+     * Provide the collator, decorator, or indexer to be tested.
+     */
+    static withSubject(subject: Readable | Transform | Writable): TestPipeline;
+    /**
+     * Provide documents for testing decorators and indexers.
+     */
+    withDocuments(documents: IndexableDocument[]): TestPipeline;
+    /**
+     * Execute the test pipeline so that you can make assertions about the result
+     * or behavior of the given test subject.
+     */
+    execute(): Promise<TestPipelineResult>;
+}
+
+export { BatchSearchEngineIndexer, BatchSearchEngineOptions, ConcreteLunrQuery, DecoratorBase, IndexBuilder, IndexBuilderOptions, LunrQueryTranslator, LunrSearchEngine, LunrSearchEngineIndexer, RegisterCollatorParameters, RegisterDecoratorParameters, Scheduler, TestPipeline, TestPipelineResult };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@backstage/plugin-search-backend-node",
   "description": "A library for Backstage backend plugins that want to interact with the search backend plugin",
-  "version": "0.4.6",
+  "version": "0.5.1-next.0",
   "main": "dist/index.cjs.js",
   "types": "dist/index.d.ts",
   "license": "Apache-2.0",
@@ -23,17 +23,19 @@
     "clean": "backstage-cli package clean"
   },
   "dependencies": {
-    "@backstage/search-common": "^0.2.3",
+    "@backstage/errors": "^0.2.2",
+    "@backstage/plugin-search-common": "^0.3.1-next.0",
     "@types/lunr": "^2.3.3",
+    "lodash": "^4.17.21",
     "lunr": "^2.3.9",
     "winston": "^3.2.1"
   },
   "devDependencies": {
-    "@backstage/backend-common": "^0.10.8",
-    "@backstage/cli": "^0.14.0"
+    "@backstage/backend-common": "^0.13.0-next.0",
+    "@backstage/cli": "^0.15.2-next.0"
   },
   "files": [
     "dist"
   ],
-  "gitHead": "4805c3d13ce9bfc369e53c271b1b95e722b3b4dc"
+  "gitHead": "e90d3ed129ebfce978f1adfa40c1dc2cef3f7e65"
 }