@backstage/plugin-search-backend-node 0.6.1 → 0.6.2-next.2

package/CHANGELOG.md

@@ -1,5 +1,38 @@
 # @backstage/plugin-search-backend-node
 
+## 0.6.2-next.2
+
+### Patch Changes
+
+- 7d8acfc32e: Replaced all `@beta` exports with `@public` exports
+- Updated dependencies
+  - @backstage/plugin-search-common@0.3.5-next.1
+  - @backstage/backend-common@0.14.0-next.2
+  - @backstage/backend-tasks@0.3.2-next.2
+
+## 0.6.2-next.1
+
+### Patch Changes
+
+- 3bb25a9acc: Introducing a `NewlineDelimitedJsonCollatorFactory`, which can be used to create search indices from newline delimited JSON files stored in external storage readable via a configured `UrlReader` instance.
+
+  This is useful if you have an independent process periodically generating `*.ndjson` files consisting of `IndexableDocument` objects and want to be able to generate a fresh index based on the latest version of such a file.
+
+- 3bb25a9acc: Fixed a bug that prevented `TestPipeline.withSubject` from identifying valid `Readable` subjects that were technically transform streams.
+- Updated dependencies
+  - @backstage/backend-tasks@0.3.2-next.1
+  - @backstage/backend-common@0.13.6-next.1
+  - @backstage/plugin-permission-common@0.6.2-next.0
+  - @backstage/plugin-search-common@0.3.5-next.0
+
+## 0.6.2-next.0
+
+### Patch Changes
+
+- e7794a0aaa: propagate indexing errors so they don't appear successful to the task scheduler
+- Updated dependencies
+  - @backstage/backend-tasks@0.3.2-next.0
+
 ## 0.6.1
 
 ### Patch Changes

package/dist/index.cjs.js

@@ -4,6 +4,7 @@ Object.defineProperty(exports, '__esModule', { value: true });
 
 var stream = require('stream');
 var nodeAbortController = require('node-abort-controller');
+var ndjson = require('ndjson');
 var lunr = require('lunr');
 var uuid = require('uuid');
 var errors = require('@backstage/errors');
@@ -99,14 +100,15 @@ class IndexBuilder {
             return decorator;
           }));
           const indexer = await this.searchEngine.getIndexer(type);
-          return new Promise((done) => {
+          return new Promise((resolve, reject) => {
             stream.pipeline([collator, ...decorators, indexer], (error) => {
               if (error) {
                 this.logger.error(`Collating documents for ${type} failed: ${error}`);
+                reject(error);
               } else {
                 this.logger.info(`Collating documents for ${type} succeeded`);
+                resolve();
               }
-              done();
             });
           });
         }
@@ -118,6 +120,44 @@ class IndexBuilder {
   }
 }
 
+class NewlineDelimitedJsonCollatorFactory {
+  constructor(type, searchPattern, reader, logger, visibilityPermission) {
+    this.searchPattern = searchPattern;
+    this.reader = reader;
+    this.logger = logger;
+    this.type = type;
+    this.visibilityPermission = visibilityPermission;
+  }
+  static fromConfig(_config, options) {
+    return new NewlineDelimitedJsonCollatorFactory(options.type, options.searchPattern, options.reader, options.logger, options.visibilityPermission);
+  }
+  async lastUrl() {
+    var _a;
+    try {
+      this.logger.info(`Attempting to find latest .ndjson matching ${this.searchPattern}`);
+      const { files } = await this.reader.search(this.searchPattern);
+      const candidates = files.filter((file) => file.url.endsWith(".ndjson")).sort((a, b) => a.url.localeCompare(b.url)).reverse();
+      return (_a = candidates[0]) == null ? void 0 : _a.url;
+    } catch (e) {
+      this.logger.error(`Could not search for ${this.searchPattern}`, e);
+      throw e;
+    }
+  }
+  async getCollator() {
+    const lastUrl = await this.lastUrl();
+    if (!lastUrl) {
+      const noMatchingFile = `Could not find an .ndjson file matching ${this.searchPattern}`;
+      this.logger.error(noMatchingFile);
+      throw new Error(noMatchingFile);
+    } else {
+      this.logger.info(`Using latest .ndjson file ${lastUrl}`);
+    }
+    const readerResponse = await this.reader.readUrl(lastUrl);
+    const stream = readerResponse.stream();
+    return stream.pipe(ndjson.parse());
+  }
+}
+
 class BatchSearchEngineIndexer extends stream.Writable {
   constructor(options) {
     super({ objectMode: true });
@@ -423,12 +463,12 @@ class TestPipeline {
     if (subject instanceof stream.Transform) {
       return new TestPipeline({ decorator: subject });
     }
-    if (subject instanceof stream.Readable) {
-      return new TestPipeline({ collator: subject });
-    }
     if (subject instanceof stream.Writable) {
       return new TestPipeline({ indexer: subject });
     }
+    if (subject.readable || subject instanceof stream.Readable) {
+      return new TestPipeline({ collator: subject });
+    }
     throw new Error("Unknown test subject: are you passing a readable, writable, or transform stream?");
   }
   withDocuments(documents) {
@@ -478,6 +518,7 @@ exports.BatchSearchEngineIndexer = BatchSearchEngineIndexer;
 exports.DecoratorBase = DecoratorBase;
 exports.IndexBuilder = IndexBuilder;
 exports.LunrSearchEngine = LunrSearchEngine;
+exports.NewlineDelimitedJsonCollatorFactory = NewlineDelimitedJsonCollatorFactory;
 exports.Scheduler = Scheduler;
 exports.TestPipeline = TestPipeline;
 //# sourceMappingURL=index.cjs.js.map

package/dist/index.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.cjs.js","sources":["../src/Scheduler.ts","../src/IndexBuilder.ts","../src/indexing/BatchSearchEngineIndexer.ts","../src/indexing/DecoratorBase.ts","../src/engines/LunrSearchEngineIndexer.ts","../src/engines/LunrSearchEngine.ts","../src/test-utils/TestPipeline.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { AbortController } from 'node-abort-controller';\nimport { Logger } from 'winston';\nimport { TaskFunction, TaskRunner } from '@backstage/backend-tasks';\n\ntype TaskEnvelope = {\n  task: TaskFunction;\n  scheduledRunner: TaskRunner;\n};\n\n/**\n * @public ScheduleTaskParameters\n */\nexport interface ScheduleTaskParameters {\n  id: string;\n  task: TaskFunction;\n  scheduledRunner: TaskRunner;\n}\n\n/**\n * @beta\n */\nexport class Scheduler {\n  private logger: Logger;\n  private schedule: { [id: string]: TaskEnvelope };\n  private abortController: AbortController;\n  private isRunning: boolean;\n\n  constructor({ logger }: { logger: Logger }) {\n    this.logger = logger;\n    this.schedule = {};\n    this.abortController = new AbortController();\n    this.isRunning = false;\n  }\n\n  /**\n   * Adds each task and interval to the schedule.\n   * When running the tasks, the scheduler waits at least for the time specified\n   * in the interval once the task was completed, before running it again.\n   */\n  addToSchedule({ id, task, scheduledRunner }: ScheduleTaskParameters) {\n    if (this.isRunning) {\n      throw new Error(\n        'Cannot add task to schedule that has already been started.',\n      );\n    }\n\n    if (this.schedule[id]) {\n      throw new Error(`Task with id ${id} already exists.`);\n    }\n\n    this.schedule[id] = { task, scheduledRunner };\n  }\n\n  /**\n   * Starts the scheduling process for each task\n   */\n  start() {\n    this.logger.info('Starting all scheduled search tasks.');\n    this.isRunning = true;\n    Object.keys(this.schedule).forEach(id => {\n      const { task, scheduledRunner } = this.schedule[id];\n      scheduledRunner.run({\n        id,\n        fn: task,\n        signal: this.abortController.signal,\n      });\n    });\n  }\n\n  /**\n   * Stop all scheduled tasks.\n   */\n  stop() {\n    this.logger.info('Stopping all scheduled search tasks.');\n    this.abortController.abort();\n    this.isRunning = false;\n  }\n}\n","/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nimport {\n  DocumentDecoratorFactory,\n  DocumentTypeInfo,\n  SearchEngine,\n} from '@backstage/plugin-search-common';\nimport { Transform, pipeline } from 'stream';\nimport { Logger } from 'winston';\nimport { Scheduler } from './Scheduler';\nimport {\n  IndexBuilderOptions,\n  RegisterCollatorParameters,\n  RegisterDecoratorParameters,\n} from './types';\n\n/**\n * @beta\n */\nexport class IndexBuilder {\n  private collators: Record<string, RegisterCollatorParameters>;\n  private decorators: Record<string, DocumentDecoratorFactory[]>;\n  private documentTypes: Record<string, DocumentTypeInfo>;\n  private searchEngine: SearchEngine;\n  private logger: Logger;\n\n  constructor({ logger, searchEngine }: IndexBuilderOptions) {\n    this.collators = {};\n    this.decorators = {};\n    this.documentTypes = {};\n    this.logger = logger;\n    this.searchEngine = searchEngine;\n  }\n\n  getSearchEngine(): SearchEngine {\n    return this.searchEngine;\n  }\n\n  getDocumentTypes(): Record<string, DocumentTypeInfo> {\n    return this.documentTypes;\n  }\n\n  /**\n   * Makes the index builder aware of a collator that should be executed at the\n   * given refresh interval.\n   */\n  addCollator({ factory, schedule }: RegisterCollatorParameters): void {\n    this.logger.info(\n      `Added ${factory.constructor.name} collator factory for type ${factory.type}`,\n    );\n    this.collators[factory.type] = {\n      factory,\n      schedule,\n    };\n    this.documentTypes[factory.type] = {\n      visibilityPermission: factory.visibilityPermission,\n    };\n  }\n\n  /**\n   * Makes the index builder aware of a decorator. If no types are provided on\n   * the decorator, it will be applied to documents from all known collators,\n   * otherwise it will only be applied to documents of the given types.\n   */\n  addDecorator({ factory }: RegisterDecoratorParameters): void {\n    const types = factory.types || ['*'];\n    this.logger.info(\n      `Added decorator ${factory.constructor.name} to types ${types.join(\n        ', ',\n      )}`,\n    );\n    types.forEach(type => {\n      if (this.decorators.hasOwnProperty(type)) {\n        this.decorators[type].push(factory);\n      } else {\n        this.decorators[type] = [factory];\n      }\n    });\n  }\n\n  /**\n   * Compiles collators and decorators into tasks, which are added to a\n   * scheduler returned to the caller.\n   */\n  async build(): Promise<{ scheduler: Scheduler }> {\n    const scheduler = new Scheduler({\n      logger: this.logger,\n    });\n\n    Object.keys(this.collators).forEach(type => {\n      scheduler.addToSchedule({\n        id: `search_index_${type.replace('-', '_').toLocaleLowerCase('en-US')}`,\n        scheduledRunner: this.collators[type].schedule,\n        task: async () => {\n          // Instantiate the collator.\n          const collator = await this.collators[type].factory.getCollator();\n          this.logger.info(\n            `Collating documents for ${type} via ${this.collators[type].factory.constructor.name}`,\n          );\n\n          // Instantiate all relevant decorators.\n          const decorators: Transform[] = await Promise.all(\n            (this.decorators['*'] || [])\n              .concat(this.decorators[type] || [])\n              .map(async factory => {\n                const decorator = await factory.getDecorator();\n                this.logger.info(\n                  `Attached decorator via ${factory.constructor.name} to ${type} index pipeline.`,\n                );\n                return decorator;\n              }),\n          );\n\n          // Instantiate the indexer.\n          const indexer = await this.searchEngine.getIndexer(type);\n\n          // Compose collator/decorators/indexer into a pipeline\n          return new Promise<void>(done => {\n            pipeline(\n              [collator, ...decorators, indexer],\n              (error: NodeJS.ErrnoException | null) => {\n                if (error) {\n                  this.logger.error(\n                    `Collating documents for ${type} failed: ${error}`,\n                  );\n                } else {\n                  this.logger.info(`Collating documents for ${type} succeeded`);\n                }\n\n                // Signal index pipeline completion!\n                done();\n              },\n            );\n          });\n        },\n      });\n    });\n\n    return {\n      scheduler,\n    };\n  }\n}\n","/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { assertError } from '@backstage/errors';\nimport { IndexableDocument } from '@backstage/plugin-search-common';\nimport { Writable } from 'stream';\n\n/**\n * @beta\n */\nexport type BatchSearchEngineOptions = {\n  batchSize: number;\n};\n\n/**\n * Base class encapsulating batch-based stream processing. Useful as a base\n * class for search engine indexers.\n * @beta\n */\nexport abstract class BatchSearchEngineIndexer extends Writable {\n  private batchSize: number;\n  private currentBatch: IndexableDocument[] = [];\n  private initialized: Promise<undefined | Error>;\n\n  constructor(options: BatchSearchEngineOptions) {\n    super({ objectMode: true });\n    this.batchSize = options.batchSize;\n\n    // @todo Once node v15 is minimum, convert to _construct implementation.\n    this.initialized = new Promise(done => {\n      // Necessary to allow concrete implementation classes to construct\n      // themselves before calling their initialize() methods.\n      setImmediate(async () => {\n        try {\n          await this.initialize();\n          done(undefined);\n        } catch (e) {\n          assertError(e);\n          done(e);\n        }\n      });\n    });\n  }\n\n  /**\n   * Receives an array of indexable documents (of size this.batchSize) which\n   * should be written to the search engine. This method won't be called again\n   * at least until it resolves.\n   */\n  public abstract index(documents: IndexableDocument[]): Promise<void>;\n\n  /**\n   * Any asynchronous setup tasks can be performed here.\n   */\n  public abstract initialize(): Promise<void>;\n\n  /**\n   * Any asynchronous teardown tasks can be performed here.\n   */\n  public abstract finalize(): Promise<void>;\n\n  /**\n   * Encapsulates batch stream write logic.\n   * @internal\n   */\n  async _write(\n    doc: IndexableDocument,\n    _e: any,\n    done: (error?: Error | null) => void,\n  ) {\n    // Wait for init before proceeding. Throw error if initialization failed.\n    const maybeError = await this.initialized;\n    if (maybeError) {\n      done(maybeError);\n      return;\n    }\n\n    this.currentBatch.push(doc);\n    if (this.currentBatch.length < this.batchSize) {\n      done();\n      return;\n    }\n\n    try {\n      await this.index(this.currentBatch);\n      this.currentBatch = [];\n      done();\n    } catch (e) {\n      assertError(e);\n      done(e);\n    }\n  }\n\n  /**\n   * Encapsulates finalization and final error handling logic.\n   * @internal\n   */\n  async _final(done: (error?: Error | null) => void) {\n    try {\n      // Index any remaining documents.\n      if (this.currentBatch.length) {\n        await this.index(this.currentBatch);\n        this.currentBatch = [];\n      }\n      await this.finalize();\n      done();\n    } catch (e) {\n      assertError(e);\n      done(e);\n    }\n  }\n}\n","/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { assertError } from '@backstage/errors';\nimport { IndexableDocument } from '@backstage/plugin-search-common';\nimport { Transform } from 'stream';\n\n/**\n * Base class encapsulating simple async transformations. Useful as a base\n * class for Backstage search decorators.\n * @beta\n */\nexport abstract class DecoratorBase extends Transform {\n  private initialized: Promise<undefined | Error>;\n\n  constructor() {\n    super({ objectMode: true });\n\n    // @todo Once node v15 is minimum, convert to _construct implementation.\n    this.initialized = new Promise(done => {\n      // Necessary to allow concrete implementation classes to construct\n      // themselves before calling their initialize() methods.\n      setImmediate(async () => {\n        try {\n          await this.initialize();\n          done(undefined);\n        } catch (e) {\n          assertError(e);\n          done(e);\n        }\n      });\n    });\n  }\n\n  /**\n   * Any asynchronous setup tasks can be performed here.\n   */\n  public abstract initialize(): Promise<void>;\n\n  /**\n   * Receives a single indexable document. In your decorate method, you can:\n   *\n   * - Resolve `undefined` to indicate the record should be omitted.\n   * - Resolve a single modified document, which could contain new fields,\n   *   edited fields, or removed fields.\n   * - Resolve an array of indexable documents, if the purpose if the decorator\n   *   is to convert one document into multiple derivative documents.\n   */\n  public abstract decorate(\n    document: IndexableDocument,\n  ): Promise<IndexableDocument | IndexableDocument[] | undefined>;\n\n  /**\n   * Any asynchronous teardown tasks can be performed here.\n   */\n  public abstract finalize(): Promise<void>;\n\n  /**\n   * Encapsulates simple transform stream logic.\n   * @internal\n   */\n  async _transform(\n    document: IndexableDocument,\n    _: any,\n    done: (error?: Error | null) => void,\n  ) {\n    // Wait for init before proceeding. Throw error if initialization failed.\n    const maybeError = await this.initialized;\n    if (maybeError) {\n      done(maybeError);\n      return;\n    }\n\n    try {\n      const decorated = await this.decorate(document);\n\n      // If undefined was returned, omit the record and move on.\n      if (decorated === undefined) {\n        done();\n        return;\n      }\n\n      // If an array of documents was given, push them all.\n      if (Array.isArray(decorated)) {\n        decorated.forEach(doc => {\n          this.push(doc);\n        });\n        done();\n        return;\n      }\n\n      // Otherwise, just push the decorated document.\n      this.push(decorated);\n      done();\n    } catch (e) {\n      assertError(e);\n      done(e);\n    }\n  }\n\n  /**\n   * Encapsulates finalization and final error handling logic.\n   * @internal\n   */\n  async _final(done: (error?: Error | null) => void) {\n    try {\n      await this.finalize();\n      done();\n    } catch (e) {\n      assertError(e);\n      done(e);\n    }\n  }\n}\n","/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { IndexableDocument } from '@backstage/plugin-search-common';\nimport lunr from 'lunr';\nimport { BatchSearchEngineIndexer } from '../indexing';\n\n/**\n * @beta\n */\nexport class LunrSearchEngineIndexer extends BatchSearchEngineIndexer {\n  private schemaInitialized = false;\n  private builder: lunr.Builder;\n  private docStore: Record<string, IndexableDocument> = {};\n\n  constructor() {\n    super({ batchSize: 1000 });\n\n    this.builder = new lunr.Builder();\n    this.builder.pipeline.add(lunr.trimmer, lunr.stopWordFilter, lunr.stemmer);\n    this.builder.searchPipeline.add(lunr.stemmer);\n    this.builder.metadataWhitelist = ['position'];\n  }\n\n  // No async initialization required.\n  async initialize(): Promise<void> {}\n  async finalize(): Promise<void> {}\n\n  async index(documents: IndexableDocument[]): Promise<void> {\n    if (!this.schemaInitialized) {\n      // Make this lunr index aware of all relevant fields.\n      Object.keys(documents[0]).forEach(field => {\n        this.builder.field(field);\n      });\n\n      // Set \"location\" field as reference field\n      this.builder.ref('location');\n\n      this.schemaInitialized = true;\n    }\n\n    documents.forEach(document => {\n      // Add document to Lunar index\n      this.builder.add(document);\n\n      // Store documents in memory to be able to look up document using the ref during query time\n      // This is not how you should implement your SearchEngine implementation! Do not copy!\n      this.docStore[document.location] = document;\n    });\n  }\n\n  buildIndex() {\n    return this.builder.build();\n  }\n\n  getDocumentStore() {\n    return this.docStore;\n  }\n}\n","/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  IndexableDocument,\n  IndexableResultSet,\n  SearchQuery,\n  QueryTranslator,\n  SearchEngine,\n} from '@backstage/plugin-search-common';\nimport lunr from 'lunr';\nimport { v4 as uuid } from 'uuid';\nimport { Logger } from 'winston';\nimport { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer';\n\n/**\n * @beta\n */\nexport type ConcreteLunrQuery = {\n  lunrQueryBuilder: lunr.Index.QueryBuilder;\n  documentTypes?: string[];\n  pageSize: number;\n};\n\ntype LunrResultEnvelope = {\n  result: lunr.Index.Result;\n  type: string;\n};\n\n/**\n * @beta\n */\nexport type LunrQueryTranslator = (query: SearchQuery) => ConcreteLunrQuery;\n\n/**\n * @beta\n */\nexport class LunrSearchEngine implements SearchEngine {\n  protected lunrIndices: Record<string, lunr.Index> = {};\n  protected docStore: Record<string, IndexableDocument>;\n  protected logger: Logger;\n  protected highlightPreTag: string;\n  protected highlightPostTag: string;\n\n  constructor({ logger }: { logger: Logger }) {\n    this.logger = logger;\n    this.docStore = {};\n    const uuidTag = uuid();\n    this.highlightPreTag = `<${uuidTag}>`;\n    this.highlightPostTag = `</${uuidTag}>`;\n  }\n\n  protected translator: QueryTranslator = ({\n    term,\n    filters,\n    types,\n  }: SearchQuery): ConcreteLunrQuery => {\n    const pageSize = 25;\n\n    return {\n      lunrQueryBuilder: q => {\n        const termToken = lunr.tokenizer(term);\n\n        // Support for typeahead search is based on https://github.com/olivernn/lunr.js/issues/256#issuecomment-295407852\n        // look for an exact match and apply a large positive boost\n        q.term(termToken, {\n          usePipeline: true,\n          boost: 100,\n        });\n        // look for terms that match the beginning of this term and apply a\n        // medium boost\n        q.term(termToken, {\n          usePipeline: false,\n          boost: 10,\n          wildcard: lunr.Query.wildcard.TRAILING,\n        });\n        // look for terms that match with an edit distance of 2 and apply a\n        // small boost\n        q.term(termToken, {\n          usePipeline: false,\n          editDistance: 2,\n          boost: 1,\n        });\n\n        if (filters) {\n          Object.entries(filters).forEach(([field, fieldValue]) => {\n            if (!q.allFields.includes(field)) {\n              // Throw for unknown field, as this will be a non match\n              throw new Error(`unrecognised field ${field}`);\n            }\n            // Arrays are poorly supported, but we can make it better for single-item arrays,\n            // which should be a common case\n            const value =\n              Array.isArray(fieldValue) && fieldValue.length === 1\n                ? fieldValue[0]\n                : fieldValue;\n\n            // Require that the given field has the given value\n            if (['string', 'number', 'boolean'].includes(typeof value)) {\n              q.term(lunr.tokenizer(value?.toString()), {\n                presence: lunr.Query.presence.REQUIRED,\n                fields: [field],\n              });\n            } else if (Array.isArray(value)) {\n              // Illustrate how multi-value filters could work.\n              // But warn that Lurn supports this poorly.\n              this.logger.warn(\n                `Non-scalar filter value used for field ${field}. Consider using a different Search Engine for better results.`,\n              );\n              q.term(lunr.tokenizer(value), {\n                presence: lunr.Query.presence.OPTIONAL,\n                fields: [field],\n              });\n            } else {\n              // Log a warning or something about unknown filter value\n              this.logger.warn(`Unknown filter type used on field ${field}`);\n            }\n          });\n        }\n      },\n      documentTypes: types,\n      pageSize,\n    };\n  };\n\n  setTranslator(translator: LunrQueryTranslator) {\n    this.translator = translator;\n  }\n\n  async getIndexer(type: string) {\n    const indexer = new LunrSearchEngineIndexer();\n\n    indexer.on('close', () => {\n      // Once the stream is closed, build the index and store the documents in\n      // memory for later retrieval.\n      this.lunrIndices[type] = indexer.buildIndex();\n      this.docStore = { ...this.docStore, ...indexer.getDocumentStore() };\n    });\n\n    return indexer;\n  }\n\n  async query(query: SearchQuery): Promise<IndexableResultSet> {\n    const { lunrQueryBuilder, documentTypes, pageSize } = this.translator(\n      query,\n    ) as ConcreteLunrQuery;\n\n    const results: LunrResultEnvelope[] = [];\n\n    // Iterate over the filtered list of this.lunrIndex keys.\n    Object.keys(this.lunrIndices)\n      .filter(type => !documentTypes || documentTypes.includes(type))\n      .forEach(type => {\n        try {\n          results.push(\n            ...this.lunrIndices[type].query(lunrQueryBuilder).map(result => {\n              return {\n                result: result,\n                type: type,\n              };\n            }),\n          );\n        } catch (err) {\n          // if a field does not exist on a index, we can see that as a no-match\n          if (\n            err instanceof Error &&\n            err.message.startsWith('unrecognised field')\n          ) {\n            return;\n          }\n          throw err;\n        }\n      });\n\n    // Sort results.\n    results.sort((doc1, doc2) => {\n      return doc2.result.score - doc1.result.score;\n    });\n\n    // Perform paging\n    const { page } = decodePageCursor(query.pageCursor);\n    const offset = page * pageSize;\n    const hasPreviousPage = page > 0;\n    const hasNextPage = results.length > offset + pageSize;\n    const nextPageCursor = hasNextPage\n      ? encodePageCursor({ page: page + 1 })\n      : undefined;\n    const previousPageCursor = hasPreviousPage\n      ? encodePageCursor({ page: page - 1 })\n      : undefined;\n\n    // Translate results into IndexableResultSet\n    const realResultSet: IndexableResultSet = {\n      results: results.slice(offset, offset + pageSize).map(d => ({\n        type: d.type,\n        document: this.docStore[d.result.ref],\n        highlight: {\n          preTag: this.highlightPreTag,\n          postTag: this.highlightPostTag,\n          fields: parseHighlightFields({\n            preTag: this.highlightPreTag,\n            postTag: this.highlightPostTag,\n            doc: this.docStore[d.result.ref],\n            positionMetadata: d.result.matchData.metadata as any,\n          }),\n        },\n      })),\n      nextPageCursor,\n      previousPageCursor,\n    };\n\n    return realResultSet;\n  }\n}\n\nexport function decodePageCursor(pageCursor?: string): { page: number } {\n  if (!pageCursor) {\n    return { page: 0 };\n  }\n\n  return {\n    page: Number(Buffer.from(pageCursor, 'base64').toString('utf-8')),\n  };\n}\n\nexport function encodePageCursor({ page }: { page: number }): string {\n  return Buffer.from(`${page}`, 'utf-8').toString('base64');\n}\n\ntype ParseHighlightFieldsProps = {\n  preTag: string;\n  postTag: string;\n  doc: any;\n  positionMetadata: {\n    [term: string]: {\n      [field: string]: {\n        position: number[][];\n      };\n    };\n  };\n};\n\nexport function parseHighlightFields({\n  preTag,\n  postTag,\n  doc,\n  positionMetadata,\n}: ParseHighlightFieldsProps): { [field: string]: string } {\n  // Merge the field positions across all query terms\n  const highlightFieldPositions = Object.values(positionMetadata).reduce(\n    (fieldPositions, metadata) => {\n      Object.keys(metadata).map(fieldKey => {\n        fieldPositions[fieldKey] = fieldPositions[fieldKey] ?? [];\n        fieldPositions[fieldKey].push(...metadata[fieldKey].position);\n      });\n\n      return fieldPositions;\n    },\n    {} as { [field: string]: number[][] },\n  );\n\n  return Object.fromEntries(\n    Object.entries(highlightFieldPositions).map(([field, positions]) => {\n      positions.sort((a, b) => b[0] - a[0]);\n\n      const highlightedField = positions.reduce((content, pos) => {\n        return (\n          `${content.substring(0, pos[0])}${preTag}` +\n          `${content.substring(pos[0], pos[0] + pos[1])}` +\n          `${postTag}${content.substring(pos[0] + pos[1])}`\n        );\n      }, doc[field]);\n\n      return [field, highlightedField];\n    }),\n  );\n}\n","/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { IndexableDocument } from '@backstage/plugin-search-common';\nimport { pipeline, Readable, Transform, Writable } from 'stream';\n\n/**\n * Object resolved after a test pipeline is executed.\n * @beta\n */\nexport type TestPipelineResult = {\n  /**\n   * If an error was emitted by the pipeline, it will be set here.\n   */\n  error: unknown;\n\n  /**\n   * A list of documents collected at the end of the pipeline. If the subject\n   * under test is an indexer, this will be an empty array (because your\n   * indexer should have received the documents instead).\n   */\n  documents: IndexableDocument[];\n};\n\n/**\n * Test utility for Backstage Search collators, decorators, and indexers.\n * @beta\n */\nexport class TestPipeline {\n  private collator?: Readable;\n  private decorator?: Transform;\n  private indexer?: Writable;\n\n  private constructor({\n    collator,\n    decorator,\n    indexer,\n  }: {\n    collator?: Readable;\n    decorator?: Transform;\n    indexer?: Writable;\n  }) {\n    this.collator = collator;\n    this.decorator = decorator;\n    this.indexer = indexer;\n  }\n\n  /**\n   * Provide the collator, decorator, or indexer to be tested.\n   */\n  static withSubject(subject: Readable | Transform | Writable) {\n    if (subject instanceof Transform) {\n      return new TestPipeline({ decorator: subject });\n    }\n\n    if (subject instanceof Readable) {\n      return new TestPipeline({ collator: subject });\n    }\n\n    if (subject instanceof Writable) {\n      return new TestPipeline({ indexer: subject });\n    }\n\n    throw new Error(\n      'Unknown test subject: are you passing a readable, writable, or transform stream?',\n    );\n  }\n\n  /**\n   * Provide documents for testing decorators and indexers.\n   */\n  withDocuments(documents: IndexableDocument[]): TestPipeline {\n    if (this.collator) {\n      throw new Error('Cannot provide documents when testing a collator.');\n    }\n\n    // Set a naive readable stream that just pushes all given documents.\n    this.collator = new Readable({ objectMode: true });\n    this.collator._read = () => {};\n    process.nextTick(() => {\n      documents.forEach(document => {\n        this.collator!.push(document);\n      });\n      this.collator!.push(null);\n    });\n\n    return this;\n  }\n\n  /**\n   * Execute the test pipeline so that you can make assertions about the result\n   * or behavior of the given test subject.\n   */\n  async execute(): Promise<TestPipelineResult> {\n    const documents: IndexableDocument[] = [];\n    if (!this.collator) {\n      throw new Error(\n        'Cannot execute pipeline without a collator or documents',\n      );\n    }\n\n    // If we are here and there is no indexer, we are testing a collator or a\n    // decorator. Set up a naive writable that captures documents in memory.\n    if (!this.indexer) {\n      this.indexer = new Writable({ objectMode: true });\n      this.indexer._write = (document: IndexableDocument, _, done) => {\n        documents.push(document);\n        done();\n      };\n    }\n\n    return new Promise<TestPipelineResult>(done => {\n      const pipes: (Readable | Transform | Writable)[] = [this.collator!];\n      if (this.decorator) {\n        pipes.push(this.decorator);\n      }\n      pipes.push(this.indexer!);\n\n      pipeline(pipes, (error: NodeJS.ErrnoException | null) => {\n        done({\n          error,\n          documents,\n        });\n      });\n    });\n  }\n}\n"],"names":["AbortController","pipeline","Writable","assertError","Transform","lunr","uuid","Readable"],"mappings":";;;;;;;;;;;;;;AACO,MAAM,SAAS,CAAC;AACvB,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE;AAC1B,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;AACzB,IAAI,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;AACvB,IAAI,IAAI,CAAC,eAAe,GAAG,IAAIA,mCAAe,EAAE,CAAC;AACjD,IAAI,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;AAC3B,GAAG;AACH,EAAE,aAAa,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,EAAE;AAC/C,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE;AACxB,MAAM,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;AACpF,KAAK;AACL,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE;AAC3B,MAAM,MAAM,IAAI,KAAK,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC;AAC5D,KAAK;AACL,IAAI,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;AAClD,GAAG;AACH,EAAE,KAAK,GAAG;AACV,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;AAC7D,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;AAC1B,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK;AAC/C,MAAM,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AAC1D,MAAM,eAAe,CAAC,GAAG,CAAC;AAC1B,QAAQ,EAAE;AACV,QAAQ,EAAE,EAAE,IAAI;AAChB,QAAQ,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;AAC3C,OAAO,CAAC,CAAC;AACT,KAAK,CAAC,CAAC;AACP,GAAG;AACH,EAAE,IAAI,GAAG;AACT,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;AAC7D,IAAI,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;AACjC,IAAI,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;AAC3B,GAAG;AACH;;AChCO,MAAM,YAAY,CAAC;AAC1B,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,EAAE;AACxC,IAAI,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;AACxB,IAAI,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;AACzB,IAAI,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;AAC5B,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;AACzB,IAAI,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;AACrC,GAAG;AACH,EAAE,eAAe,GAAG;AACpB,IAAI,OAAO,IAAI,CAAC,YAAY,CAAC;AAC7B,GAAG;AACH,EAAE,gBAAgB,GAAG;AACrB,IAAI,OAAO,IAAI,CAAC,aAAa,CAAC;AAC9B,GAAG;AACH,EAAE,WAAW,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;AACrC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,2BAA2B,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACpG,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG;AACnC,MAAM,OAAO;AACb,MAAM,QAAQ;AACd,KAAK,CAAC;AACN,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG;AACvC,MAAM,oBAAoB,EAAE,OAAO,CAAC,oBAAoB;AACxD,KAAK,CAAC;AACN,GAAG;AACH,EAAE,YAAY,CAAC,EAAE,OAAO,EAAE,EAAE;AAC5B,IAAI,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC;AACzC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AACjG,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK;AAC5B,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE;AAChD,QAAQ,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC5C,OAAO,MAAM;AACb,QAAQ,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AAC1C,OAAO;AACP,KAAK,CAAC,CAAC;AACP,GAAG;AACH,EAAE,MAAM,KAAK,GAAG;AAChB,IAAI,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC;AACpC,MAAM,MAAM,EAAE,IAAI,CAAC,MAAM;AACzB,KAAK,CAAC,CAAC;AACP,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK;AAClD,MAAM,SAAS,CAAC,aAAa,CAAC;AAC9B,QAAQ,EAAE,EAAE,CAAC,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;AAC/E,QAAQ,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ;AACtD,QAAQ,IAAI,EAAE,YAAY;AAC1B,UAAU,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;AAC5E,UAAU,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,wBAAwB,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACnH,UAAU,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,OAAO,KAAK;AACvI,YAAY,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;AAC3D,YAAY,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,uBAAuB,EAAE,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;AAC9G,YAAY,OAAO,SAAS,CAAC;AAC7B,WAAW,CAAC,CAAC,CAAC;AACd,UAAU,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;AACnE,UAAU,OAAO,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK;AACvC,YAAYC,eAAQ,CAAC,CAAC,QAAQ,EAAE,GAAG,UAAU,EAAE,OAAO,CAAC,EAAE,CAAC,KAAK,KAAK;AACpE,cAAc,IAAI,KAAK,EAAE;AACzB,gBAAgB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,wBAAwB,EAAE,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;AACtF,eAAe,MAAM;AACrB,gBAAgB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,wBAAwB,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;AAC9E,eAAe;AACf,cAAc,IAAI,EAAE,CAAC;AACrB,aAAa,CAAC,CAAC;AACf,WAAW,CAAC,CAAC;AACb,SAAS;AACT,OAAO,CAAC,CAAC;AACT,KAAK,CAAC,CAAC;AACP,IAAI,OAAO;AACX,MAAM,SAAS;AACf,KAAK,CAAC;AACN,GAAG;AACH;;ACrEO,MAAM,wBAAwB,SAASC,eAAQ,CAAC;AACvD,EAAE,WAAW,CAAC,OAAO,EAAE;AACvB,IAAI,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;AAChC,IAAI,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;AAC3B,IAAI,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;AACvC,IAAI,IAAI,CAAC,WAAW,GAAG,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK;AAC7C,MAAM,YAAY,CAAC,YAAY;AAC/B,QAAQ,IAAI;AACZ,UAAU,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;AAClC,UAAU,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACvB,SAAS,CAAC,OAAO,CAAC,EAAE;AACpB,UAAUC,kBAAW,CAAC,CAAC,CAAC,CAAC;AACzB,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,SAAS;AACT,OAAO,CAAC,CAAC;AACT,KAAK,CAAC,CAAC;AACP,GAAG;AACH,EAAE,MAAM,MAAM,CAAC,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE;AAC9B,IAAI,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC;AAC9C,IAAI,IAAI,UAAU,EAAE;AACpB,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;AACvB,MAAM,OAAO;AACb,KAAK;AACL,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAChC,IAAI,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE;AACnD,MAAM,IAAI,EAAE,CAAC;AACb,MAAM,OAAO;AACb,KAAK;AACL,IAAI,IAAI;AACR,MAAM,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;AAC1C,MAAM,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;AAC7B,MAAM,IAAI,EAAE,CAAC;AACb,KAAK,CAAC,OAAO,CAAC,EAAE;AAChB,MAAMA,kBAAW,CAAC,CAAC,CAAC,CAAC;AACrB,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC;AACd,KAAK;AACL,GAAG;AACH,EAAE,MAAM,MAAM,CAAC,IAAI,EAAE;AACrB,IAAI,IAAI;AACR,MAAM,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE;AACpC,QAAQ,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;AAC5C,QAAQ,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;AAC/B,OAAO;AACP,MAAM,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;AAC5B,MAAM,IAAI,EAAE,CAAC;AACb,KAAK,CAAC,OAAO,CAAC,EAAE;AAChB,MAAMA,kBAAW,CAAC,CAAC,CAAC,CAAC;AACrB,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC;AACd,KAAK;AACL,GAAG;AACH;;AClDO,MAAM,aAAa,SAASC,gBAAS,CAAC;AAC7C,EAAE,WAAW,GAAG;AAChB,IAAI,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;AAChC,IAAI,IAAI,CAAC,WAAW,GAAG,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK;AAC7C,MAAM,YAAY,CAAC,YAAY;AAC/B,QAAQ,IAAI;AACZ,UAAU,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;AAClC,UAAU,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACvB,SAAS,CAAC,OAAO,CAAC,EAAE;AACpB,UAAUD,kBAAW,CAAC,CAAC,CAAC,CAAC;AACzB,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,SAAS;AACT,OAAO,CAAC,CAAC;AACT,KAAK,CAAC,CAAC;AACP,GAAG;AACH,EAAE,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE;AACtC,IAAI,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC;AAC9C,IAAI,IAAI,UAAU,EAAE;AACpB,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;AACvB,MAAM,OAAO;AACb,KAAK;AACL,IAAI,IAAI;AACR,MAAM,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AACtD,MAAM,IAAI,SAAS,KAAK,KAAK,CAAC,EAAE;AAChC,QAAQ,IAAI,EAAE,CAAC;AACf,QAAQ,OAAO;AACf,OAAO;AACP,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;AACpC,QAAQ,SAAS,CAAC,OAAO,CAAC,CAAC,GAAG,KAAK;AACnC,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,SAAS,CAAC,CAAC;AACX,QAAQ,IAAI,EAAE,CAAC;AACf,QAAQ,OAAO;AACf,OAAO;AACP,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AAC3B,MAAM,IAAI,EAAE,CAAC;AACb,KAAK,CAAC,OAAO,CAAC,EAAE;AAChB,MAAMA,kBAAW,CAAC,CAAC,CAAC,CAAC;AACrB,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC;AACd,KAAK;AACL,GAAG;AACH,EAAE,MAAM,MAAM,CAAC,IAAI,EAAE;AACrB,IAAI,IAAI;AACR,MAAM,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;AAC5B,MAAM,IAAI,EAAE,CAAC;AACb,KAAK,CAAC,OAAO,CAAC,EAAE;AAChB,MAAMA,kBAAW,CAAC,CAAC,CAAC,CAAC;AACrB,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC;AACd,KAAK;AACL,GAAG;AACH;;AClDO,MAAM,uBAAuB,SAAS,wBAAwB,CAAC;AACtE,EAAE,WAAW,GAAG;AAChB,IAAI,KAAK,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;AAC9B,IAAI,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;AACnC,IAAI,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;AACvB,IAAI,IAAI,CAAC,OAAO,GAAG,IAAIE,wBAAI,CAAC,OAAO,EAAE,CAAC;AACtC,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAACA,wBAAI,CAAC,OAAO,EAAEA,wBAAI,CAAC,cAAc,EAAEA,wBAAI,CAAC,OAAO,CAAC,CAAC;AAC/E,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAACA,wBAAI,CAAC,OAAO,CAAC,CAAC;AAClD,IAAI,IAAI,CAAC,OAAO,CAAC,iBAAiB,GAAG,CAAC,UAAU,CAAC,CAAC;AAClD,GAAG;AACH,EAAE,MAAM,UAAU,GAAG;AACrB,GAAG;AACH,EAAE,MAAM,QAAQ,GAAG;AACnB,GAAG;AACH,EAAE,MAAM,KAAK,CAAC,SAAS,EAAE;AACzB,IAAI,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE;AACjC,MAAM,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,KAAK;AACnD,QAAQ,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAClC,OAAO,CAAC,CAAC;AACT,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;AACnC,MAAM,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;AACpC,KAAK;AACL,IAAI,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,KAAK;AACpC,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;AACjC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,QAAQ,CAAC;AAClD,KAAK,CAAC,CAAC;AACP,GAAG;AACH,EAAE,UAAU,GAAG;AACf,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;AAChC,GAAG;AACH,EAAE,gBAAgB,GAAG;AACrB,IAAI,OAAO,IAAI,CAAC,QAAQ,CAAC;AACzB,GAAG;AACH;;AChCO,MAAM,gBAAgB,CAAC;AAC9B,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE;AAC1B,IAAI,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;AAC1B,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC;AACvB,MAAM,IAAI;AACV,MAAM,OAAO;AACb,MAAM,KAAK;AACX,KAAK,KAAK;AACV,MAAM,MAAM,QAAQ,GAAG,EAAE,CAAC;AAC1B,MAAM,OAAO;AACb,QAAQ,gBAAgB,EAAE,CAAC,CAAC,KAAK;AACjC,UAAU,MAAM,SAAS,GAAGA,wBAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;AACjD,UAAU,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE;AAC5B,YAAY,WAAW,EAAE,IAAI;AAC7B,YAAY,KAAK,EAAE,GAAG;AACtB,WAAW,CAAC,CAAC;AACb,UAAU,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE;AAC5B,YAAY,WAAW,EAAE,KAAK;AAC9B,YAAY,KAAK,EAAE,EAAE;AACrB,YAAY,QAAQ,EAAEA,wBAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ;AAClD,WAAW,CAAC,CAAC;AACb,UAAU,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE;AAC5B,YAAY,WAAW,EAAE,KAAK;AAC9B,YAAY,YAAY,EAAE,CAAC;AAC3B,YAAY,KAAK,EAAE,CAAC;AACpB,WAAW,CAAC,CAAC;AACb,UAAU,IAAI,OAAO,EAAE;AACvB,YAAY,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,KAAK;AACrE,cAAc,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;AAChD,gBAAgB,MAAM,IAAI,KAAK,CAAC,CAAC,mBAAmB,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;AAC/D,eAAe;AACf,cAAc,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9G,cAAc,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,OAAO,KAAK,CAAC,EAAE;AAC1E,gBAAgB,CAAC,CAAC,IAAI,CAACA,wBAAI,CAAC,SAAS,CAAC,KAAK,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,EAAE;AAClF,kBAAkB,QAAQ,EAAEA,wBAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ;AACxD,kBAAkB,MAAM,EAAE,CAAC,KAAK,CAAC;AACjC,iBAAiB,CAAC,CAAC;AACnB,eAAe,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;AAC/C,gBAAgB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,uCAAuC,EAAE,KAAK,CAAC,8DAA8D,CAAC,CAAC,CAAC;AAClJ,gBAAgB,CAAC,CAAC,IAAI,CAACA,wBAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE;AAC9C,kBAAkB,QAAQ,EAAEA,wBAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ;AACxD,kBAAkB,MAAM,EAAE,CAAC,KAAK,CAAC;AACjC,iBAAiB,CAAC,CAAC;AACnB,eAAe,MAAM;AACrB,gBAAgB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,kCAAkC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;AAC/E,eAAe;AACf,aAAa,CAAC,CAAC;AACf,WAAW;AACX,SAAS;AACT,QAAQ,aAAa,EAAE,KAAK;AAC5B,QAAQ,QAAQ;AAChB,OAAO,CAAC;AACR,KAAK,CAAC;AACN,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;AACzB,IAAI,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;AACvB,IAAI,MAAM,OAAO,GAAGC,OAAI,EAAE,CAAC;AAC3B,IAAI,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;AAC1C,IAAI,IAAI,CAAC,gBAAgB,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;AAC5C,GAAG;AACH,EAAE,aAAa,CAAC,UAAU,EAAE;AAC5B,IAAI,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;AACjC,GAAG;AACH,EAAE,MAAM,UAAU,CAAC,IAAI,EAAE;AACzB,IAAI,MAAM,OAAO,GAAG,IAAI,uBAAuB,EAAE,CAAC;AAClD,IAAI,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM;AAC9B,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;AACpD,MAAM,IAAI,CAAC,QAAQ,GAAG,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC;AAC1E,KAAK,CAAC,CAAC;AACP,IAAI,OAAO,OAAO,CAAC;AACnB,GAAG;AACH,EAAE,MAAM,KAAK,CAAC,KAAK,EAAE;AACrB,IAAI,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;AACjF,IAAI,MAAM,OAAO,GAAG,EAAE,CAAC;AACvB,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,CAAC,aAAa,IAAI,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK;AACrH,MAAM,IAAI;AACV,QAAQ,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK;AACvF,UAAU,OAAO;AACjB,YAAY,MAAM;AAClB,YAAY,IAAI;AAChB,WAAW,CAAC;AACZ,SAAS,CAAC,CAAC,CAAC;AACZ,OAAO,CAAC,OAAO,GAAG,EAAE;AACpB,QAAQ,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,oBAAoB,CAAC,EAAE;AAClF,UAAU,OAAO;AACjB,SAAS;AACT,QAAQ,MAAM,GAAG,CAAC;AAClB,OAAO;AACP,KAAK,CAAC,CAAC;AACP,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK;AACjC,MAAM,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;AACnD,KAAK,CAAC,CAAC;AACP,IAAI,MAAM,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;AACxD,IAAI,MAAM,MAAM,GAAG,IAAI,GAAG,QAAQ,CAAC;AACnC,IAAI,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC;AACrC,IAAI,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;AAC3D,IAAI,MAAM,cAAc,GAAG,WAAW,GAAG,gBAAgB,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC;AACvF,IAAI,MAAM,kBAAkB,GAAG,eAAe,GAAG,gBAAgB,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC;AAC/F,IAAI,MAAM,aAAa,GAAG;AAC1B,MAAM,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM;AACpE,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI;AACpB,QAAQ,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;AAC7C,QAAQ,SAAS,EAAE;AACnB,UAAU,MAAM,EAAE,IAAI,CAAC,eAAe;AACtC,UAAU,OAAO,EAAE,IAAI,CAAC,gBAAgB;AACxC,UAAU,MAAM,EAAE,oBAAoB,CAAC;AACvC,YAAY,MAAM,EAAE,IAAI,CAAC,eAAe;AACxC,YAAY,OAAO,EAAE,IAAI,CAAC,gBAAgB;AAC1C,YAAY,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;AAC5C,YAAY,gBAAgB,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ;AACzD,WAAW,CAAC;AACZ,SAAS;AACT,OAAO,CAAC,CAAC;AACT,MAAM,cAAc;AACpB,MAAM,kBAAkB;AACxB,KAAK,CAAC;AACN,IAAI,OAAO,aAAa,CAAC;AACzB,GAAG;AACH,CAAC;AACM,SAAS,gBAAgB,CAAC,UAAU,EAAE;AAC7C,EAAE,IAAI,CAAC,UAAU,EAAE;AACnB,IAAI,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;AACvB,GAAG;AACH,EAAE,OAAO;AACT,IAAI,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;AACrE,GAAG,CAAC;AACJ,CAAC;AACM,SAAS,gBAAgB,CAAC,EAAE,IAAI,EAAE,EAAE;AAC3C,EAAE,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAC5D,CAAC;AACM,SAAS,oBAAoB,CAAC;AACrC,EAAE,MAAM;AACR,EAAE,OAAO;AACT,EAAE,GAAG;AACL,EAAE,gBAAgB;AAClB,CAAC,EAAE;AACH,EAAE,MAAM,uBAAuB,GAAG,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC,cAAc,EAAE,QAAQ,KAAK;AACvG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,KAAK;AAC5C,MAAM,IAAI,EAAE,CAAC;AACb,MAAM,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,GAAG,cAAc,CAAC,QAAQ,CAAC,KAAK,IAAI,GAAG,EAAE,GAAG,EAAE,CAAC;AACnF,MAAM,cAAc,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC;AACpE,KAAK,CAAC,CAAC;AACP,IAAI,OAAO,cAAc,CAAC;AAC1B,GAAG,EAAE,EAAE,CAAC,CAAC;AACT,EAAE,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,SAAS,CAAC,KAAK;AAChG,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC1C,IAAI,MAAM,gBAAgB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,GAAG,KAAK;AAChE,MAAM,OAAO,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACpJ,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;AACnB,IAAI,OAAO,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC;AACrC,GAAG,CAAC,CAAC,CAAC;AACN;;ACxJO,MAAM,YAAY,CAAC;AAC1B,EAAE,WAAW,CAAC;AACd,IAAI,QAAQ;AACZ,IAAI,SAAS;AACb,IAAI,OAAO;AACX,GAAG,EAAE;AACL,IAAI,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;AAC7B,IAAI,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;AAC/B,IAAI,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;AAC3B,GAAG;AACH,EAAE,OAAO,WAAW,CAAC,OAAO,EAAE;AAC9B,IAAI,IAAI,OAAO,YAAYF,gBAAS,EAAE;AACtC,MAAM,OAAO,IAAI,YAAY,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;AACtD,KAAK;AACL,IAAI,IAAI,OAAO,YAAYG,eAAQ,EAAE;AACrC,MAAM,OAAO,IAAI,YAAY,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;AACrD,KAAK;AACL,IAAI,IAAI,OAAO,YAAYL,eAAQ,EAAE;AACrC,MAAM,OAAO,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;AACpD,KAAK;AACL,IAAI,MAAM,IAAI,KAAK,CAAC,kFAAkF,CAAC,CAAC;AACxG,GAAG;AACH,EAAE,aAAa,CAAC,SAAS,EAAE;AAC3B,IAAI,IAAI,IAAI,CAAC,QAAQ,EAAE;AACvB,MAAM,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;AAC3E,KAAK;AACL,IAAI,IAAI,CAAC,QAAQ,GAAG,IAAIK,eAAQ,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;AACvD,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,MAAM;AAChC,KAAK,CAAC;AACN,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM;AAC3B,MAAM,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,KAAK;AACtC,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AACrC,OAAO,CAAC,CAAC;AACT,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC/B,KAAK,CAAC,CAAC;AACP,IAAI,OAAO,IAAI,CAAC;AAChB,GAAG;AACH,EAAE,MAAM,OAAO,GAAG;AAClB,IAAI,MAAM,SAAS,GAAG,EAAE,CAAC;AACzB,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;AACxB,MAAM,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;AACjF,KAAK;AACL,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;AACvB,MAAM,IAAI,CAAC,OAAO,GAAG,IAAIL,eAAQ,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;AACxD,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK;AACnD,QAAQ,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AACjC,QAAQ,IAAI,EAAE,CAAC;AACf,OAAO,CAAC;AACR,KAAK;AACL,IAAI,OAAO,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK;AACjC,MAAM,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AACpC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE;AAC1B,QAAQ,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AACnC,OAAO;AACP,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC/B,MAAMD,eAAQ,CAAC,KAAK,EAAE,CAAC,KAAK,KAAK;AACjC,QAAQ,IAAI,CAAC;AACb,UAAU,KAAK;AACf,UAAU,SAAS;AACnB,SAAS,CAAC,CAAC;AACX,OAAO,CAAC,CAAC;AACT,KAAK,CAAC,CAAC;AACP,GAAG;AACH;;;;;;;;;"}
+{"version":3,"file":"index.cjs.js","sources":["../src/Scheduler.ts","../src/IndexBuilder.ts","../src/collators/NewlineDelimitedJsonCollatorFactory.ts","../src/indexing/BatchSearchEngineIndexer.ts","../src/indexing/DecoratorBase.ts","../src/engines/LunrSearchEngineIndexer.ts","../src/engines/LunrSearchEngine.ts","../src/test-utils/TestPipeline.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { AbortController } from 'node-abort-controller';\nimport { Logger } from 'winston';\nimport { TaskFunction, TaskRunner } from '@backstage/backend-tasks';\n\ntype TaskEnvelope = {\n  task: TaskFunction;\n  scheduledRunner: TaskRunner;\n};\n\n/**\n * ScheduleTaskParameters\n * @public\n */\nexport type ScheduleTaskParameters = {\n  id: string;\n  task: TaskFunction;\n  scheduledRunner: TaskRunner;\n};\n\n/**\n * Scheduler responsible for all search tasks.\n * @public\n */\nexport class Scheduler {\n  private logger: Logger;\n  private schedule: { [id: string]: TaskEnvelope };\n  private abortController: AbortController;\n  private isRunning: boolean;\n\n  constructor({ logger }: { logger: Logger }) {\n    this.logger = logger;\n    this.schedule = {};\n    this.abortController = new AbortController();\n    this.isRunning = false;\n  }\n\n  /**\n   * Adds each task and interval to the schedule.\n   * When running the tasks, the scheduler waits at least for the time specified\n   * in the interval once the task was completed, before running it again.\n   */\n  addToSchedule({ id, task, scheduledRunner }: ScheduleTaskParameters) {\n    if (this.isRunning) {\n      throw new Error(\n        'Cannot add task to schedule that has already been started.',\n      );\n    }\n\n    if (this.schedule[id]) {\n      throw new Error(`Task with id ${id} already exists.`);\n    }\n\n    this.schedule[id] = { task, scheduledRunner };\n  }\n\n  /**\n   * Starts the scheduling process for each task\n   */\n  start() {\n    this.logger.info('Starting all scheduled search tasks.');\n    this.isRunning = true;\n    Object.keys(this.schedule).forEach(id => {\n      const { task, scheduledRunner } = this.schedule[id];\n      scheduledRunner.run({\n        id,\n        fn: task,\n        signal: this.abortController.signal,\n      });\n    });\n  }\n\n  /**\n   * Stop all scheduled tasks.\n   */\n  stop() {\n    this.logger.info('Stopping all scheduled search tasks.');\n    this.abortController.abort();\n    this.isRunning = false;\n  }\n}\n","/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nimport {\n  DocumentDecoratorFactory,\n  DocumentTypeInfo,\n  SearchEngine,\n} from '@backstage/plugin-search-common';\nimport { Transform, pipeline } from 'stream';\nimport { Logger } from 'winston';\nimport { Scheduler } from './Scheduler';\nimport {\n  IndexBuilderOptions,\n  RegisterCollatorParameters,\n  RegisterDecoratorParameters,\n} from './types';\n\n/**\n * Used for adding collators, decorators and compile them into tasks which are added to a scheduler returned to the caller.\n * @public\n */\nexport class IndexBuilder {\n  private collators: Record<string, RegisterCollatorParameters>;\n  private decorators: Record<string, DocumentDecoratorFactory[]>;\n  private documentTypes: Record<string, DocumentTypeInfo>;\n  private searchEngine: SearchEngine;\n  private logger: Logger;\n\n  constructor({ logger, searchEngine }: IndexBuilderOptions) {\n    this.collators = {};\n    this.decorators = {};\n    this.documentTypes = {};\n    this.logger = logger;\n    this.searchEngine = searchEngine;\n  }\n\n  /**\n   * Responsible for returning the registered search engine.\n   */\n  getSearchEngine(): SearchEngine {\n    return this.searchEngine;\n  }\n\n  /**\n   * Responsible for returning the registered document types.\n   */\n  getDocumentTypes(): Record<string, DocumentTypeInfo> {\n    return this.documentTypes;\n  }\n\n  /**\n   * Makes the index builder aware of a collator that should be executed at the\n   * given refresh interval.\n   */\n  addCollator({ factory, schedule }: RegisterCollatorParameters): void {\n    this.logger.info(\n      `Added ${factory.constructor.name} collator factory for type ${factory.type}`,\n    );\n    this.collators[factory.type] = {\n      factory,\n      schedule,\n    };\n    this.documentTypes[factory.type] = {\n      visibilityPermission: factory.visibilityPermission,\n    };\n  }\n\n  /**\n   * Makes the index builder aware of a decorator. If no types are provided on\n   * the decorator, it will be applied to documents from all known collators,\n   * otherwise it will only be applied to documents of the given types.\n   */\n  addDecorator({ factory }: RegisterDecoratorParameters): void {\n    const types = factory.types || ['*'];\n    this.logger.info(\n      `Added decorator ${factory.constructor.name} to types ${types.join(\n        ', ',\n      )}`,\n    );\n    types.forEach(type => {\n      if (this.decorators.hasOwnProperty(type)) {\n        this.decorators[type].push(factory);\n      } else {\n        this.decorators[type] = [factory];\n      }\n    });\n  }\n\n  /**\n   * Compiles collators and decorators into tasks, which are added to a\n   * scheduler returned to the caller.\n   */\n  async build(): Promise<{ scheduler: Scheduler }> {\n    const scheduler = new Scheduler({\n      logger: this.logger,\n    });\n\n    Object.keys(this.collators).forEach(type => {\n      scheduler.addToSchedule({\n        id: `search_index_${type.replace('-', '_').toLocaleLowerCase('en-US')}`,\n        scheduledRunner: this.collators[type].schedule,\n        task: async () => {\n          // Instantiate the collator.\n          const collator = await this.collators[type].factory.getCollator();\n          this.logger.info(\n            `Collating documents for ${type} via ${this.collators[type].factory.constructor.name}`,\n          );\n\n          // Instantiate all relevant decorators.\n          const decorators: Transform[] = await Promise.all(\n            (this.decorators['*'] || [])\n              .concat(this.decorators[type] || [])\n              .map(async factory => {\n                const decorator = await factory.getDecorator();\n                this.logger.info(\n                  `Attached decorator via ${factory.constructor.name} to ${type} index pipeline.`,\n                );\n                return decorator;\n              }),\n          );\n\n          // Instantiate the indexer.\n          const indexer = await this.searchEngine.getIndexer(type);\n\n          // Compose collator/decorators/indexer into a pipeline\n          return new Promise<void>((resolve, reject) => {\n            pipeline(\n              [collator, ...decorators, indexer],\n              (error: NodeJS.ErrnoException | null) => {\n                if (error) {\n                  this.logger.error(\n                    `Collating documents for ${type} failed: ${error}`,\n                  );\n                  reject(error);\n                } else {\n                  // Signal index pipeline completion!\n                  this.logger.info(`Collating documents for ${type} succeeded`);\n                  resolve();\n                }\n              },\n            );\n          });\n        },\n      });\n    });\n\n    return {\n      scheduler,\n    };\n  }\n}\n","/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { UrlReader } from '@backstage/backend-common';\nimport { Config } from '@backstage/config';\nimport { Permission } from '@backstage/plugin-permission-common';\nimport { DocumentCollatorFactory } from '@backstage/plugin-search-common';\nimport { parse as parseNdjson } from 'ndjson';\nimport { Readable } from 'stream';\nimport { Logger } from 'winston';\n\n/**\n * Options for instansiate NewlineDelimitedJsonCollatorFactory\n * @public\n */\nexport type NewlineDelimitedJsonCollatorFactoryOptions = {\n  type: string;\n  searchPattern: string;\n  reader: UrlReader;\n  logger: Logger;\n  visibilityPermission?: Permission;\n};\n\n/**\n * Factory class producing a collator that can be used to index documents\n * sourced from the latest newline delimited JSON file matching a given search\n * pattern. \"Latest\" is determined by the name of the file (last alphabetically\n * is considered latest).\n *\n * @remarks\n * The reader provided must implement the `search()` method as well as the\n * `readUrl` method whose response includes the `stream()` method. Naturally,\n * the reader must also be configured to understand the given search pattern.\n *\n * @example\n * Here's an example configuration using Google Cloud Storage, which would\n * return the latest file under the `bucket` GCS bucket with files like\n * `xyz-2021.ndjson` or `xyz-2022.ndjson`.\n * ```ts\n * indexBuilder.addCollator({\n *   schedule,\n *   factory: NewlineDelimitedJsonCollatorFactory.fromConfig(env.config, {\n *     type: 'techdocs',\n *     searchPattern: 'https://storage.cloud.google.com/bucket/xyz-*',\n *     reader: env.reader,\n *     logger: env.logger,\n *   })\n * });\n * ```\n *\n * @public\n */\nexport class NewlineDelimitedJsonCollatorFactory\n  implements DocumentCollatorFactory\n{\n  readonly type: string;\n\n  public readonly visibilityPermission: Permission | undefined;\n\n  private constructor(\n    type: string,\n    private readonly searchPattern: string,\n    private readonly reader: UrlReader,\n    private readonly logger: Logger,\n    visibilityPermission: Permission | undefined,\n  ) {\n    this.type = type;\n    this.visibilityPermission = visibilityPermission;\n  }\n\n  /**\n   * Returns a NewlineDelimitedJsonCollatorFactory instance from configuration\n   * and a set of options.\n   */\n  static fromConfig(\n    _config: Config,\n    options: NewlineDelimitedJsonCollatorFactoryOptions,\n  ): NewlineDelimitedJsonCollatorFactory {\n    return new NewlineDelimitedJsonCollatorFactory(\n      options.type,\n      options.searchPattern,\n      options.reader,\n      options.logger,\n      options.visibilityPermission,\n    );\n  }\n\n  /**\n   * Returns the \"latest\" URL for the given search pattern (e.g. the one at the\n   * end of the list, sorted alphabetically).\n   */\n  private async lastUrl(): Promise<string | undefined> {\n    try {\n      // Search for files matching the given pattern, then sort/reverse. The\n      // first item in the list will be the \"latest\" file.\n      this.logger.info(\n        `Attempting to find latest .ndjson matching ${this.searchPattern}`,\n      );\n      const { files } = await this.reader.search(this.searchPattern);\n      const candidates = files\n        .filter(file => file.url.endsWith('.ndjson'))\n        .sort((a, b) => a.url.localeCompare(b.url))\n        .reverse();\n\n      return candidates[0]?.url;\n    } catch (e) {\n      this.logger.error(`Could not search for ${this.searchPattern}`, e);\n      throw e;\n    }\n  }\n\n  async getCollator(): Promise<Readable> {\n    // Search for files matching the given pattern.\n    const lastUrl = await this.lastUrl();\n\n    // Abort if no such file could be found.\n    if (!lastUrl) {\n      const noMatchingFile = `Could not find an .ndjson file matching ${this.searchPattern}`;\n      this.logger.error(noMatchingFile);\n      throw new Error(noMatchingFile);\n    } else {\n      this.logger.info(`Using latest .ndjson file ${lastUrl}`);\n    }\n\n    // Use the UrlReader to try and stream the file.\n    const readerResponse = await this.reader.readUrl!(lastUrl);\n    const stream = readerResponse.stream!();\n\n    // Use ndjson's parser to turn the raw file into an object-mode stream.\n    return stream.pipe(parseNdjson());\n  }\n}\n","/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { assertError } from '@backstage/errors';\nimport { IndexableDocument } from '@backstage/plugin-search-common';\nimport { Writable } from 'stream';\n\n/**\n * Options for {@link BatchSearchEngineIndexer}\n * @public\n */\nexport type BatchSearchEngineOptions = {\n  batchSize: number;\n};\n\n/**\n * Base class encapsulating batch-based stream processing. Useful as a base\n * class for search engine indexers.\n * @public\n */\nexport abstract class BatchSearchEngineIndexer extends Writable {\n  private batchSize: number;\n  private currentBatch: IndexableDocument[] = [];\n  private initialized: Promise<undefined | Error>;\n\n  constructor(options: BatchSearchEngineOptions) {\n    super({ objectMode: true });\n    this.batchSize = options.batchSize;\n\n    // @todo Once node v15 is minimum, convert to _construct implementation.\n    this.initialized = new Promise(done => {\n      // Necessary to allow concrete implementation classes to construct\n      // themselves before calling their initialize() methods.\n      setImmediate(async () => {\n        try {\n          await this.initialize();\n          done(undefined);\n        } catch (e) {\n          assertError(e);\n          done(e);\n        }\n      });\n    });\n  }\n\n  /**\n   * Receives an array of indexable documents (of size this.batchSize) which\n   * should be written to the search engine. This method won't be called again\n   * at least until it resolves.\n   */\n  public abstract index(documents: IndexableDocument[]): Promise<void>;\n\n  /**\n   * Any asynchronous setup tasks can be performed here.\n   */\n  public abstract initialize(): Promise<void>;\n\n  /**\n   * Any asynchronous teardown tasks can be performed here.\n   */\n  public abstract finalize(): Promise<void>;\n\n  /**\n   * Encapsulates batch stream write logic.\n   * @internal\n   */\n  async _write(\n    doc: IndexableDocument,\n    _e: any,\n    done: (error?: Error | null) => void,\n  ) {\n    // Wait for init before proceeding. Throw error if initialization failed.\n    const maybeError = await this.initialized;\n    if (maybeError) {\n      done(maybeError);\n      return;\n    }\n\n    this.currentBatch.push(doc);\n    if (this.currentBatch.length < this.batchSize) {\n      done();\n      return;\n    }\n\n    try {\n      await this.index(this.currentBatch);\n      this.currentBatch = [];\n      done();\n    } catch (e) {\n      assertError(e);\n      done(e);\n    }\n  }\n\n  /**\n   * Encapsulates finalization and final error handling logic.\n   * @internal\n   */\n  async _final(done: (error?: Error | null) => void) {\n    try {\n      // Index any remaining documents.\n      if (this.currentBatch.length) {\n        await this.index(this.currentBatch);\n        this.currentBatch = [];\n      }\n      await this.finalize();\n      done();\n    } catch (e) {\n      assertError(e);\n      done(e);\n    }\n  }\n}\n","/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { assertError } from '@backstage/errors';\nimport { IndexableDocument } from '@backstage/plugin-search-common';\nimport { Transform } from 'stream';\n\n/**\n * Base class encapsulating simple async transformations. Useful as a base\n * class for Backstage search decorators.\n * @public\n */\nexport abstract class DecoratorBase extends Transform {\n  private initialized: Promise<undefined | Error>;\n\n  constructor() {\n    super({ objectMode: true });\n\n    // @todo Once node v15 is minimum, convert to _construct implementation.\n    this.initialized = new Promise(done => {\n      // Necessary to allow concrete implementation classes to construct\n      // themselves before calling their initialize() methods.\n      setImmediate(async () => {\n        try {\n          await this.initialize();\n          done(undefined);\n        } catch (e) {\n          assertError(e);\n          done(e);\n        }\n      });\n    });\n  }\n\n  /**\n   * Any asynchronous setup tasks can be performed here.\n   */\n  public abstract initialize(): Promise<void>;\n\n  /**\n   * Receives a single indexable document. In your decorate method, you can:\n   *\n   * - Resolve `undefined` to indicate the record should be omitted.\n   * - Resolve a single modified document, which could contain new fields,\n   *   edited fields, or removed fields.\n   * - Resolve an array of indexable documents, if the purpose if the decorator\n   *   is to convert one document into multiple derivative documents.\n   */\n  public abstract decorate(\n    document: IndexableDocument,\n  ): Promise<IndexableDocument | IndexableDocument[] | undefined>;\n\n  /**\n   * Any asynchronous teardown tasks can be performed here.\n   */\n  public abstract finalize(): Promise<void>;\n\n  /**\n   * Encapsulates simple transform stream logic.\n   * @internal\n   */\n  async _transform(\n    document: IndexableDocument,\n    _: any,\n    done: (error?: Error | null) => void,\n  ) {\n    // Wait for init before proceeding. Throw error if initialization failed.\n    const maybeError = await this.initialized;\n    if (maybeError) {\n      done(maybeError);\n      return;\n    }\n\n    try {\n      const decorated = await this.decorate(document);\n\n      // If undefined was returned, omit the record and move on.\n      if (decorated === undefined) {\n        done();\n        return;\n      }\n\n      // If an array of documents was given, push them all.\n      if (Array.isArray(decorated)) {\n        decorated.forEach(doc => {\n          this.push(doc);\n        });\n        done();\n        return;\n      }\n\n      // Otherwise, just push the decorated document.\n      this.push(decorated);\n      done();\n    } catch (e) {\n      assertError(e);\n      done(e);\n    }\n  }\n\n  /**\n   * Encapsulates finalization and final error handling logic.\n   * @internal\n   */\n  async _final(done: (error?: Error | null) => void) {\n    try {\n      await this.finalize();\n      done();\n    } catch (e) {\n      assertError(e);\n      done(e);\n    }\n  }\n}\n","/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { IndexableDocument } from '@backstage/plugin-search-common';\nimport lunr from 'lunr';\nimport { BatchSearchEngineIndexer } from '../indexing';\n\n/**\n * Lunr specific search engine indexer\n * @public\n */\nexport class LunrSearchEngineIndexer extends BatchSearchEngineIndexer {\n  private schemaInitialized = false;\n  private builder: lunr.Builder;\n  private docStore: Record<string, IndexableDocument> = {};\n\n  constructor() {\n    super({ batchSize: 1000 });\n\n    this.builder = new lunr.Builder();\n    this.builder.pipeline.add(lunr.trimmer, lunr.stopWordFilter, lunr.stemmer);\n    this.builder.searchPipeline.add(lunr.stemmer);\n    this.builder.metadataWhitelist = ['position'];\n  }\n\n  // No async initialization required.\n  async initialize(): Promise<void> {}\n  async finalize(): Promise<void> {}\n\n  async index(documents: IndexableDocument[]): Promise<void> {\n    if (!this.schemaInitialized) {\n      // Make this lunr index aware of all relevant fields.\n      Object.keys(documents[0]).forEach(field => {\n        this.builder.field(field);\n      });\n\n      // Set \"location\" field as reference field\n      this.builder.ref('location');\n\n      this.schemaInitialized = true;\n    }\n\n    documents.forEach(document => {\n      // Add document to Lunar index\n      this.builder.add(document);\n\n      // Store documents in memory to be able to look up document using the ref during query time\n      // This is not how you should implement your SearchEngine implementation! Do not copy!\n      this.docStore[document.location] = document;\n    });\n  }\n\n  buildIndex() {\n    return this.builder.build();\n  }\n\n  getDocumentStore() {\n    return this.docStore;\n  }\n}\n","/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  IndexableDocument,\n  IndexableResultSet,\n  SearchQuery,\n  QueryTranslator,\n  SearchEngine,\n} from '@backstage/plugin-search-common';\nimport lunr from 'lunr';\nimport { v4 as uuid } from 'uuid';\nimport { Logger } from 'winston';\nimport { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer';\n\n/**\n * Type of translated query for the Lunr Search Engine.\n * @public\n */\nexport type ConcreteLunrQuery = {\n  lunrQueryBuilder: lunr.Index.QueryBuilder;\n  documentTypes?: string[];\n  pageSize: number;\n};\n\ntype LunrResultEnvelope = {\n  result: lunr.Index.Result;\n  type: string;\n};\n\n/**\n * Translator repsonsible for translating search term and filters to a query that the Lunr Search Engine understands.\n * @public\n */\nexport type LunrQueryTranslator = (query: SearchQuery) => ConcreteLunrQuery;\n\n/**\n * Lunr specific search engine implementation.\n * @public\n */\nexport class LunrSearchEngine implements SearchEngine {\n  protected lunrIndices: Record<string, lunr.Index> = {};\n  protected docStore: Record<string, IndexableDocument>;\n  protected logger: Logger;\n  protected highlightPreTag: string;\n  protected highlightPostTag: string;\n\n  constructor({ logger }: { logger: Logger }) {\n    this.logger = logger;\n    this.docStore = {};\n    const uuidTag = uuid();\n    this.highlightPreTag = `<${uuidTag}>`;\n    this.highlightPostTag = `</${uuidTag}>`;\n  }\n\n  protected translator: QueryTranslator = ({\n    term,\n    filters,\n    types,\n  }: SearchQuery): ConcreteLunrQuery => {\n    const pageSize = 25;\n\n    return {\n      lunrQueryBuilder: q => {\n        const termToken = lunr.tokenizer(term);\n\n        // Support for typeahead search is based on https://github.com/olivernn/lunr.js/issues/256#issuecomment-295407852\n        // look for an exact match and apply a large positive boost\n        q.term(termToken, {\n          usePipeline: true,\n          boost: 100,\n        });\n        // look for terms that match the beginning of this term and apply a\n        // medium boost\n        q.term(termToken, {\n          usePipeline: false,\n          boost: 10,\n          wildcard: lunr.Query.wildcard.TRAILING,\n        });\n        // look for terms that match with an edit distance of 2 and apply a\n        // small boost\n        q.term(termToken, {\n          usePipeline: false,\n          editDistance: 2,\n          boost: 1,\n        });\n\n        if (filters) {\n          Object.entries(filters).forEach(([field, fieldValue]) => {\n            if (!q.allFields.includes(field)) {\n              // Throw for unknown field, as this will be a non match\n              throw new Error(`unrecognised field ${field}`);\n            }\n            // Arrays are poorly supported, but we can make it better for single-item arrays,\n            // which should be a common case\n            const value =\n              Array.isArray(fieldValue) && fieldValue.length === 1\n                ? fieldValue[0]\n                : fieldValue;\n\n            // Require that the given field has the given value\n            if (['string', 'number', 'boolean'].includes(typeof value)) {\n              q.term(lunr.tokenizer(value?.toString()), {\n                presence: lunr.Query.presence.REQUIRED,\n                fields: [field],\n              });\n            } else if (Array.isArray(value)) {\n              // Illustrate how multi-value filters could work.\n              // But warn that Lurn supports this poorly.\n              this.logger.warn(\n                `Non-scalar filter value used for field ${field}. Consider using a different Search Engine for better results.`,\n              );\n              q.term(lunr.tokenizer(value), {\n                presence: lunr.Query.presence.OPTIONAL,\n                fields: [field],\n              });\n            } else {\n              // Log a warning or something about unknown filter value\n              this.logger.warn(`Unknown filter type used on field ${field}`);\n            }\n          });\n        }\n      },\n      documentTypes: types,\n      pageSize,\n    };\n  };\n\n  setTranslator(translator: LunrQueryTranslator) {\n    this.translator = translator;\n  }\n\n  async getIndexer(type: string) {\n    const indexer = new LunrSearchEngineIndexer();\n\n    indexer.on('close', () => {\n      // Once the stream is closed, build the index and store the documents in\n      // memory for later retrieval.\n      this.lunrIndices[type] = indexer.buildIndex();\n      this.docStore = { ...this.docStore, ...indexer.getDocumentStore() };\n    });\n\n    return indexer;\n  }\n\n  async query(query: SearchQuery): Promise<IndexableResultSet> {\n    const { lunrQueryBuilder, documentTypes, pageSize } = this.translator(\n      query,\n    ) as ConcreteLunrQuery;\n\n    const results: LunrResultEnvelope[] = [];\n\n    // Iterate over the filtered list of this.lunrIndex keys.\n    Object.keys(this.lunrIndices)\n      .filter(type => !documentTypes || documentTypes.includes(type))\n      .forEach(type => {\n        try {\n          results.push(\n            ...this.lunrIndices[type].query(lunrQueryBuilder).map(result => {\n              return {\n                result: result,\n                type: type,\n              };\n            }),\n          );\n        } catch (err) {\n          // if a field does not exist on a index, we can see that as a no-match\n          if (\n            err instanceof Error &&\n            err.message.startsWith('unrecognised field')\n          ) {\n            return;\n          }\n          throw err;\n        }\n      });\n\n    // Sort results.\n    results.sort((doc1, doc2) => {\n      return doc2.result.score - doc1.result.score;\n    });\n\n    // Perform paging\n    const { page } = decodePageCursor(query.pageCursor);\n    const offset = page * pageSize;\n    const hasPreviousPage = page > 0;\n    const hasNextPage = results.length > offset + pageSize;\n    const nextPageCursor = hasNextPage\n      ? encodePageCursor({ page: page + 1 })\n      : undefined;\n    const previousPageCursor = hasPreviousPage\n      ? encodePageCursor({ page: page - 1 })\n      : undefined;\n\n    // Translate results into IndexableResultSet\n    const realResultSet: IndexableResultSet = {\n      results: results.slice(offset, offset + pageSize).map(d => ({\n        type: d.type,\n        document: this.docStore[d.result.ref],\n        highlight: {\n          preTag: this.highlightPreTag,\n          postTag: this.highlightPostTag,\n          fields: parseHighlightFields({\n            preTag: this.highlightPreTag,\n            postTag: this.highlightPostTag,\n            doc: this.docStore[d.result.ref],\n            positionMetadata: d.result.matchData.metadata as any,\n          }),\n        },\n      })),\n      nextPageCursor,\n      previousPageCursor,\n    };\n\n    return realResultSet;\n  }\n}\n\nexport function decodePageCursor(pageCursor?: string): { page: number } {\n  if (!pageCursor) {\n    return { page: 0 };\n  }\n\n  return {\n    page: Number(Buffer.from(pageCursor, 'base64').toString('utf-8')),\n  };\n}\n\nexport function encodePageCursor({ page }: { page: number }): string {\n  return Buffer.from(`${page}`, 'utf-8').toString('base64');\n}\n\ntype ParseHighlightFieldsProps = {\n  preTag: string;\n  postTag: string;\n  doc: any;\n  positionMetadata: {\n    [term: string]: {\n      [field: string]: {\n        position: number[][];\n      };\n    };\n  };\n};\n\nexport function parseHighlightFields({\n  preTag,\n  postTag,\n  doc,\n  positionMetadata,\n}: ParseHighlightFieldsProps): { [field: string]: string } {\n  // Merge the field positions across all query terms\n  const highlightFieldPositions = Object.values(positionMetadata).reduce(\n    (fieldPositions, metadata) => {\n      Object.keys(metadata).map(fieldKey => {\n        fieldPositions[fieldKey] = fieldPositions[fieldKey] ?? [];\n        fieldPositions[fieldKey].push(...metadata[fieldKey].position);\n      });\n\n      return fieldPositions;\n    },\n    {} as { [field: string]: number[][] },\n  );\n\n  return Object.fromEntries(\n    Object.entries(highlightFieldPositions).map(([field, positions]) => {\n      positions.sort((a, b) => b[0] - a[0]);\n\n      const highlightedField = positions.reduce((content, pos) => {\n        return (\n          `${content.substring(0, pos[0])}${preTag}` +\n          `${content.substring(pos[0], pos[0] + pos[1])}` +\n          `${postTag}${content.substring(pos[0] + pos[1])}`\n        );\n      }, doc[field]);\n\n      return [field, highlightedField];\n    }),\n  );\n}\n","/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { IndexableDocument } from '@backstage/plugin-search-common';\nimport { pipeline, Readable, Transform, Writable } from 'stream';\n\n/**\n * Object resolved after a test pipeline is executed.\n * @public\n */\nexport type TestPipelineResult = {\n  /**\n   * If an error was emitted by the pipeline, it will be set here.\n   */\n  error: unknown;\n\n  /**\n   * A list of documents collected at the end of the pipeline. If the subject\n   * under test is an indexer, this will be an empty array (because your\n   * indexer should have received the documents instead).\n   */\n  documents: IndexableDocument[];\n};\n\n/**\n * Test utility for Backstage Search collators, decorators, and indexers.\n * @public\n */\nexport class TestPipeline {\n  private collator?: Readable;\n  private decorator?: Transform;\n  private indexer?: Writable;\n\n  private constructor({\n    collator,\n    decorator,\n    indexer,\n  }: {\n    collator?: Readable;\n    decorator?: Transform;\n    indexer?: Writable;\n  }) {\n    this.collator = collator;\n    this.decorator = decorator;\n    this.indexer = indexer;\n  }\n\n  /**\n   * Provide the collator, decorator, or indexer to be tested.\n   */\n  static withSubject(subject: Readable | Transform | Writable) {\n    if (subject instanceof Transform) {\n      return new TestPipeline({ decorator: subject });\n    }\n\n    if (subject instanceof Writable) {\n      return new TestPipeline({ indexer: subject });\n    }\n\n    if (subject.readable || subject instanceof Readable) {\n      return new TestPipeline({ collator: subject });\n    }\n\n    throw new Error(\n      'Unknown test subject: are you passing a readable, writable, or transform stream?',\n    );\n  }\n\n  /**\n   * Provide documents for testing decorators and indexers.\n   */\n  withDocuments(documents: IndexableDocument[]): TestPipeline {\n    if (this.collator) {\n      throw new Error('Cannot provide documents when testing a collator.');\n    }\n\n    // Set a naive readable stream that just pushes all given documents.\n    this.collator = new Readable({ objectMode: true });\n    this.collator._read = () => {};\n    process.nextTick(() => {\n      documents.forEach(document => {\n        this.collator!.push(document);\n      });\n      this.collator!.push(null);\n    });\n\n    return this;\n  }\n\n  /**\n   * Execute the test pipeline so that you can make assertions about the result\n   * or behavior of the given test subject.\n   */\n  async execute(): Promise<TestPipelineResult> {\n    const documents: IndexableDocument[] = [];\n    if (!this.collator) {\n      throw new Error(\n        'Cannot execute pipeline without a collator or documents',\n      );\n    }\n\n    // If we are here and there is no indexer, we are testing a collator or a\n    // decorator. Set up a naive writable that captures documents in memory.\n    if (!this.indexer) {\n      this.indexer = new Writable({ objectMode: true });\n      this.indexer._write = (document: IndexableDocument, _, done) => {\n        documents.push(document);\n        done();\n      };\n    }\n\n    return new Promise<TestPipelineResult>(done => {\n      const pipes: (Readable | Transform | Writable)[] = [this.collator!];\n      if (this.decorator) {\n        pipes.push(this.decorator);\n      }\n      pipes.push(this.indexer!);\n\n      pipeline(pipes, (error: NodeJS.ErrnoException | null) => {\n        done({\n          error,\n          documents,\n        });\n      });\n    });\n  }\n}\n"],"names":["AbortController","pipeline","parseNdjson","Writable","assertError","Transform","lunr","uuid","Readable"],"mappings":";;;;;;;;;;;;;;;AACO,MAAM,SAAS,CAAC;AACvB,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE;AAC1B,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;AACzB,IAAI,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;AACvB,IAAI,IAAI,CAAC,eAAe,GAAG,IAAIA,mCAAe,EAAE,CAAC;AACjD,IAAI,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;AAC3B,GAAG;AACH,EAAE,aAAa,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,EAAE;AAC/C,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE;AACxB,MAAM,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;AACpF,KAAK;AACL,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE;AAC3B,MAAM,MAAM,IAAI,KAAK,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC;AAC5D,KAAK;AACL,IAAI,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;AAClD,GAAG;AACH,EAAE,KAAK,GAAG;AACV,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;AAC7D,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;AAC1B,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK;AAC/C,MAAM,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AAC1D,MAAM,eAAe,CAAC,GAAG,CAAC;AAC1B,QAAQ,EAAE;AACV,QAAQ,EAAE,EAAE,IAAI;AAChB,QAAQ,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;AAC3C,OAAO,CAAC,CAAC;AACT,KAAK,CAAC,CAAC;AACP,GAAG;AACH,EAAE,IAAI,GAAG;AACT,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;AAC7D,IAAI,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;AACjC,IAAI,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;AAC3B,GAAG;AACH;;AChCO,MAAM,YAAY,CAAC;AAC1B,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,EAAE;AACxC,IAAI,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;AACxB,IAAI,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;AACzB,IAAI,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;AAC5B,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;AACzB,IAAI,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;AACrC,GAAG;AACH,EAAE,eAAe,GAAG;AACpB,IAAI,OAAO,IAAI,CAAC,YAAY,CAAC;AAC7B,GAAG;AACH,EAAE,gBAAgB,GAAG;AACrB,IAAI,OAAO,IAAI,CAAC,aAAa,CAAC;AAC9B,GAAG;AACH,EAAE,WAAW,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;AACrC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,2BAA2B,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACpG,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG;AACnC,MAAM,OAAO;AACb,MAAM,QAAQ;AACd,KAAK,CAAC;AACN,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG;AACvC,MAAM,oBAAoB,EAAE,OAAO,CAAC,oBAAoB;AACxD,KAAK,CAAC;AACN,GAAG;AACH,EAAE,YAAY,CAAC,EAAE,OAAO,EAAE,EAAE;AAC5B,IAAI,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC;AACzC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AACjG,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK;AAC5B,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE;AAChD,QAAQ,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC5C,OAAO,MAAM;AACb,QAAQ,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AAC1C,OAAO;AACP,KAAK,CAAC,CAAC;AACP,GAAG;AACH,EAAE,MAAM,KAAK,GAAG;AAChB,IAAI,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC;AACpC,MAAM,MAAM,EAAE,IAAI,CAAC,MAAM;AACzB,KAAK,CAAC,CAAC;AACP,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK;AAClD,MAAM,SAAS,CAAC,aAAa,CAAC;AAC9B,QAAQ,EAAE,EAAE,CAAC,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;AAC/E,QAAQ,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ;AACtD,QAAQ,IAAI,EAAE,YAAY;AAC1B,UAAU,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;AAC5E,UAAU,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,wBAAwB,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACnH,UAAU,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,OAAO,KAAK;AACvI,YAAY,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;AAC3D,YAAY,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,uBAAuB,EAAE,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;AAC9G,YAAY,OAAO,SAAS,CAAC;AAC7B,WAAW,CAAC,CAAC,CAAC;AACd,UAAU,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;AACnE,UAAU,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,KAAK;AAClD,YAAYC,eAAQ,CAAC,CAAC,QAAQ,EAAE,GAAG,UAAU,EAAE,OAAO,CAAC,EAAE,CAAC,KAAK,KAAK;AACpE,cAAc,IAAI,KAAK,EAAE;AACzB,gBAAgB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,wBAAwB,EAAE,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;AACtF,gBAAgB,MAAM,CAAC,KAAK,CAAC,CAAC;AAC9B,eAAe,MAAM;AACrB,gBAAgB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,wBAAwB,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;AAC9E,gBAAgB,OAAO,EAAE,CAAC;AAC1B,eAAe;AACf,aAAa,CAAC,CAAC;AACf,WAAW,CAAC,CAAC;AACb,SAAS;AACT,OAAO,CAAC,CAAC;AACT,KAAK,CAAC,CAAC;AACP,IAAI,OAAO;AACX,MAAM,SAAS;AACf,KAAK,CAAC;AACN,GAAG;AACH;;ACvEO,MAAM,mCAAmC,CAAC;AACjD,EAAE,WAAW,CAAC,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,oBAAoB,EAAE;AACzE,IAAI,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;AACvC,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;AACzB,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;AACzB,IAAI,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;AACrB,IAAI,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;AACrD,GAAG;AACH,EAAE,OAAO,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE;AACtC,IAAI,OAAO,IAAI,mCAAmC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;AACtJ,GAAG;AACH,EAAE,MAAM,OAAO,GAAG;AAClB,IAAI,IAAI,EAAE,CAAC;AACX,IAAI,IAAI;AACR,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,2CAA2C,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;AAC3F,MAAM,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;AACrE,MAAM,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;AACnI,MAAM,OAAO,CAAC,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC;AAC5D,KAAK,CAAC,OAAO,CAAC,EAAE;AAChB,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,qBAAqB,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACzE,MAAM,MAAM,CAAC,CAAC;AACd,KAAK;AACL,GAAG;AACH,EAAE,MAAM,WAAW,GAAG;AACtB,IAAI,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;AACzC,IAAI,IAAI,CAAC,OAAO,EAAE;AAClB,MAAM,MAAM,cAAc,GAAG,CAAC,wCAAwC,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC;AAC7F,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;AACxC,MAAM,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC;AACtC,KAAK,MAAM;AACX,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;AAC/D,KAAK;AACL,IAAI,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;AAC9D,IAAI,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;AAC3C,IAAI,OAAO,MAAM,CAAC,IAAI,CAACC,YAAW,EAAE,CAAC,CAAC;AACtC,GAAG;AACH;;ACnCO,MAAM,wBAAwB,SAASC,eAAQ,CAAC;AACvD,EAAE,WAAW,CAAC,OAAO,EAAE;AACvB,IAAI,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;AAChC,IAAI,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;AAC3B,IAAI,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;AACvC,IAAI,IAAI,CAAC,WAAW,GAAG,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK;AAC7C,MAAM,YAAY,CAAC,YAAY;AAC/B,QAAQ,IAAI;AACZ,UAAU,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;AAClC,UAAU,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACvB,SAAS,CAAC,OAAO,CAAC,EAAE;AACpB,UAAUC,kBAAW,CAAC,CAAC,CAAC,CAAC;AACzB,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,SAAS;AACT,OAAO,CAAC,CAAC;AACT,KAAK,CAAC,CAAC;AACP,GAAG;AACH,EAAE,MAAM,MAAM,CAAC,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE;AAC9B,IAAI,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC;AAC9C,IAAI,IAAI,UAAU,EAAE;AACpB,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;AACvB,MAAM,OAAO;AACb,KAAK;AACL,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAChC,IAAI,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE;AACnD,MAAM,IAAI,EAAE,CAAC;AACb,MAAM,OAAO;AACb,KAAK;AACL,IAAI,IAAI;AACR,MAAM,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;AAC1C,MAAM,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;AAC7B,MAAM,IAAI,EAAE,CAAC;AACb,KAAK,CAAC,OAAO,CAAC,EAAE;AAChB,MAAMA,kBAAW,CAAC,CAAC,CAAC,CAAC;AACrB,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC;AACd,KAAK;AACL,GAAG;AACH,EAAE,MAAM,MAAM,CAAC,IAAI,EAAE;AACrB,IAAI,IAAI;AACR,MAAM,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE;AACpC,QAAQ,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;AAC5C,QAAQ,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;AAC/B,OAAO;AACP,MAAM,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;AAC5B,MAAM,IAAI,EAAE,CAAC;AACb,KAAK,CAAC,OAAO,CAAC,EAAE;AAChB,MAAMA,kBAAW,CAAC,CAAC,CAAC,CAAC;AACrB,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC;AACd,KAAK;AACL,GAAG;AACH;;AClDO,MAAM,aAAa,SAASC,gBAAS,CAAC;AAC7C,EAAE,WAAW,GAAG;AAChB,IAAI,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;AAChC,IAAI,IAAI,CAAC,WAAW,GAAG,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK;AAC7C,MAAM,YAAY,CAAC,YAAY;AAC/B,QAAQ,IAAI;AACZ,UAAU,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;AAClC,UAAU,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACvB,SAAS,CAAC,OAAO,CAAC,EAAE;AACpB,UAAUD,kBAAW,CAAC,CAAC,CAAC,CAAC;AACzB,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,SAAS;AACT,OAAO,CAAC,CAAC;AACT,KAAK,CAAC,CAAC;AACP,GAAG;AACH,EAAE,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE;AACtC,IAAI,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC;AAC9C,IAAI,IAAI,UAAU,EAAE;AACpB,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;AACvB,MAAM,OAAO;AACb,KAAK;AACL,IAAI,IAAI;AACR,MAAM,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AACtD,MAAM,IAAI,SAAS,KAAK,KAAK,CAAC,EAAE;AAChC,QAAQ,IAAI,EAAE,CAAC;AACf,QAAQ,OAAO;AACf,OAAO;AACP,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;AACpC,QAAQ,SAAS,CAAC,OAAO,CAAC,CAAC,GAAG,KAAK;AACnC,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,SAAS,CAAC,CAAC;AACX,QAAQ,IAAI,EAAE,CAAC;AACf,QAAQ,OAAO;AACf,OAAO;AACP,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AAC3B,MAAM,IAAI,EAAE,CAAC;AACb,KAAK,CAAC,OAAO,CAAC,EAAE;AAChB,MAAMA,kBAAW,CAAC,CAAC,CAAC,CAAC;AACrB,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC;AACd,KAAK;AACL,GAAG;AACH,EAAE,MAAM,MAAM,CAAC,IAAI,EAAE;AACrB,IAAI,IAAI;AACR,MAAM,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;AAC5B,MAAM,IAAI,EAAE,CAAC;AACb,KAAK,CAAC,OAAO,CAAC,EAAE;AAChB,MAAMA,kBAAW,CAAC,CAAC,CAAC,CAAC;AACrB,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC;AACd,KAAK;AACL,GAAG;AACH;;AClDO,MAAM,uBAAuB,SAAS,wBAAwB,CAAC;AACtE,EAAE,WAAW,GAAG;AAChB,IAAI,KAAK,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;AAC9B,IAAI,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;AACnC,IAAI,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;AACvB,IAAI,IAAI,CAAC,OAAO,GAAG,IAAIE,wBAAI,CAAC,OAAO,EAAE,CAAC;AACtC,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAACA,wBAAI,CAAC,OAAO,EAAEA,wBAAI,CAAC,cAAc,EAAEA,wBAAI,CAAC,OAAO,CAAC,CAAC;AAC/E,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAACA,wBAAI,CAAC,OAAO,CAAC,CAAC;AAClD,IAAI,IAAI,CAAC,OAAO,CAAC,iBAAiB,GAAG,CAAC,UAAU,CAAC,CAAC;AAClD,GAAG;AACH,EAAE,MAAM,UAAU,GAAG;AACrB,GAAG;AACH,EAAE,MAAM,QAAQ,GAAG;AACnB,GAAG;AACH,EAAE,MAAM,KAAK,CAAC,SAAS,EAAE;AACzB,IAAI,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE;AACjC,MAAM,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,KAAK;AACnD,QAAQ,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAClC,OAAO,CAAC,CAAC;AACT,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;AACnC,MAAM,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;AACpC,KAAK;AACL,IAAI,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,KAAK;AACpC,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;AACjC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,QAAQ,CAAC;AAClD,KAAK,CAAC,CAAC;AACP,GAAG;AACH,EAAE,UAAU,GAAG;AACf,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;AAChC,GAAG;AACH,EAAE,gBAAgB,GAAG;AACrB,IAAI,OAAO,IAAI,CAAC,QAAQ,CAAC;AACzB,GAAG;AACH;;AChCO,MAAM,gBAAgB,CAAC;AAC9B,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE;AAC1B,IAAI,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;AAC1B,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC;AACvB,MAAM,IAAI;AACV,MAAM,OAAO;AACb,MAAM,KAAK;AACX,KAAK,KAAK;AACV,MAAM,MAAM,QAAQ,GAAG,EAAE,CAAC;AAC1B,MAAM,OAAO;AACb,QAAQ,gBAAgB,EAAE,CAAC,CAAC,KAAK;AACjC,UAAU,MAAM,SAAS,GAAGA,wBAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;AACjD,UAAU,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE;AAC5B,YAAY,WAAW,EAAE,IAAI;AAC7B,YAAY,KAAK,EAAE,GAAG;AACtB,WAAW,CAAC,CAAC;AACb,UAAU,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE;AAC5B,YAAY,WAAW,EAAE,KAAK;AAC9B,YAAY,KAAK,EAAE,EAAE;AACrB,YAAY,QAAQ,EAAEA,wBAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ;AAClD,WAAW,CAAC,CAAC;AACb,UAAU,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE;AAC5B,YAAY,WAAW,EAAE,KAAK;AAC9B,YAAY,YAAY,EAAE,CAAC;AAC3B,YAAY,KAAK,EAAE,CAAC;AACpB,WAAW,CAAC,CAAC;AACb,UAAU,IAAI,OAAO,EAAE;AACvB,YAAY,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,KAAK;AACrE,cAAc,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;AAChD,gBAAgB,MAAM,IAAI,KAAK,CAAC,CAAC,mBAAmB,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;AAC/D,eAAe;AACf,cAAc,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9G,cAAc,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,OAAO,KAAK,CAAC,EAAE;AAC1E,gBAAgB,CAAC,CAAC,IAAI,CAACA,wBAAI,CAAC,SAAS,CAAC,KAAK,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,EAAE;AAClF,kBAAkB,QAAQ,EAAEA,wBAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ;AACxD,kBAAkB,MAAM,EAAE,CAAC,KAAK,CAAC;AACjC,iBAAiB,CAAC,CAAC;AACnB,eAAe,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;AAC/C,gBAAgB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,uCAAuC,EAAE,KAAK,CAAC,8DAA8D,CAAC,CAAC,CAAC;AAClJ,gBAAgB,CAAC,CAAC,IAAI,CAACA,wBAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE;AAC9C,kBAAkB,QAAQ,EAAEA,wBAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ;AACxD,kBAAkB,MAAM,EAAE,CAAC,KAAK,CAAC;AACjC,iBAAiB,CAAC,CAAC;AACnB,eAAe,MAAM;AACrB,gBAAgB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,kCAAkC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;AAC/E,eAAe;AACf,aAAa,CAAC,CAAC;AACf,WAAW;AACX,SAAS;AACT,QAAQ,aAAa,EAAE,KAAK;AAC5B,QAAQ,QAAQ;AAChB,OAAO,CAAC;AACR,KAAK,CAAC;AACN,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;AACzB,IAAI,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;AACvB,IAAI,MAAM,OAAO,GAAGC,OAAI,EAAE,CAAC;AAC3B,IAAI,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;AAC1C,IAAI,IAAI,CAAC,gBAAgB,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;AAC5C,GAAG;AACH,EAAE,aAAa,CAAC,UAAU,EAAE;AAC5B,IAAI,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;AACjC,GAAG;AACH,EAAE,MAAM,UAAU,CAAC,IAAI,EAAE;AACzB,IAAI,MAAM,OAAO,GAAG,IAAI,uBAAuB,EAAE,CAAC;AAClD,IAAI,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM;AAC9B,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;AACpD,MAAM,IAAI,CAAC,QAAQ,GAAG,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC;AAC1E,KAAK,CAAC,CAAC;AACP,IAAI,OAAO,OAAO,CAAC;AACnB,GAAG;AACH,EAAE,MAAM,KAAK,CAAC,KAAK,EAAE;AACrB,IAAI,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;AACjF,IAAI,MAAM,OAAO,GAAG,EAAE,CAAC;AACvB,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,CAAC,aAAa,IAAI,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK;AACrH,MAAM,IAAI;AACV,QAAQ,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK;AACvF,UAAU,OAAO;AACjB,YAAY,MAAM;AAClB,YAAY,IAAI;AAChB,WAAW,CAAC;AACZ,SAAS,CAAC,CAAC,CAAC;AACZ,OAAO,CAAC,OAAO,GAAG,EAAE;AACpB,QAAQ,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,oBAAoB,CAAC,EAAE;AAClF,UAAU,OAAO;AACjB,SAAS;AACT,QAAQ,MAAM,GAAG,CAAC;AAClB,OAAO;AACP,KAAK,CAAC,CAAC;AACP,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK;AACjC,MAAM,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;AACnD,KAAK,CAAC,CAAC;AACP,IAAI,MAAM,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;AACxD,IAAI,MAAM,MAAM,GAAG,IAAI,GAAG,QAAQ,CAAC;AACnC,IAAI,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC;AACrC,IAAI,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;AAC3D,IAAI,MAAM,cAAc,GAAG,WAAW,GAAG,gBAAgB,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC;AACvF,IAAI,MAAM,kBAAkB,GAAG,eAAe,GAAG,gBAAgB,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC;AAC/F,IAAI,MAAM,aAAa,GAAG;AAC1B,MAAM,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM;AACpE,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI;AACpB,QAAQ,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;AAC7C,QAAQ,SAAS,EAAE;AACnB,UAAU,MAAM,EAAE,IAAI,CAAC,eAAe;AACtC,UAAU,OAAO,EAAE,IAAI,CAAC,gBAAgB;AACxC,UAAU,MAAM,EAAE,oBAAoB,CAAC;AACvC,YAAY,MAAM,EAAE,IAAI,CAAC,eAAe;AACxC,YAAY,OAAO,EAAE,IAAI,CAAC,gBAAgB;AAC1C,YAAY,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;AAC5C,YAAY,gBAAgB,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ;AACzD,WAAW,CAAC;AACZ,SAAS;AACT,OAAO,CAAC,CAAC;AACT,MAAM,cAAc;AACpB,MAAM,kBAAkB;AACxB,KAAK,CAAC;AACN,IAAI,OAAO,aAAa,CAAC;AACzB,GAAG;AACH,CAAC;AACM,SAAS,gBAAgB,CAAC,UAAU,EAAE;AAC7C,EAAE,IAAI,CAAC,UAAU,EAAE;AACnB,IAAI,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;AACvB,GAAG;AACH,EAAE,OAAO;AACT,IAAI,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;AACrE,GAAG,CAAC;AACJ,CAAC;AACM,SAAS,gBAAgB,CAAC,EAAE,IAAI,EAAE,EAAE;AAC3C,EAAE,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAC5D,CAAC;AACM,SAAS,oBAAoB,CAAC;AACrC,EAAE,MAAM;AACR,EAAE,OAAO;AACT,EAAE,GAAG;AACL,EAAE,gBAAgB;AAClB,CAAC,EAAE;AACH,EAAE,MAAM,uBAAuB,GAAG,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC,cAAc,EAAE,QAAQ,KAAK;AACvG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,KAAK;AAC5C,MAAM,IAAI,EAAE,CAAC;AACb,MAAM,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,GAAG,cAAc,CAAC,QAAQ,CAAC,KAAK,IAAI,GAAG,EAAE,GAAG,EAAE,CAAC;AACnF,MAAM,cAAc,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC;AACpE,KAAK,CAAC,CAAC;AACP,IAAI,OAAO,cAAc,CAAC;AAC1B,GAAG,EAAE,EAAE,CAAC,CAAC;AACT,EAAE,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,SAAS,CAAC,KAAK;AAChG,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC1C,IAAI,MAAM,gBAAgB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,GAAG,KAAK;AAChE,MAAM,OAAO,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACpJ,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;AACnB,IAAI,OAAO,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC;AACrC,GAAG,CAAC,CAAC,CAAC;AACN;;ACxJO,MAAM,YAAY,CAAC;AAC1B,EAAE,WAAW,CAAC;AACd,IAAI,QAAQ;AACZ,IAAI,SAAS;AACb,IAAI,OAAO;AACX,GAAG,EAAE;AACL,IAAI,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;AAC7B,IAAI,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;AAC/B,IAAI,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;AAC3B,GAAG;AACH,EAAE,OAAO,WAAW,CAAC,OAAO,EAAE;AAC9B,IAAI,IAAI,OAAO,YAAYF,gBAAS,EAAE;AACtC,MAAM,OAAO,IAAI,YAAY,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;AACtD,KAAK;AACL,IAAI,IAAI,OAAO,YAAYF,eAAQ,EAAE;AACrC,MAAM,OAAO,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;AACpD,KAAK;AACL,IAAI,IAAI,OAAO,CAAC,QAAQ,IAAI,OAAO,YAAYK,eAAQ,EAAE;AACzD,MAAM,OAAO,IAAI,YAAY,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;AACrD,KAAK;AACL,IAAI,MAAM,IAAI,KAAK,CAAC,kFAAkF,CAAC,CAAC;AACxG,GAAG;AACH,EAAE,aAAa,CAAC,SAAS,EAAE;AAC3B,IAAI,IAAI,IAAI,CAAC,QAAQ,EAAE;AACvB,MAAM,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;AAC3E,KAAK;AACL,IAAI,IAAI,CAAC,QAAQ,GAAG,IAAIA,eAAQ,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;AACvD,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,MAAM;AAChC,KAAK,CAAC;AACN,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM;AAC3B,MAAM,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,KAAK;AACtC,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AACrC,OAAO,CAAC,CAAC;AACT,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC/B,KAAK,CAAC,CAAC;AACP,IAAI,OAAO,IAAI,CAAC;AAChB,GAAG;AACH,EAAE,MAAM,OAAO,GAAG;AAClB,IAAI,MAAM,SAAS,GAAG,EAAE,CAAC;AACzB,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;AACxB,MAAM,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;AACjF,KAAK;AACL,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;AACvB,MAAM,IAAI,CAAC,OAAO,GAAG,IAAIL,eAAQ,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;AACxD,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK;AACnD,QAAQ,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AACjC,QAAQ,IAAI,EAAE,CAAC;AACf,OAAO,CAAC;AACR,KAAK;AACL,IAAI,OAAO,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK;AACjC,MAAM,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AACpC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE;AAC1B,QAAQ,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AACnC,OAAO;AACP,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC/B,MAAMF,eAAQ,CAAC,KAAK,EAAE,CAAC,KAAK,KAAK;AACjC,QAAQ,IAAI,CAAC;AACb,UAAU,KAAK;AACf,UAAU,SAAS;AACnB,SAAS,CAAC,CAAC;AACX,OAAO,CAAC,CAAC;AACT,KAAK,CAAC,CAAC;AACP,GAAG;AACH;;;;;;;;;;"}

package/dist/index.d.ts

@@ -3,19 +3,24 @@ import { SearchEngine, DocumentCollatorFactory, DocumentDecoratorFactory, Docume
 export { SearchEngine } from '@backstage/plugin-search-common';
 import { Logger } from 'winston';
 import { TaskFunction, TaskRunner } from '@backstage/backend-tasks';
+import { UrlReader } from '@backstage/backend-common';
+import { Config } from '@backstage/config';
+import { Permission } from '@backstage/plugin-permission-common';
+import { Readable, Writable, Transform } from 'stream';
 import lunr from 'lunr';
-import { Writable, Transform, Readable } from 'stream';
 
 /**
- * @public ScheduleTaskParameters
+ * ScheduleTaskParameters
+ * @public
  */
-interface ScheduleTaskParameters {
+declare type ScheduleTaskParameters = {
     id: string;
     task: TaskFunction;
     scheduledRunner: TaskRunner;
-}
+};
 /**
- * @beta
+ * Scheduler responsible for all search tasks.
+ * @public
  */
 declare class Scheduler {
     private logger;
@@ -42,7 +47,8 @@ declare class Scheduler {
 }
 
 /**
- * @beta
+ * Options required to instantiate the index builder.
+ * @public
  */
 declare type IndexBuilderOptions = {
     searchEngine: SearchEngine;
@@ -50,7 +56,7 @@ declare type IndexBuilderOptions = {
 };
 /**
  * Parameters required to register a collator.
- * @beta
+ * @public
  */
 interface RegisterCollatorParameters {
     /**
@@ -65,7 +71,7 @@ interface RegisterCollatorParameters {
 }
 /**
  * Parameters required to register a decorator
- * @beta
+ * @public
  */
 interface RegisterDecoratorParameters {
     /**
@@ -75,7 +81,8 @@ interface RegisterDecoratorParameters {
 }
 
 /**
- * @beta
+ * Used for adding collators, decorators and compile them into tasks which are added to a scheduler returned to the caller.
+ * @public
  */
 declare class IndexBuilder {
     private collators;
@@ -84,7 +91,13 @@ declare class IndexBuilder {
     private searchEngine;
     private logger;
     constructor({ logger, searchEngine }: IndexBuilderOptions);
+    /**
+     * Responsible for returning the registered search engine.
+     */
     getSearchEngine(): SearchEngine;
+    /**
+     * Responsible for returning the registered document types.
+     */
     getDocumentTypes(): Record<string, DocumentTypeInfo>;
     /**
      * Makes the index builder aware of a collator that should be executed at the
@@ -107,7 +120,68 @@ declare class IndexBuilder {
 }
 
 /**
- * @beta
+ * Options for instansiate NewlineDelimitedJsonCollatorFactory
+ * @public
+ */
+declare type NewlineDelimitedJsonCollatorFactoryOptions = {
+    type: string;
+    searchPattern: string;
+    reader: UrlReader;
+    logger: Logger;
+    visibilityPermission?: Permission;
+};
+/**
+ * Factory class producing a collator that can be used to index documents
+ * sourced from the latest newline delimited JSON file matching a given search
+ * pattern. "Latest" is determined by the name of the file (last alphabetically
+ * is considered latest).
+ *
+ * @remarks
+ * The reader provided must implement the `search()` method as well as the
+ * `readUrl` method whose response includes the `stream()` method. Naturally,
+ * the reader must also be configured to understand the given search pattern.
+ *
+ * @example
+ * Here's an example configuration using Google Cloud Storage, which would
+ * return the latest file under the `bucket` GCS bucket with files like
+ * `xyz-2021.ndjson` or `xyz-2022.ndjson`.
+ * ```ts
+ * indexBuilder.addCollator({
+ *   schedule,
+ *   factory: NewlineDelimitedJsonCollatorFactory.fromConfig(env.config, {
+ *     type: 'techdocs',
+ *     searchPattern: 'https://storage.cloud.google.com/bucket/xyz-*',
+ *     reader: env.reader,
+ *     logger: env.logger,
+ *   })
+ * });
+ * ```
+ *
+ * @public
+ */
+declare class NewlineDelimitedJsonCollatorFactory implements DocumentCollatorFactory {
+    private readonly searchPattern;
+    private readonly reader;
+    private readonly logger;
+    readonly type: string;
+    readonly visibilityPermission: Permission | undefined;
+    private constructor();
+    /**
+     * Returns a NewlineDelimitedJsonCollatorFactory instance from configuration
+     * and a set of options.
+     */
+    static fromConfig(_config: Config, options: NewlineDelimitedJsonCollatorFactoryOptions): NewlineDelimitedJsonCollatorFactory;
+    /**
+     * Returns the "latest" URL for the given search pattern (e.g. the one at the
+     * end of the list, sorted alphabetically).
+     */
+    private lastUrl;
+    getCollator(): Promise<Readable>;
+}
+
+/**
+ * Options for {@link BatchSearchEngineIndexer}
+ * @public
  */
 declare type BatchSearchEngineOptions = {
     batchSize: number;
@@ -115,7 +189,7 @@ declare type BatchSearchEngineOptions = {
 /**
  * Base class encapsulating batch-based stream processing. Useful as a base
  * class for search engine indexers.
- * @beta
+ * @public
  */
 declare abstract class BatchSearchEngineIndexer extends Writable {
     private batchSize;
@@ -141,7 +215,7 @@ declare abstract class BatchSearchEngineIndexer extends Writable {
 /**
  * Base class encapsulating simple async transformations. Useful as a base
  * class for Backstage search decorators.
- * @beta
+ * @public
  */
 declare abstract class DecoratorBase extends Transform {
     private initialized;
@@ -167,7 +241,8 @@ declare abstract class DecoratorBase extends Transform {
 }
 
 /**
- * @beta
+ * Lunr specific search engine indexer
+ * @public
  */
 declare class LunrSearchEngineIndexer extends BatchSearchEngineIndexer {
     private schemaInitialized;
@@ -182,7 +257,8 @@ declare class LunrSearchEngineIndexer extends BatchSearchEngineIndexer {
 }
 
 /**
- * @beta
+ * Type of translated query for the Lunr Search Engine.
+ * @public
  */
 declare type ConcreteLunrQuery = {
     lunrQueryBuilder: lunr.Index.QueryBuilder;
@@ -190,11 +266,13 @@ declare type ConcreteLunrQuery = {
     pageSize: number;
 };
 /**
- * @beta
+ * Translator repsonsible for translating search term and filters to a query that the Lunr Search Engine understands.
+ * @public
  */
 declare type LunrQueryTranslator = (query: SearchQuery) => ConcreteLunrQuery;
 /**
- * @beta
+ * Lunr specific search engine implementation.
+ * @public
  */
 declare class LunrSearchEngine implements SearchEngine {
     protected lunrIndices: Record<string, lunr.Index>;
@@ -213,7 +291,7 @@ declare class LunrSearchEngine implements SearchEngine {
 
 /**
  * Object resolved after a test pipeline is executed.
- * @beta
+ * @public
  */
 declare type TestPipelineResult = {
     /**
@@ -229,7 +307,7 @@ declare type TestPipelineResult = {
 };
 /**
  * Test utility for Backstage Search collators, decorators, and indexers.
- * @beta
+ * @public
  */
 declare class TestPipeline {
     private collator?;
@@ -251,4 +329,4 @@ declare class TestPipeline {
     execute(): Promise<TestPipelineResult>;
 }
 
-export { BatchSearchEngineIndexer, BatchSearchEngineOptions, ConcreteLunrQuery, DecoratorBase, IndexBuilder, IndexBuilderOptions, LunrQueryTranslator, LunrSearchEngine, LunrSearchEngineIndexer, RegisterCollatorParameters, RegisterDecoratorParameters, ScheduleTaskParameters, Scheduler, TestPipeline, TestPipelineResult };
+export { BatchSearchEngineIndexer, BatchSearchEngineOptions, ConcreteLunrQuery, DecoratorBase, IndexBuilder, IndexBuilderOptions, LunrQueryTranslator, LunrSearchEngine, LunrSearchEngineIndexer, NewlineDelimitedJsonCollatorFactory, NewlineDelimitedJsonCollatorFactoryOptions, RegisterCollatorParameters, RegisterDecoratorParameters, ScheduleTaskParameters, Scheduler, TestPipeline, TestPipelineResult };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@backstage/plugin-search-backend-node",
   "description": "A library for Backstage backend plugins that want to interact with the search backend plugin",
-  "version": "0.6.1",
+  "version": "0.6.2-next.2",
   "main": "dist/index.cjs.js",
   "types": "dist/index.d.ts",
   "license": "Apache-2.0",
@@ -23,22 +23,27 @@
     "clean": "backstage-cli package clean"
   },
   "dependencies": {
-    "@backstage/backend-tasks": "^0.3.1",
+    "@backstage/backend-common": "^0.14.0-next.2",
+    "@backstage/backend-tasks": "^0.3.2-next.2",
+    "@backstage/config": "^1.0.1",
     "@backstage/errors": "^1.0.0",
-    "@backstage/plugin-search-common": "^0.3.4",
+    "@backstage/plugin-permission-common": "^0.6.2-next.0",
+    "@backstage/plugin-search-common": "^0.3.5-next.1",
     "@types/lunr": "^2.3.3",
     "lodash": "^4.17.21",
     "lunr": "^2.3.9",
+    "ndjson": "^2.0.0",
     "node-abort-controller": "^3.0.1",
     "uuid": "^8.3.2",
     "winston": "^3.2.1"
   },
   "devDependencies": {
-    "@backstage/backend-common": "^0.13.3",
-    "@backstage/cli": "^0.17.1"
+    "@backstage/backend-common": "^0.14.0-next.2",
+    "@backstage/cli": "^0.17.2-next.2",
+    "@types/ndjson": "^2.0.1"
   },
   "files": [
     "dist"
   ],
-  "gitHead": "96323f280ba32ee526c5b151cda42260aee927c9"
+  "gitHead": "afc672d59763a835574e6c47819107d22cfd1ad7"
 }