@backstage/plugin-search-backend-module-elasticsearch 0.0.8 → 0.1.0

package/CHANGELOG.md

@@ -1,5 +1,43 @@
 # @backstage/plugin-search-backend-module-elasticsearch
 
+## 0.1.0
+
+### Minor Changes
+
+- 022507c860: **BREAKING**
+
+  The `ElasticSearchSearchEngine` implements the new stream-based indexing
+  process expected by the latest `@backstage/search-backend-node`.
+
+  When updating to this version, you must also update to the latest version of
+  `@backstage/search-backend-node`. Check [this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta)
+  for further details.
+
+### Patch Changes
+
+- Updated dependencies
+  - @backstage/plugin-search-backend-node@0.5.0
+  - @backstage/search-common@0.3.0
+
+## 0.0.10
+
+### Patch Changes
+
+- Fix for the previous release with missing type declarations.
+- Updated dependencies
+  - @backstage/config@0.1.15
+  - @backstage/search-common@0.2.4
+
+## 0.0.9
+
+### Patch Changes
+
+- c77c5c7eb6: Added `backstage.role` to `package.json`
+- 4c0332e55c: chore(deps-dev): bump `@elastic/elasticsearch-mock` from 0.3.0 to 1.0.0
+- Updated dependencies
+  - @backstage/config@0.1.14
+  - @backstage/search-common@0.2.3
+
 ## 0.0.8
 
 ### Patch Changes

package/dist/index.cjs.js

@@ -6,6 +6,8 @@ var awsEsConnection = require('@acuris/aws-es-connection');
 var elasticsearch = require('@elastic/elasticsearch');
 var esb = require('elastic-builder');
 var lodash = require('lodash');
+var pluginSearchBackendNode = require('@backstage/plugin-search-backend-node');
+var stream = require('stream');
 
 function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
 
@@ -16,6 +18,94 @@ function duration(startTimestamp) {
   const seconds = delta[0] + delta[1] / 1e9;
   return `${seconds.toFixed(1)}s`;
 }
+class ElasticSearchSearchEngineIndexer extends pluginSearchBackendNode.BatchSearchEngineIndexer {
+  constructor(options) {
+    super({ batchSize: 100 });
+    this.received = 0;
+    this.processed = 0;
+    this.removableIndices = [];
+    this.logger = options.logger;
+    this.startTimestamp = process.hrtime();
+    this.type = options.type;
+    this.indexPrefix = options.indexPrefix;
+    this.indexSeparator = options.indexSeparator;
+    this.indexName = this.constructIndexName(`${Date.now()}`);
+    this.alias = options.alias;
+    this.elasticSearchClient = options.elasticSearchClient;
+    this.sourceStream = new stream.Readable({ objectMode: true });
+    this.sourceStream._read = () => {
+    };
+    const that = this;
+    this.bulkResult = this.elasticSearchClient.helpers.bulk({
+      datasource: this.sourceStream,
+      onDocument() {
+        that.processed++;
+        return {
+          index: { _index: that.indexName }
+        };
+      },
+      refreshOnCompletion: that.indexName
+    });
+  }
+  async initialize() {
+    this.logger.info(`Started indexing documents for index ${this.type}`);
+    const aliases = await this.elasticSearchClient.cat.aliases({
+      format: "json",
+      name: this.alias
+    });
+    this.removableIndices = aliases.body.map((r) => r.index);
+    await this.elasticSearchClient.indices.create({
+      index: this.indexName
+    });
+  }
+  async index(documents) {
+    await this.isReady();
+    documents.forEach((document) => {
+      this.received++;
+      this.sourceStream.push(document);
+    });
+  }
+  async finalize() {
+    await this.isReady();
+    this.sourceStream.push(null);
+    const result = await this.bulkResult;
+    this.logger.info(`Indexing completed for index ${this.type} in ${duration(this.startTimestamp)}`, result);
+    await this.elasticSearchClient.indices.updateAliases({
+      body: {
+        actions: [
+          {
+            remove: { index: this.constructIndexName("*"), alias: this.alias }
+          },
+          { add: { index: this.indexName, alias: this.alias } }
+        ]
+      }
+    });
+    if (this.removableIndices.length) {
+      this.logger.info("Removing stale search indices", this.removableIndices);
+      try {
+        await this.elasticSearchClient.indices.delete({
+          index: this.removableIndices
+        });
+      } catch (e) {
+        this.logger.warn(`Failed to remove stale search indices: ${e}`);
+      }
+    }
+  }
+  isReady() {
+    return new Promise((resolve) => {
+      const interval = setInterval(() => {
+        if (this.received === this.processed) {
+          clearInterval(interval);
+          resolve();
+        }
+      }, 50);
+    });
+  }
+  constructIndexName(postFix) {
+    return `${this.indexPrefix}${this.type}${this.indexSeparator}${postFix}`;
+  }
+}
+
 function isBlank(str) {
   return lodash.isEmpty(str) && !lodash.isNumber(str) || lodash.isNaN(str);
 }
@@ -71,57 +161,34 @@ class ElasticSearchSearchEngine {
   setTranslator(translator) {
     this.translator = translator;
   }
-  async index(type, documents) {
-    this.logger.info(`Started indexing ${documents.length} documents for index ${type}`);
-    const startTimestamp = process.hrtime();
+  async getIndexer(type) {
     const alias = this.constructSearchAlias(type);
-    const index = this.constructIndexName(type, `${Date.now()}`);
-    try {
-      const aliases = await this.elasticSearchClient.cat.aliases({
-        format: "json",
-        name: alias
-      });
-      const removableIndices = aliases.body.map((r) => r.index);
-      await this.elasticSearchClient.indices.create({
-        index
-      });
-      const result = await this.elasticSearchClient.helpers.bulk({
-        datasource: documents,
-        onDocument() {
-          return {
-            index: { _index: index }
-          };
-        },
-        refreshOnCompletion: index
-      });
-      this.logger.info(`Indexing completed for index ${type} in ${duration(startTimestamp)}`, result);
-      await this.elasticSearchClient.indices.updateAliases({
-        body: {
-          actions: [
-            { remove: { index: this.constructIndexName(type, "*"), alias } },
-            { add: { index, alias } }
-          ]
-        }
-      });
-      this.logger.info("Removing stale search indices", removableIndices);
-      if (removableIndices.length) {
-        await this.elasticSearchClient.indices.delete({
-          index: removableIndices
-        });
-      }
-    } catch (e) {
+    const indexer = new ElasticSearchSearchEngineIndexer({
+      type,
+      indexPrefix: this.indexPrefix,
+      indexSeparator: this.indexSeparator,
+      alias,
+      elasticSearchClient: this.elasticSearchClient,
+      logger: this.logger
+    });
+    indexer.on("error", async (e) => {
       this.logger.error(`Failed to index documents for type ${type}`, e);
-      const response = await this.elasticSearchClient.indices.exists({
-        index
-      });
-      const indexCreated = response.body;
-      if (indexCreated) {
-        this.logger.info(`Removing created index ${index}`);
-        await this.elasticSearchClient.indices.delete({
-          index
+      try {
+        const response = await this.elasticSearchClient.indices.exists({
+          index: indexer.indexName
         });
+        const indexCreated = response.body;
+        if (indexCreated) {
+          this.logger.info(`Removing created index ${indexer.indexName}`);
+          await this.elasticSearchClient.indices.delete({
+            index: indexer.indexName
+          });
+        }
+      } catch (error) {
+        this.logger.error(`Unable to clean up elastic index: ${error}`);
       }
-    }
+    });
+    return indexer;
   }
   async query(query) {
     const { elasticSearchQuery, documentTypes, pageSize } = this.translator(query);
@@ -149,9 +216,6 @@ class ElasticSearchSearchEngine {
       return Promise.reject({ results: [] });
     }
   }
-  constructIndexName(type, postFix) {
-    return `${this.indexPrefix}${type}${this.indexSeparator}${postFix}`;
-  }
   getTypeFromIndex(index) {
     return index.substring(this.indexPrefix.length).split(this.indexSeparator)[0];
   }

package/dist/index.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.cjs.js","sources":["../src/engines/ElasticSearchSearchEngine.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  awsGetCredentials,\n  createAWSConnection,\n} from '@acuris/aws-es-connection';\nimport { Config } from '@backstage/config';\nimport {\n  IndexableDocument,\n  SearchEngine,\n  SearchQuery,\n  SearchResultSet,\n} from '@backstage/search-common';\nimport { Client } from '@elastic/elasticsearch';\nimport esb from 'elastic-builder';\nimport { isEmpty, isNaN as nan, isNumber } from 'lodash';\nimport { Logger } from 'winston';\n\nimport type { ElasticSearchClientOptions } from './ElasticSearchClientOptions';\n\nexport type { ElasticSearchClientOptions };\n\nexport type ConcreteElasticSearchQuery = {\n  documentTypes?: string[];\n  elasticSearchQuery: Object;\n  pageSize: number;\n};\n\ntype ElasticSearchQueryTranslator = (\n  query: SearchQuery,\n) => ConcreteElasticSearchQuery;\n\ntype ElasticSearchOptions = {\n  logger: Logger;\n  config: Config;\n  aliasPostfix?: string;\n  indexPrefix?: string;\n};\n\ntype ElasticSearchResult = {\n  _index: string;\n  _type: string;\n  _score: number;\n  _source: IndexableDocument;\n};\n\nfunction duration(startTimestamp: [number, number]): string {\n  const delta = process.hrtime(startTimestamp);\n  const seconds = delta[0] + delta[1] / 1e9;\n  return `${seconds.toFixed(1)}s`;\n}\n\nfunction isBlank(str: string) {\n  return (isEmpty(str) && !isNumber(str)) || nan(str);\n}\n\n/**\n * @public\n */\nexport class ElasticSearchSearchEngine implements SearchEngine {\n  private readonly elasticSearchClient: Client;\n\n  constructor(\n    private readonly elasticSearchClientOptions: ElasticSearchClientOptions,\n    private readonly aliasPostfix: string,\n    private readonly indexPrefix: string,\n    private readonly logger: Logger,\n  ) {\n    this.elasticSearchClient = this.newClient(options => new Client(options));\n  }\n\n  static async fromConfig({\n    logger,\n    config,\n    aliasPostfix = `search`,\n    indexPrefix = ``,\n  }: ElasticSearchOptions) {\n    const options = await createElasticSearchClientOptions(\n      config.getConfig('search.elasticsearch'),\n    );\n    if (options.provider === 'elastic') {\n      logger.info('Initializing Elastic.co ElasticSearch search engine.');\n    } else if (options.provider === 'aws') {\n      logger.info('Initializing AWS ElasticSearch search engine.');\n    } else {\n      logger.info('Initializing ElasticSearch search engine.');\n    }\n\n    return new ElasticSearchSearchEngine(\n      options,\n      aliasPostfix,\n      indexPrefix,\n      logger,\n    );\n  }\n\n  /**\n   * Create a custom search client from the derived elastic search\n   * configuration. This need not be the same client that the engine uses\n   * internally.\n   */\n  newClient<T>(create: (options: ElasticSearchClientOptions) => T): T {\n    return create(this.elasticSearchClientOptions);\n  }\n\n  protected translator(query: SearchQuery): ConcreteElasticSearchQuery {\n    const { term, filters = {}, types, pageCursor } = query;\n\n    const filter = Object.entries(filters)\n      .filter(([_, value]) => Boolean(value))\n      .map(([key, value]: [key: string, value: any]) => {\n        if (['string', 'number', 'boolean'].includes(typeof value)) {\n          return esb.matchQuery(key, value.toString());\n        }\n        if (Array.isArray(value)) {\n          return esb\n            .boolQuery()\n            .should(value.map(it => esb.matchQuery(key, it.toString())));\n        }\n        this.logger.error(\n          'Failed to query, unrecognized filter type',\n          key,\n          value,\n        );\n        throw new Error(\n          'Failed to add filters to query. Unrecognized filter type',\n        );\n      });\n    const esbQuery = isBlank(term)\n      ? esb.matchAllQuery()\n      : esb\n          .multiMatchQuery(['*'], term)\n          .fuzziness('auto')\n          .minimumShouldMatch(1);\n    const pageSize = 25;\n    const { page } = decodePageCursor(pageCursor);\n\n    return {\n      elasticSearchQuery: esb\n        .requestBodySearch()\n        .query(esb.boolQuery().filter(filter).must([esbQuery]))\n        .from(page * pageSize)\n        .size(pageSize)\n        .toJSON(),\n      documentTypes: types,\n      pageSize,\n    };\n  }\n\n  setTranslator(translator: ElasticSearchQueryTranslator) {\n    this.translator = translator;\n  }\n\n  async index(type: string, documents: IndexableDocument[]): Promise<void> {\n    this.logger.info(\n      `Started indexing ${documents.length} documents for index ${type}`,\n    );\n    const startTimestamp = process.hrtime();\n    const alias = this.constructSearchAlias(type);\n    const index = this.constructIndexName(type, `${Date.now()}`);\n    try {\n      const aliases = await this.elasticSearchClient.cat.aliases({\n        format: 'json',\n        name: alias,\n      });\n      const removableIndices = aliases.body.map(\n        (r: Record<string, any>) => r.index,\n      );\n\n      await this.elasticSearchClient.indices.create({\n        index,\n      });\n      const result = await this.elasticSearchClient.helpers.bulk({\n        datasource: documents,\n        onDocument() {\n          return {\n            index: { _index: index },\n          };\n        },\n        refreshOnCompletion: index,\n      });\n\n      this.logger.info(\n        `Indexing completed for index ${type} in ${duration(startTimestamp)}`,\n        result,\n      );\n      await this.elasticSearchClient.indices.updateAliases({\n        body: {\n          actions: [\n            { remove: { index: this.constructIndexName(type, '*'), alias } },\n            { add: { index, alias } },\n          ],\n        },\n      });\n\n      this.logger.info('Removing stale search indices', removableIndices);\n      if (removableIndices.length) {\n        await this.elasticSearchClient.indices.delete({\n          index: removableIndices,\n        });\n      }\n    } catch (e) {\n      this.logger.error(`Failed to index documents for type ${type}`, e);\n      const response = await this.elasticSearchClient.indices.exists({\n        index,\n      });\n      const indexCreated = response.body;\n      if (indexCreated) {\n        this.logger.info(`Removing created index ${index}`);\n        await this.elasticSearchClient.indices.delete({\n          index,\n        });\n      }\n    }\n  }\n\n  async query(query: SearchQuery): Promise<SearchResultSet> {\n    const { elasticSearchQuery, documentTypes, pageSize } =\n      this.translator(query);\n    const queryIndices = documentTypes\n      ? documentTypes.map(it => this.constructSearchAlias(it))\n      : this.constructSearchAlias('*');\n    try {\n      const result = await this.elasticSearchClient.search({\n        index: queryIndices,\n        body: elasticSearchQuery,\n      });\n      const { page } = decodePageCursor(query.pageCursor);\n      const hasNextPage = result.body.hits.total.value > page * pageSize;\n      const hasPreviousPage = page > 0;\n      const nextPageCursor = hasNextPage\n        ? encodePageCursor({ page: page + 1 })\n        : undefined;\n      const previousPageCursor = hasPreviousPage\n        ? encodePageCursor({ page: page - 1 })\n        : undefined;\n\n      return {\n        results: result.body.hits.hits.map((d: ElasticSearchResult) => ({\n          type: this.getTypeFromIndex(d._index),\n          document: d._source,\n        })),\n        nextPageCursor,\n        previousPageCursor,\n      };\n    } catch (e) {\n      this.logger.error(\n        `Failed to query documents for indices ${queryIndices}`,\n        e,\n      );\n      return Promise.reject({ results: [] });\n    }\n  }\n\n  private readonly indexSeparator = '-index__';\n\n  private constructIndexName(type: string, postFix: string) {\n    return `${this.indexPrefix}${type}${this.indexSeparator}${postFix}`;\n  }\n\n  private getTypeFromIndex(index: string) {\n    return index\n      .substring(this.indexPrefix.length)\n      .split(this.indexSeparator)[0];\n  }\n\n  private constructSearchAlias(type: string) {\n    const postFix = this.aliasPostfix ? `__${this.aliasPostfix}` : '';\n    return `${this.indexPrefix}${type}${postFix}`;\n  }\n}\n\nexport function decodePageCursor(pageCursor?: string): { page: number } {\n  if (!pageCursor) {\n    return { page: 0 };\n  }\n\n  return {\n    page: Number(Buffer.from(pageCursor, 'base64').toString('utf-8')),\n  };\n}\n\nexport function encodePageCursor({ page }: { page: number }): string {\n  return Buffer.from(`${page}`, 'utf-8').toString('base64');\n}\n\nasync function createElasticSearchClientOptions(\n  config?: Config,\n): Promise<ElasticSearchClientOptions> {\n  if (!config) {\n    throw new Error('No elastic search config found');\n  }\n  const clientOptionsConfig = config.getOptionalConfig('clientOptions');\n  const sslConfig = clientOptionsConfig?.getOptionalConfig('ssl');\n\n  if (config.getOptionalString('provider') === 'elastic') {\n    const authConfig = config.getConfig('auth');\n    return {\n      provider: 'elastic',\n      cloud: {\n        id: config.getString('cloudId'),\n      },\n      auth: {\n        username: authConfig.getString('username'),\n        password: authConfig.getString('password'),\n      },\n      ...(sslConfig\n        ? {\n            ssl: {\n              rejectUnauthorized:\n                sslConfig?.getOptionalBoolean('rejectUnauthorized'),\n            },\n          }\n        : {}),\n    };\n  }\n  if (config.getOptionalString('provider') === 'aws') {\n    const awsCredentials = await awsGetCredentials();\n    const AWSConnection = createAWSConnection(awsCredentials);\n    return {\n      provider: 'aws',\n      node: config.getString('node'),\n      ...AWSConnection,\n      ...(sslConfig\n        ? {\n            ssl: {\n              rejectUnauthorized:\n                sslConfig?.getOptionalBoolean('rejectUnauthorized'),\n            },\n          }\n        : {}),\n    };\n  }\n  const authConfig = config.getOptionalConfig('auth');\n  const auth =\n    authConfig &&\n    (authConfig.has('apiKey')\n      ? {\n          apiKey: authConfig.getString('apiKey'),\n        }\n      : {\n          username: authConfig.getString('username'),\n          password: authConfig.getString('password'),\n        });\n  return {\n    node: config.getString('node'),\n    auth,\n    ...(sslConfig\n      ? {\n          ssl: {\n            rejectUnauthorized:\n              sslConfig?.getOptionalBoolean('rejectUnauthorized'),\n          },\n        }\n      : {}),\n  };\n}\n"],"names":["isEmpty","isNumber","nan","Client","esb","awsGetCredentials","createAWSConnection"],"mappings":";;;;;;;;;;;;;AA4DA,kBAAkB,gBAA0C;AAC1D,QAAM,QAAQ,QAAQ,OAAO;AAC7B,QAAM,UAAU,MAAM,KAAK,MAAM,KAAK;AACtC,SAAO,GAAG,QAAQ,QAAQ;AAAA;AAG5B,iBAAiB,KAAa;AAC5B,SAAQA,eAAQ,QAAQ,CAACC,gBAAS,QAASC,aAAI;AAAA;gCAMc;AAAA,EAG7D,YACmB,4BACA,cACA,aACA,QACjB;AAJiB;AACA;AACA;AACA;AA4LF,0BAAiB;AA1LhC,SAAK,sBAAsB,KAAK,UAAU,aAAW,IAAIC,qBAAO;AAAA;AAAA,eAGrD,WAAW;AAAA,IACtB;AAAA,IACA;AAAA,IACA,eAAe;AAAA,IACf,cAAc;AAAA,KACS;AACvB,UAAM,UAAU,MAAM,iCACpB,OAAO,UAAU;AAEnB,QAAI,QAAQ,aAAa,WAAW;AAClC,aAAO,KAAK;AAAA,eACH,QAAQ,aAAa,OAAO;AACrC,aAAO,KAAK;AAAA,WACP;AACL,aAAO,KAAK;AAAA;AAGd,WAAO,IAAI,0BACT,SACA,cACA,aACA;AAAA;AAAA,EASJ,UAAa,QAAuD;AAClE,WAAO,OAAO,KAAK;AAAA;AAAA,EAGX,WAAW,OAAgD;AACnE,UAAM,EAAE,MAAM,UAAU,IAAI,OAAO,eAAe;AAElD,UAAM,SAAS,OAAO,QAAQ,SAC3B,OAAO,CAAC,CAAC,GAAG,WAAW,QAAQ,QAC/B,IAAI,CAAC,CAAC,KAAK,WAAsC;AAChD,UAAI,CAAC,UAAU,UAAU,WAAW,SAAS,OAAO,QAAQ;AAC1D,eAAOC,wBAAI,WAAW,KAAK,MAAM;AAAA;AAEnC,UAAI,MAAM,QAAQ,QAAQ;AACxB,eAAOA,wBACJ,YACA,OAAO,MAAM,IAAI,QAAMA,wBAAI,WAAW,KAAK,GAAG;AAAA;AAEnD,WAAK,OAAO,MACV,6CACA,KACA;AAEF,YAAM,IAAI,MACR;AAAA;AAGN,UAAM,WAAW,QAAQ,QACrBA,wBAAI,kBACJA,wBACG,gBAAgB,CAAC,MAAM,MACvB,UAAU,QACV,mBAAmB;AAC1B,UAAM,WAAW;AACjB,UAAM,EAAE,SAAS,iBAAiB;AAElC,WAAO;AAAA,MACL,oBAAoBA,wBACjB,oBACA,MAAMA,wBAAI,YAAY,OAAO,QAAQ,KAAK,CAAC,YAC3C,KAAK,OAAO,UACZ,KAAK,UACL;AAAA,MACH,eAAe;AAAA,MACf;AAAA;AAAA;AAAA,EAIJ,cAAc,YAA0C;AACtD,SAAK,aAAa;AAAA;AAAA,QAGd,MAAM,MAAc,WAA+C;AACvE,SAAK,OAAO,KACV,oBAAoB,UAAU,8BAA8B;AAE9D,UAAM,iBAAiB,QAAQ;AAC/B,UAAM,QAAQ,KAAK,qBAAqB;AACxC,UAAM,QAAQ,KAAK,mBAAmB,MAAM,GAAG,KAAK;AACpD,QAAI;AACF,YAAM,UAAU,MAAM,KAAK,oBAAoB,IAAI,QAAQ;AAAA,QACzD,QAAQ;AAAA,QACR,MAAM;AAAA;AAER,YAAM,mBAAmB,QAAQ,KAAK,IACpC,CAAC,MAA2B,EAAE;AAGhC,YAAM,KAAK,oBAAoB,QAAQ,OAAO;AAAA,QAC5C;AAAA;AAEF,YAAM,SAAS,MAAM,KAAK,oBAAoB,QAAQ,KAAK;AAAA,QACzD,YAAY;AAAA,QACZ,aAAa;AACX,iBAAO;AAAA,YACL,OAAO,EAAE,QAAQ;AAAA;AAAA;AAAA,QAGrB,qBAAqB;AAAA;AAGvB,WAAK,OAAO,KACV,gCAAgC,WAAW,SAAS,mBACpD;AAEF,YAAM,KAAK,oBAAoB,QAAQ,cAAc;AAAA,QACnD,MAAM;AAAA,UACJ,SAAS;AAAA,YACP,EAAE,QAAQ,EAAE,OAAO,KAAK,mBAAmB,MAAM,MAAM;AAAA,YACvD,EAAE,KAAK,EAAE,OAAO;AAAA;AAAA;AAAA;AAKtB,WAAK,OAAO,KAAK,iCAAiC;AAClD,UAAI,iBAAiB,QAAQ;AAC3B,cAAM,KAAK,oBAAoB,QAAQ,OAAO;AAAA,UAC5C,OAAO;AAAA;AAAA;AAAA,aAGJ,GAAP;AACA,WAAK,OAAO,MAAM,sCAAsC,QAAQ;AAChE,YAAM,WAAW,MAAM,KAAK,oBAAoB,QAAQ,OAAO;AAAA,QAC7D;AAAA;AAEF,YAAM,eAAe,SAAS;AAC9B,UAAI,cAAc;AAChB,aAAK,OAAO,KAAK,0BAA0B;AAC3C,cAAM,KAAK,oBAAoB,QAAQ,OAAO;AAAA,UAC5C;AAAA;AAAA;AAAA;AAAA;AAAA,QAMF,MAAM,OAA8C;AACxD,UAAM,EAAE,oBAAoB,eAAe,aACzC,KAAK,WAAW;AAClB,UAAM,eAAe,gBACjB,cAAc,IAAI,QAAM,KAAK,qBAAqB,OAClD,KAAK,qBAAqB;AAC9B,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,oBAAoB,OAAO;AAAA,QACnD,OAAO;AAAA,QACP,MAAM;AAAA;AAER,YAAM,EAAE,SAAS,iBAAiB,MAAM;AACxC,YAAM,cAAc,OAAO,KAAK,KAAK,MAAM,QAAQ,OAAO;AAC1D,YAAM,kBAAkB,OAAO;AAC/B,YAAM,iBAAiB,cACnB,iBAAiB,EAAE,MAAM,OAAO,OAChC;AACJ,YAAM,qBAAqB,kBACvB,iBAAiB,EAAE,MAAM,OAAO,OAChC;AAEJ,aAAO;AAAA,QACL,SAAS,OAAO,KAAK,KAAK,KAAK,IAAI,CAAC;AAA4B,UAC9D,MAAM,KAAK,iBAAiB,EAAE;AAAA,UAC9B,UAAU,EAAE;AAAA;AAAA,QAEd;AAAA,QACA;AAAA;AAAA,aAEK,GAAP;AACA,WAAK,OAAO,MACV,yCAAyC,gBACzC;AAEF,aAAO,QAAQ,OAAO,EAAE,SAAS;AAAA;AAAA;AAAA,EAM7B,mBAAmB,MAAc,SAAiB;AACxD,WAAO,GAAG,KAAK,cAAc,OAAO,KAAK,iBAAiB;AAAA;AAAA,EAGpD,iBAAiB,OAAe;AACtC,WAAO,MACJ,UAAU,KAAK,YAAY,QAC3B,MAAM,KAAK,gBAAgB;AAAA;AAAA,EAGxB,qBAAqB,MAAc;AACzC,UAAM,UAAU,KAAK,eAAe,KAAK,KAAK,iBAAiB;AAC/D,WAAO,GAAG,KAAK,cAAc,OAAO;AAAA;AAAA;0BAIP,YAAuC;AACtE,MAAI,CAAC,YAAY;AACf,WAAO,EAAE,MAAM;AAAA;AAGjB,SAAO;AAAA,IACL,MAAM,OAAO,OAAO,KAAK,YAAY,UAAU,SAAS;AAAA;AAAA;0BAI3B,EAAE,QAAkC;AACnE,SAAO,OAAO,KAAK,GAAG,QAAQ,SAAS,SAAS;AAAA;AAGlD,gDACE,QACqC;AACrC,MAAI,CAAC,QAAQ;AACX,UAAM,IAAI,MAAM;AAAA;AAElB,QAAM,sBAAsB,OAAO,kBAAkB;AACrD,QAAM,YAAY,2DAAqB,kBAAkB;AAEzD,MAAI,OAAO,kBAAkB,gBAAgB,WAAW;AACtD,UAAM,cAAa,OAAO,UAAU;AACpC,WAAO;AAAA,MACL,UAAU;AAAA,MACV,OAAO;AAAA,QACL,IAAI,OAAO,UAAU;AAAA;AAAA,MAEvB,MAAM;AAAA,QACJ,UAAU,YAAW,UAAU;AAAA,QAC/B,UAAU,YAAW,UAAU;AAAA;AAAA,SAE7B,YACA;AAAA,QACE,KAAK;AAAA,UACH,oBACE,uCAAW,mBAAmB;AAAA;AAAA,UAGpC;AAAA;AAAA;AAGR,MAAI,OAAO,kBAAkB,gBAAgB,OAAO;AAClD,UAAM,iBAAiB,MAAMC;AAC7B,UAAM,gBAAgBC,oCAAoB;AAC1C,WAAO;AAAA,MACL,UAAU;AAAA,MACV,MAAM,OAAO,UAAU;AAAA,SACpB;AAAA,SACC,YACA;AAAA,QACE,KAAK;AAAA,UACH,oBACE,uCAAW,mBAAmB;AAAA;AAAA,UAGpC;AAAA;AAAA;AAGR,QAAM,aAAa,OAAO,kBAAkB;AAC5C,QAAM,OACJ,0BACY,IAAI,YACZ;AAAA,IACE,QAAQ,WAAW,UAAU;AAAA,MAE/B;AAAA,IACE,UAAU,WAAW,UAAU;AAAA,IAC/B,UAAU,WAAW,UAAU;AAAA;AAEvC,SAAO;AAAA,IACL,MAAM,OAAO,UAAU;AAAA,IACvB;AAAA,OACI,YACA;AAAA,MACE,KAAK;AAAA,QACH,oBACE,uCAAW,mBAAmB;AAAA;AAAA,QAGpC;AAAA;AAAA;;;;"}
+{"version":3,"file":"index.cjs.js","sources":["../src/engines/ElasticSearchSearchEngineIndexer.ts","../src/engines/ElasticSearchSearchEngine.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node';\nimport { IndexableDocument } from '@backstage/search-common';\nimport { Client } from '@elastic/elasticsearch';\nimport { Readable } from 'stream';\nimport { Logger } from 'winston';\n\nexport type ElasticSearchSearchEngineIndexerOptions = {\n  type: string;\n  indexPrefix: string;\n  indexSeparator: string;\n  alias: string;\n  logger: Logger;\n  elasticSearchClient: Client;\n};\n\nfunction duration(startTimestamp: [number, number]): string {\n  const delta = process.hrtime(startTimestamp);\n  const seconds = delta[0] + delta[1] / 1e9;\n  return `${seconds.toFixed(1)}s`;\n}\n\nexport class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer {\n  private received: number = 0;\n  private processed: number = 0;\n  private removableIndices: string[] = [];\n\n  private readonly startTimestamp: [number, number];\n  private readonly type: string;\n  public readonly indexName: string;\n  private readonly indexPrefix: string;\n  private readonly indexSeparator: string;\n  private readonly alias: string;\n  private readonly logger: Logger;\n  private readonly sourceStream: Readable;\n  private readonly elasticSearchClient: Client;\n  private bulkResult: Promise<any>;\n\n  constructor(options: ElasticSearchSearchEngineIndexerOptions) {\n    super({ batchSize: 100 });\n    this.logger = options.logger;\n    this.startTimestamp = process.hrtime();\n    this.type = options.type;\n    this.indexPrefix = options.indexPrefix;\n    this.indexSeparator = options.indexSeparator;\n    this.indexName = this.constructIndexName(`${Date.now()}`);\n    this.alias = options.alias;\n    this.elasticSearchClient = options.elasticSearchClient;\n\n    // The ES client bulk helper supports stream-based indexing, but we have to\n    // supply the stream directly to it at instantiation-time. We can't supply\n    // this class itself, so instead, we create this inline stream instead.\n    this.sourceStream = new Readable({ objectMode: true });\n    this.sourceStream._read = () => {};\n\n    // eslint-disable-next-line consistent-this\n    const that = this;\n\n    // Keep a reference to the ES Bulk helper so that we can know when all\n    // documents have been successfully written to ES.\n    this.bulkResult = this.elasticSearchClient.helpers.bulk({\n      datasource: this.sourceStream,\n      onDocument() {\n        that.processed++;\n        return {\n          index: { _index: that.indexName },\n        };\n      },\n      refreshOnCompletion: that.indexName,\n    });\n  }\n\n  async initialize(): Promise<void> {\n    this.logger.info(`Started indexing documents for index ${this.type}`);\n\n    const aliases = await this.elasticSearchClient.cat.aliases({\n      format: 'json',\n      name: this.alias,\n    });\n\n    this.removableIndices = aliases.body.map(\n      (r: Record<string, any>) => r.index,\n    );\n\n    await this.elasticSearchClient.indices.create({\n      index: this.indexName,\n    });\n  }\n\n  async index(documents: IndexableDocument[]): Promise<void> {\n    await this.isReady();\n    documents.forEach(document => {\n      this.received++;\n      this.sourceStream.push(document);\n    });\n  }\n\n  async finalize(): Promise<void> {\n    // Wait for all documents to be processed.\n    await this.isReady();\n\n    // Close off the underlying stream connected to ES, indicating that no more\n    // documents will be written.\n    this.sourceStream.push(null);\n\n    // Wait for the bulk helper to finish processing.\n    const result = await this.bulkResult;\n\n    // Rotate aliases upon completion. Allow errors to bubble up so that we can\n    // clean up the create index.\n    this.logger.info(\n      `Indexing completed for index ${this.type} in ${duration(\n        this.startTimestamp,\n      )}`,\n      result,\n    );\n    await this.elasticSearchClient.indices.updateAliases({\n      body: {\n        actions: [\n          {\n            remove: { index: this.constructIndexName('*'), alias: this.alias },\n          },\n          { add: { index: this.indexName, alias: this.alias } },\n        ],\n      },\n    });\n\n    // If any indices are removable, remove them. Do not bubble up this error,\n    // as doing so would delete the now aliased index. Log instead.\n    if (this.removableIndices.length) {\n      this.logger.info('Removing stale search indices', this.removableIndices);\n      try {\n        await this.elasticSearchClient.indices.delete({\n          index: this.removableIndices,\n        });\n      } catch (e) {\n        this.logger.warn(`Failed to remove stale search indices: ${e}`);\n      }\n    }\n  }\n\n  /**\n   * Ensures that the number of documents sent over the wire to ES matches the\n   * number of documents this stream has received so far. This helps manage\n   * backpressure in other parts of the indexing pipeline.\n   */\n  private isReady(): Promise<void> {\n    return new Promise(resolve => {\n      const interval = setInterval(() => {\n        if (this.received === this.processed) {\n          clearInterval(interval);\n          resolve();\n        }\n      }, 50);\n    });\n  }\n\n  private constructIndexName(postFix: string) {\n    return `${this.indexPrefix}${this.type}${this.indexSeparator}${postFix}`;\n  }\n}\n","/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  awsGetCredentials,\n  createAWSConnection,\n} from '@acuris/aws-es-connection';\nimport { Config } from '@backstage/config';\nimport {\n  IndexableDocument,\n  SearchEngine,\n  SearchQuery,\n  SearchResultSet,\n} from '@backstage/search-common';\nimport { Client } from '@elastic/elasticsearch';\nimport esb from 'elastic-builder';\nimport { isEmpty, isNaN as nan, isNumber } from 'lodash';\nimport { Logger } from 'winston';\nimport type { ElasticSearchClientOptions } from './ElasticSearchClientOptions';\nimport { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer';\n\nexport type { ElasticSearchClientOptions };\n\nexport type ConcreteElasticSearchQuery = {\n  documentTypes?: string[];\n  elasticSearchQuery: Object;\n  pageSize: number;\n};\n\ntype ElasticSearchQueryTranslator = (\n  query: SearchQuery,\n) => ConcreteElasticSearchQuery;\n\ntype ElasticSearchOptions = {\n  logger: Logger;\n  config: Config;\n  aliasPostfix?: string;\n  indexPrefix?: string;\n};\n\ntype ElasticSearchResult = {\n  _index: string;\n  _type: string;\n  _score: number;\n  _source: IndexableDocument;\n};\n\nfunction isBlank(str: string) {\n  return (isEmpty(str) && !isNumber(str)) || nan(str);\n}\n\n/**\n * @public\n */\nexport class ElasticSearchSearchEngine implements SearchEngine {\n  private readonly elasticSearchClient: Client;\n\n  constructor(\n    private readonly elasticSearchClientOptions: ElasticSearchClientOptions,\n    private readonly aliasPostfix: string,\n    private readonly indexPrefix: string,\n    private readonly logger: Logger,\n  ) {\n    this.elasticSearchClient = this.newClient(options => new Client(options));\n  }\n\n  static async fromConfig({\n    logger,\n    config,\n    aliasPostfix = `search`,\n    indexPrefix = ``,\n  }: ElasticSearchOptions) {\n    const options = await createElasticSearchClientOptions(\n      config.getConfig('search.elasticsearch'),\n    );\n    if (options.provider === 'elastic') {\n      logger.info('Initializing Elastic.co ElasticSearch search engine.');\n    } else if (options.provider === 'aws') {\n      logger.info('Initializing AWS ElasticSearch search engine.');\n    } else {\n      logger.info('Initializing ElasticSearch search engine.');\n    }\n\n    return new ElasticSearchSearchEngine(\n      options,\n      aliasPostfix,\n      indexPrefix,\n      logger,\n    );\n  }\n\n  /**\n   * Create a custom search client from the derived elastic search\n   * configuration. This need not be the same client that the engine uses\n   * internally.\n   */\n  newClient<T>(create: (options: ElasticSearchClientOptions) => T): T {\n    return create(this.elasticSearchClientOptions);\n  }\n\n  protected translator(query: SearchQuery): ConcreteElasticSearchQuery {\n    const { term, filters = {}, types, pageCursor } = query;\n\n    const filter = Object.entries(filters)\n      .filter(([_, value]) => Boolean(value))\n      .map(([key, value]: [key: string, value: any]) => {\n        if (['string', 'number', 'boolean'].includes(typeof value)) {\n          return esb.matchQuery(key, value.toString());\n        }\n        if (Array.isArray(value)) {\n          return esb\n            .boolQuery()\n            .should(value.map(it => esb.matchQuery(key, it.toString())));\n        }\n        this.logger.error(\n          'Failed to query, unrecognized filter type',\n          key,\n          value,\n        );\n        throw new Error(\n          'Failed to add filters to query. Unrecognized filter type',\n        );\n      });\n    const esbQuery = isBlank(term)\n      ? esb.matchAllQuery()\n      : esb\n          .multiMatchQuery(['*'], term)\n          .fuzziness('auto')\n          .minimumShouldMatch(1);\n    const pageSize = 25;\n    const { page } = decodePageCursor(pageCursor);\n\n    return {\n      elasticSearchQuery: esb\n        .requestBodySearch()\n        .query(esb.boolQuery().filter(filter).must([esbQuery]))\n        .from(page * pageSize)\n        .size(pageSize)\n        .toJSON(),\n      documentTypes: types,\n      pageSize,\n    };\n  }\n\n  setTranslator(translator: ElasticSearchQueryTranslator) {\n    this.translator = translator;\n  }\n\n  async getIndexer(type: string) {\n    const alias = this.constructSearchAlias(type);\n    const indexer = new ElasticSearchSearchEngineIndexer({\n      type,\n      indexPrefix: this.indexPrefix,\n      indexSeparator: this.indexSeparator,\n      alias,\n      elasticSearchClient: this.elasticSearchClient,\n      logger: this.logger,\n    });\n\n    // Attempt cleanup upon failure.\n    indexer.on('error', async e => {\n      this.logger.error(`Failed to index documents for type ${type}`, e);\n      try {\n        const response = await this.elasticSearchClient.indices.exists({\n          index: indexer.indexName,\n        });\n        const indexCreated = response.body;\n        if (indexCreated) {\n          this.logger.info(`Removing created index ${indexer.indexName}`);\n          await this.elasticSearchClient.indices.delete({\n            index: indexer.indexName,\n          });\n        }\n      } catch (error) {\n        this.logger.error(`Unable to clean up elastic index: ${error}`);\n      }\n    });\n\n    return indexer;\n  }\n\n  async query(query: SearchQuery): Promise<SearchResultSet> {\n    const { elasticSearchQuery, documentTypes, pageSize } =\n      this.translator(query);\n    const queryIndices = documentTypes\n      ? documentTypes.map(it => this.constructSearchAlias(it))\n      : this.constructSearchAlias('*');\n    try {\n      const result = await this.elasticSearchClient.search({\n        index: queryIndices,\n        body: elasticSearchQuery,\n      });\n      const { page } = decodePageCursor(query.pageCursor);\n      const hasNextPage = result.body.hits.total.value > page * pageSize;\n      const hasPreviousPage = page > 0;\n      const nextPageCursor = hasNextPage\n        ? encodePageCursor({ page: page + 1 })\n        : undefined;\n      const previousPageCursor = hasPreviousPage\n        ? encodePageCursor({ page: page - 1 })\n        : undefined;\n\n      return {\n        results: result.body.hits.hits.map((d: ElasticSearchResult) => ({\n          type: this.getTypeFromIndex(d._index),\n          document: d._source,\n        })),\n        nextPageCursor,\n        previousPageCursor,\n      };\n    } catch (e) {\n      this.logger.error(\n        `Failed to query documents for indices ${queryIndices}`,\n        e,\n      );\n      return Promise.reject({ results: [] });\n    }\n  }\n\n  private readonly indexSeparator = '-index__';\n\n  private getTypeFromIndex(index: string) {\n    return index\n      .substring(this.indexPrefix.length)\n      .split(this.indexSeparator)[0];\n  }\n\n  private constructSearchAlias(type: string) {\n    const postFix = this.aliasPostfix ? `__${this.aliasPostfix}` : '';\n    return `${this.indexPrefix}${type}${postFix}`;\n  }\n}\n\nexport function decodePageCursor(pageCursor?: string): { page: number } {\n  if (!pageCursor) {\n    return { page: 0 };\n  }\n\n  return {\n    page: Number(Buffer.from(pageCursor, 'base64').toString('utf-8')),\n  };\n}\n\nexport function encodePageCursor({ page }: { page: number }): string {\n  return Buffer.from(`${page}`, 'utf-8').toString('base64');\n}\n\nasync function createElasticSearchClientOptions(\n  config?: Config,\n): Promise<ElasticSearchClientOptions> {\n  if (!config) {\n    throw new Error('No elastic search config found');\n  }\n  const clientOptionsConfig = config.getOptionalConfig('clientOptions');\n  const sslConfig = clientOptionsConfig?.getOptionalConfig('ssl');\n\n  if (config.getOptionalString('provider') === 'elastic') {\n    const authConfig = config.getConfig('auth');\n    return {\n      provider: 'elastic',\n      cloud: {\n        id: config.getString('cloudId'),\n      },\n      auth: {\n        username: authConfig.getString('username'),\n        password: authConfig.getString('password'),\n      },\n      ...(sslConfig\n        ? {\n            ssl: {\n              rejectUnauthorized:\n                sslConfig?.getOptionalBoolean('rejectUnauthorized'),\n            },\n          }\n        : {}),\n    };\n  }\n  if (config.getOptionalString('provider') === 'aws') {\n    const awsCredentials = await awsGetCredentials();\n    const AWSConnection = createAWSConnection(awsCredentials);\n    return {\n      provider: 'aws',\n      node: config.getString('node'),\n      ...AWSConnection,\n      ...(sslConfig\n        ? {\n            ssl: {\n              rejectUnauthorized:\n                sslConfig?.getOptionalBoolean('rejectUnauthorized'),\n            },\n          }\n        : {}),\n    };\n  }\n  const authConfig = config.getOptionalConfig('auth');\n  const auth =\n    authConfig &&\n    (authConfig.has('apiKey')\n      ? {\n          apiKey: authConfig.getString('apiKey'),\n        }\n      : {\n          username: authConfig.getString('username'),\n          password: authConfig.getString('password'),\n        });\n  return {\n    node: config.getString('node'),\n    auth,\n    ...(sslConfig\n      ? {\n          ssl: {\n            rejectUnauthorized:\n              sslConfig?.getOptionalBoolean('rejectUnauthorized'),\n          },\n        }\n      : {}),\n  };\n}\n"],"names":["BatchSearchEngineIndexer","Readable","isEmpty","isNumber","nan","Client","esb","awsGetCredentials","createAWSConnection"],"mappings":";;;;;;;;;;;;;;;AA+BA,kBAAkB,gBAA0C;AAC1D,QAAM,QAAQ,QAAQ,OAAO;AAC7B,QAAM,UAAU,MAAM,KAAK,MAAM,KAAK;AACtC,SAAO,GAAG,QAAQ,QAAQ;AAAA;+CAG0BA,iDAAyB;AAAA,EAgB7E,YAAY,SAAkD;AAC5D,UAAM,EAAE,WAAW;AAhBb,oBAAmB;AACnB,qBAAoB;AACpB,4BAA6B;AAenC,SAAK,SAAS,QAAQ;AACtB,SAAK,iBAAiB,QAAQ;AAC9B,SAAK,OAAO,QAAQ;AACpB,SAAK,cAAc,QAAQ;AAC3B,SAAK,iBAAiB,QAAQ;AAC9B,SAAK,YAAY,KAAK,mBAAmB,GAAG,KAAK;AACjD,SAAK,QAAQ,QAAQ;AACrB,SAAK,sBAAsB,QAAQ;AAKnC,SAAK,eAAe,IAAIC,gBAAS,EAAE,YAAY;AAC/C,SAAK,aAAa,QAAQ,MAAM;AAAA;AAGhC,UAAM,OAAO;AAIb,SAAK,aAAa,KAAK,oBAAoB,QAAQ,KAAK;AAAA,MACtD,YAAY,KAAK;AAAA,MACjB,aAAa;AACX,aAAK;AACL,eAAO;AAAA,UACL,OAAO,EAAE,QAAQ,KAAK;AAAA;AAAA;AAAA,MAG1B,qBAAqB,KAAK;AAAA;AAAA;AAAA,QAIxB,aAA4B;AAChC,SAAK,OAAO,KAAK,wCAAwC,KAAK;AAE9D,UAAM,UAAU,MAAM,KAAK,oBAAoB,IAAI,QAAQ;AAAA,MACzD,QAAQ;AAAA,MACR,MAAM,KAAK;AAAA;AAGb,SAAK,mBAAmB,QAAQ,KAAK,IACnC,CAAC,MAA2B,EAAE;AAGhC,UAAM,KAAK,oBAAoB,QAAQ,OAAO;AAAA,MAC5C,OAAO,KAAK;AAAA;AAAA;AAAA,QAIV,MAAM,WAA+C;AACzD,UAAM,KAAK;AACX,cAAU,QAAQ,cAAY;AAC5B,WAAK;AACL,WAAK,aAAa,KAAK;AAAA;AAAA;AAAA,QAIrB,WAA0B;AAE9B,UAAM,KAAK;AAIX,SAAK,aAAa,KAAK;AAGvB,UAAM,SAAS,MAAM,KAAK;AAI1B,SAAK,OAAO,KACV,gCAAgC,KAAK,WAAW,SAC9C,KAAK,mBAEP;AAEF,UAAM,KAAK,oBAAoB,QAAQ,cAAc;AAAA,MACnD,MAAM;AAAA,QACJ,SAAS;AAAA,UACP;AAAA,YACE,QAAQ,EAAE,OAAO,KAAK,mBAAmB,MAAM,OAAO,KAAK;AAAA;AAAA,UAE7D,EAAE,KAAK,EAAE,OAAO,KAAK,WAAW,OAAO,KAAK;AAAA;AAAA;AAAA;AAOlD,QAAI,KAAK,iBAAiB,QAAQ;AAChC,WAAK,OAAO,KAAK,iCAAiC,KAAK;AACvD,UAAI;AACF,cAAM,KAAK,oBAAoB,QAAQ,OAAO;AAAA,UAC5C,OAAO,KAAK;AAAA;AAAA,eAEP,GAAP;AACA,aAAK,OAAO,KAAK,0CAA0C;AAAA;AAAA;AAAA;AAAA,EAUzD,UAAyB;AAC/B,WAAO,IAAI,QAAQ,aAAW;AAC5B,YAAM,WAAW,YAAY,MAAM;AACjC,YAAI,KAAK,aAAa,KAAK,WAAW;AACpC,wBAAc;AACd;AAAA;AAAA,SAED;AAAA;AAAA;AAAA,EAIC,mBAAmB,SAAiB;AAC1C,WAAO,GAAG,KAAK,cAAc,KAAK,OAAO,KAAK,iBAAiB;AAAA;AAAA;;ACjHnE,iBAAiB,KAAa;AAC5B,SAAQC,eAAQ,QAAQ,CAACC,gBAAS,QAASC,aAAI;AAAA;gCAMc;AAAA,EAG7D,YACmB,4BACA,cACA,aACA,QACjB;AAJiB;AACA;AACA;AACA;AA8JF,0BAAiB;AA5JhC,SAAK,sBAAsB,KAAK,UAAU,aAAW,IAAIC,qBAAO;AAAA;AAAA,eAGrD,WAAW;AAAA,IACtB;AAAA,IACA;AAAA,IACA,eAAe;AAAA,IACf,cAAc;AAAA,KACS;AACvB,UAAM,UAAU,MAAM,iCACpB,OAAO,UAAU;AAEnB,QAAI,QAAQ,aAAa,WAAW;AAClC,aAAO,KAAK;AAAA,eACH,QAAQ,aAAa,OAAO;AACrC,aAAO,KAAK;AAAA,WACP;AACL,aAAO,KAAK;AAAA;AAGd,WAAO,IAAI,0BACT,SACA,cACA,aACA;AAAA;AAAA,EASJ,UAAa,QAAuD;AAClE,WAAO,OAAO,KAAK;AAAA;AAAA,EAGX,WAAW,OAAgD;AACnE,UAAM,EAAE,MAAM,UAAU,IAAI,OAAO,eAAe;AAElD,UAAM,SAAS,OAAO,QAAQ,SAC3B,OAAO,CAAC,CAAC,GAAG,WAAW,QAAQ,QAC/B,IAAI,CAAC,CAAC,KAAK,WAAsC;AAChD,UAAI,CAAC,UAAU,UAAU,WAAW,SAAS,OAAO,QAAQ;AAC1D,eAAOC,wBAAI,WAAW,KAAK,MAAM;AAAA;AAEnC,UAAI,MAAM,QAAQ,QAAQ;AACxB,eAAOA,wBACJ,YACA,OAAO,MAAM,IAAI,QAAMA,wBAAI,WAAW,KAAK,GAAG;AAAA;AAEnD,WAAK,OAAO,MACV,6CACA,KACA;AAEF,YAAM,IAAI,MACR;AAAA;AAGN,UAAM,WAAW,QAAQ,QACrBA,wBAAI,kBACJA,wBACG,gBAAgB,CAAC,MAAM,MACvB,UAAU,QACV,mBAAmB;AAC1B,UAAM,WAAW;AACjB,UAAM,EAAE,SAAS,iBAAiB;AAElC,WAAO;AAAA,MACL,oBAAoBA,wBACjB,oBACA,MAAMA,wBAAI,YAAY,OAAO,QAAQ,KAAK,CAAC,YAC3C,KAAK,OAAO,UACZ,KAAK,UACL;AAAA,MACH,eAAe;AAAA,MACf;AAAA;AAAA;AAAA,EAIJ,cAAc,YAA0C;AACtD,SAAK,aAAa;AAAA;AAAA,QAGd,WAAW,MAAc;AAC7B,UAAM,QAAQ,KAAK,qBAAqB;AACxC,UAAM,UAAU,IAAI,iCAAiC;AAAA,MACnD;AAAA,MACA,aAAa,KAAK;AAAA,MAClB,gBAAgB,KAAK;AAAA,MACrB;AAAA,MACA,qBAAqB,KAAK;AAAA,MAC1B,QAAQ,KAAK;AAAA;AAIf,YAAQ,GAAG,SAAS,OAAM,MAAK;AAC7B,WAAK,OAAO,MAAM,sCAAsC,QAAQ;AAChE,UAAI;AACF,cAAM,WAAW,MAAM,KAAK,oBAAoB,QAAQ,OAAO;AAAA,UAC7D,OAAO,QAAQ;AAAA;AAEjB,cAAM,eAAe,SAAS;AAC9B,YAAI,cAAc;AAChB,eAAK,OAAO,KAAK,0BAA0B,QAAQ;AACnD,gBAAM,KAAK,oBAAoB,QAAQ,OAAO;AAAA,YAC5C,OAAO,QAAQ;AAAA;AAAA;AAAA,eAGZ,OAAP;AACA,aAAK,OAAO,MAAM,qCAAqC;AAAA;AAAA;AAI3D,WAAO;AAAA;AAAA,QAGH,MAAM,OAA8C;AACxD,UAAM,EAAE,oBAAoB,eAAe,aACzC,KAAK,WAAW;AAClB,UAAM,eAAe,gBACjB,cAAc,IAAI,QAAM,KAAK,qBAAqB,OAClD,KAAK,qBAAqB;AAC9B,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,oBAAoB,OAAO;AAAA,QACnD,OAAO;AAAA,QACP,MAAM;AAAA;AAER,YAAM,EAAE,SAAS,iBAAiB,MAAM;AACxC,YAAM,cAAc,OAAO,KAAK,KAAK,MAAM,QAAQ,OAAO;AAC1D,YAAM,kBAAkB,OAAO;AAC/B,YAAM,iBAAiB,cACnB,iBAAiB,EAAE,MAAM,OAAO,OAChC;AACJ,YAAM,qBAAqB,kBACvB,iBAAiB,EAAE,MAAM,OAAO,OAChC;AAEJ,aAAO;AAAA,QACL,SAAS,OAAO,KAAK,KAAK,KAAK,IAAI,CAAC;AAA4B,UAC9D,MAAM,KAAK,iBAAiB,EAAE;AAAA,UAC9B,UAAU,EAAE;AAAA;AAAA,QAEd;AAAA,QACA;AAAA;AAAA,aAEK,GAAP;AACA,WAAK,OAAO,MACV,yCAAyC,gBACzC;AAEF,aAAO,QAAQ,OAAO,EAAE,SAAS;AAAA;AAAA;AAAA,EAM7B,iBAAiB,OAAe;AACtC,WAAO,MACJ,UAAU,KAAK,YAAY,QAC3B,MAAM,KAAK,gBAAgB;AAAA;AAAA,EAGxB,qBAAqB,MAAc;AACzC,UAAM,UAAU,KAAK,eAAe,KAAK,KAAK,iBAAiB;AAC/D,WAAO,GAAG,KAAK,cAAc,OAAO;AAAA;AAAA;0BAIP,YAAuC;AACtE,MAAI,CAAC,YAAY;AACf,WAAO,EAAE,MAAM;AAAA;AAGjB,SAAO;AAAA,IACL,MAAM,OAAO,OAAO,KAAK,YAAY,UAAU,SAAS;AAAA;AAAA;0BAI3B,EAAE,QAAkC;AACnE,SAAO,OAAO,KAAK,GAAG,QAAQ,SAAS,SAAS;AAAA;AAGlD,gDACE,QACqC;AACrC,MAAI,CAAC,QAAQ;AACX,UAAM,IAAI,MAAM;AAAA;AAElB,QAAM,sBAAsB,OAAO,kBAAkB;AACrD,QAAM,YAAY,2DAAqB,kBAAkB;AAEzD,MAAI,OAAO,kBAAkB,gBAAgB,WAAW;AACtD,UAAM,cAAa,OAAO,UAAU;AACpC,WAAO;AAAA,MACL,UAAU;AAAA,MACV,OAAO;AAAA,QACL,IAAI,OAAO,UAAU;AAAA;AAAA,MAEvB,MAAM;AAAA,QACJ,UAAU,YAAW,UAAU;AAAA,QAC/B,UAAU,YAAW,UAAU;AAAA;AAAA,SAE7B,YACA;AAAA,QACE,KAAK;AAAA,UACH,oBACE,uCAAW,mBAAmB;AAAA;AAAA,UAGpC;AAAA;AAAA;AAGR,MAAI,OAAO,kBAAkB,gBAAgB,OAAO;AAClD,UAAM,iBAAiB,MAAMC;AAC7B,UAAM,gBAAgBC,oCAAoB;AAC1C,WAAO;AAAA,MACL,UAAU;AAAA,MACV,MAAM,OAAO,UAAU;AAAA,SACpB;AAAA,SACC,YACA;AAAA,QACE,KAAK;AAAA,UACH,oBACE,uCAAW,mBAAmB;AAAA;AAAA,UAGpC;AAAA;AAAA;AAGR,QAAM,aAAa,OAAO,kBAAkB;AAC5C,QAAM,OACJ,0BACY,IAAI,YACZ;AAAA,IACE,QAAQ,WAAW,UAAU;AAAA,MAE/B;AAAA,IACE,UAAU,WAAW,UAAU;AAAA,IAC/B,UAAU,WAAW,UAAU;AAAA;AAEvC,SAAO;AAAA,IACL,MAAM,OAAO,UAAU;AAAA,IACvB;AAAA,OACI,YACA;AAAA,MACE,KAAK;AAAA,QACH,oBACE,uCAAW,mBAAmB;AAAA;AAAA,QAGpC;AAAA;AAAA;;;;"}

package/dist/index.d.ts

@@ -1,8 +1,10 @@
 /// <reference types="node" />
 import { Config } from '@backstage/config';
-import { SearchEngine, SearchQuery, IndexableDocument, SearchResultSet } from '@backstage/search-common';
+import { IndexableDocument, SearchEngine, SearchQuery, SearchResultSet } from '@backstage/search-common';
 import { Logger } from 'winston';
 import { ConnectionOptions } from 'tls';
+import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node';
+import { Client } from '@elastic/elasticsearch';
 
 /**
  * Options used to configure the `@elastic/elasticsearch` client and
@@ -96,6 +98,41 @@ interface ElasticSearchTransportConstructor {
     };
 }
 
+declare type ElasticSearchSearchEngineIndexerOptions = {
+    type: string;
+    indexPrefix: string;
+    indexSeparator: string;
+    alias: string;
+    logger: Logger;
+    elasticSearchClient: Client;
+};
+declare class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer {
+    private received;
+    private processed;
+    private removableIndices;
+    private readonly startTimestamp;
+    private readonly type;
+    readonly indexName: string;
+    private readonly indexPrefix;
+    private readonly indexSeparator;
+    private readonly alias;
+    private readonly logger;
+    private readonly sourceStream;
+    private readonly elasticSearchClient;
+    private bulkResult;
+    constructor(options: ElasticSearchSearchEngineIndexerOptions);
+    initialize(): Promise<void>;
+    index(documents: IndexableDocument[]): Promise<void>;
+    finalize(): Promise<void>;
+    /**
+     * Ensures that the number of documents sent over the wire to ES matches the
+     * number of documents this stream has received so far. This helps manage
+     * backpressure in other parts of the indexing pipeline.
+     */
+    private isReady;
+    private constructIndexName;
+}
+
 declare type ConcreteElasticSearchQuery = {
     documentTypes?: string[];
     elasticSearchQuery: Object;
@@ -127,12 +164,11 @@ declare class ElasticSearchSearchEngine implements SearchEngine {
     newClient<T>(create: (options: ElasticSearchClientOptions) => T): T;
     protected translator(query: SearchQuery): ConcreteElasticSearchQuery;
     setTranslator(translator: ElasticSearchQueryTranslator): void;
-    index(type: string, documents: IndexableDocument[]): Promise<void>;
+    getIndexer(type: string): Promise<ElasticSearchSearchEngineIndexer>;
     query(query: SearchQuery): Promise<SearchResultSet>;
     private readonly indexSeparator;
-    private constructIndexName;
     private getTypeFromIndex;
     private constructSearchAlias;
 }
 
-export { ElasticSearchClientOptions, ElasticSearchSearchEngine };
+export { ElasticSearchClientOptions, ElasticSearchSearchEngine, ElasticSearchSearchEngineIndexer, ElasticSearchSearchEngineIndexerOptions };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@backstage/plugin-search-backend-module-elasticsearch",
   "description": "A module for the search backend that implements search using ElasticSearch",
-  "version": "0.0.8",
+  "version": "0.1.0",
   "main": "dist/index.cjs.js",
   "types": "dist/index.d.ts",
   "license": "Apache-2.0",
@@ -10,19 +10,23 @@
     "main": "dist/index.cjs.js",
     "types": "dist/index.d.ts"
   },
+  "backstage": {
+    "role": "backend-plugin-module"
+  },
   "scripts": {
-    "start": "backstage-cli backend:dev",
-    "build": "backstage-cli backend:build",
-    "lint": "backstage-cli lint",
-    "test": "backstage-cli test",
-    "prepack": "backstage-cli prepack",
-    "postpack": "backstage-cli postpack",
-    "clean": "backstage-cli clean"
+    "start": "backstage-cli package start",
+    "build": "backstage-cli package build",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack",
+    "clean": "backstage-cli package clean"
   },
   "dependencies": {
     "@acuris/aws-es-connection": "^2.2.0",
-    "@backstage/config": "^0.1.13",
-    "@backstage/search-common": "^0.2.0",
+    "@backstage/config": "^0.1.15",
+    "@backstage/plugin-search-backend-node": "^0.5.0",
+    "@backstage/search-common": "^0.3.0",
     "@elastic/elasticsearch": "7.13.0",
     "aws-sdk": "^2.948.0",
     "elastic-builder": "^2.16.0",
@@ -30,9 +34,9 @@
     "winston": "^3.2.1"
   },
   "devDependencies": {
-    "@backstage/backend-common": "^0.10.4",
-    "@backstage/cli": "^0.12.0",
-    "@elastic/elasticsearch-mock": "^0.3.0"
+    "@backstage/backend-common": "^0.12.0",
+    "@backstage/cli": "^0.15.0",
+    "@elastic/elasticsearch-mock": "^1.0.0"
   },
   "files": [
     "dist",
@@ -42,5 +46,5 @@
     "testEnvironment": "node"
   },
   "configSchema": "config.d.ts",
-  "gitHead": "600d6e3c854bbfb12a0078ca6f726d1c0d1fea0b"
+  "gitHead": "04bb0dd824b78f6b57dac62c3015e681f094045c"
 }