@backstage/plugin-search-backend-module-elasticsearch 1.5.7-next.1 → 1.6.0

package/dist/index.cjs.js

@@ -1,667 +1,16 @@
 'use strict';
 
-var lodash = require('lodash');
-var aws = require('@opensearch-project/opensearch/aws');
-var aws4 = require('aws4');
-var elasticsearch = require('@elastic/elasticsearch');
-var opensearch = require('@opensearch-project/opensearch');
-var pluginSearchBackendNode = require('@backstage/plugin-search-backend-node');
-var stream = require('stream');
-var esb = require('elastic-builder');
-var uuid = require('uuid');
-var integrationAwsNode = require('@backstage/integration-aws-node');
+Object.defineProperty(exports, '__esModule', { value: true });
 
-function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e : { default: e }; }
+var module$1 = require('./module.cjs.js');
+var ElasticSearchSearchEngine = require('./engines/ElasticSearchSearchEngine.cjs.js');
+var ElasticSearchClientOptions = require('./engines/ElasticSearchClientOptions.cjs.js');
 
-var esb__default = /*#__PURE__*/_interopDefaultCompat(esb);
 
-const isOpenSearchCompatible = (opts) => {
-  return ["aws", "opensearch"].includes(opts?.provider ?? "");
-};
 
-class ElasticSearchClientWrapper {
-  elasticSearchClient;
-  openSearchClient;
-  constructor(options) {
-    this.openSearchClient = options.openSearchClient;
-    this.elasticSearchClient = options.elasticSearchClient;
-  }
-  static fromClientOptions(options) {
-    if (isOpenSearchCompatible(options)) {
-      return new ElasticSearchClientWrapper({
-        openSearchClient: new opensearch.Client(options)
-      });
-    }
-    return new ElasticSearchClientWrapper({
-      elasticSearchClient: new elasticsearch.Client(options)
-    });
-  }
-  search(options) {
-    const searchOptions = {
-      ignore_unavailable: true,
-      allow_no_indices: true
-    };
-    if (this.openSearchClient) {
-      return this.openSearchClient.search({
-        ...options,
-        ...searchOptions
-      });
-    }
-    if (this.elasticSearchClient) {
-      return this.elasticSearchClient.search({
-        ...options,
-        ...searchOptions
-      });
-    }
-    throw new Error("No client defined");
-  }
-  bulk(bulkOptions) {
-    if (this.openSearchClient) {
-      return this.openSearchClient.helpers.bulk(bulkOptions);
-    }
-    if (this.elasticSearchClient) {
-      return this.elasticSearchClient.helpers.bulk(bulkOptions);
-    }
-    throw new Error("No client defined");
-  }
-  putIndexTemplate(template) {
-    if (this.openSearchClient) {
-      return this.openSearchClient.indices.putIndexTemplate(template);
-    }
-    if (this.elasticSearchClient) {
-      return this.elasticSearchClient.indices.putIndexTemplate(template);
-    }
-    throw new Error("No client defined");
-  }
-  listIndices(options) {
-    if (this.openSearchClient) {
-      return this.openSearchClient.indices.get(options);
-    }
-    if (this.elasticSearchClient) {
-      return this.elasticSearchClient.indices.get(options);
-    }
-    throw new Error("No client defined");
-  }
-  indexExists(options) {
-    if (this.openSearchClient) {
-      return this.openSearchClient.indices.exists(options);
-    }
-    if (this.elasticSearchClient) {
-      return this.elasticSearchClient.indices.exists(options);
-    }
-    throw new Error("No client defined");
-  }
-  deleteIndex(options) {
-    if (this.openSearchClient) {
-      return this.openSearchClient.indices.delete(options);
-    }
-    if (this.elasticSearchClient) {
-      return this.elasticSearchClient.indices.delete(options);
-    }
-    throw new Error("No client defined");
-  }
-  /**
-   * @deprecated unused by the ElasticSearch Engine, will be removed in the future
-   */
-  getAliases(options) {
-    const { aliases } = options;
-    if (this.openSearchClient) {
-      return this.openSearchClient.cat.aliases({
-        format: "json",
-        name: aliases
-      });
-    }
-    if (this.elasticSearchClient) {
-      return this.elasticSearchClient.cat.aliases({
-        format: "json",
-        name: aliases
-      });
-    }
-    throw new Error("No client defined");
-  }
-  createIndex(options) {
-    if (this.openSearchClient) {
-      return this.openSearchClient.indices.create(options);
-    }
-    if (this.elasticSearchClient) {
-      return this.elasticSearchClient.indices.create(options);
-    }
-    throw new Error("No client defined");
-  }
-  updateAliases(options) {
-    const filteredActions = options.actions.filter(Boolean);
-    if (this.openSearchClient) {
-      return this.openSearchClient.indices.updateAliases({
-        body: {
-          actions: filteredActions
-        }
-      });
-    }
-    if (this.elasticSearchClient) {
-      return this.elasticSearchClient.indices.updateAliases({
-        body: {
-          actions: filteredActions
-        }
-      });
-    }
-    throw new Error("No client defined");
-  }
-}
-
-function duration(startTimestamp) {
-  const delta = process.hrtime(startTimestamp);
-  const seconds = delta[0] + delta[1] / 1e9;
-  return `${seconds.toFixed(1)}s`;
-}
-class ElasticSearchSearchEngineIndexer extends pluginSearchBackendNode.BatchSearchEngineIndexer {
-  processed = 0;
-  removableIndices = [];
-  startTimestamp;
-  type;
-  indexName;
-  indexPrefix;
-  indexSeparator;
-  alias;
-  logger;
-  sourceStream;
-  elasticSearchClientWrapper;
-  configuredBatchSize;
-  bulkResult;
-  bulkClientError;
-  constructor(options) {
-    super({ batchSize: options.batchSize });
-    this.configuredBatchSize = options.batchSize;
-    this.logger = options.logger.child({ documentType: options.type });
-    this.startTimestamp = process.hrtime();
-    this.type = options.type;
-    this.indexPrefix = options.indexPrefix;
-    this.indexSeparator = options.indexSeparator;
-    this.indexName = this.constructIndexName(`${Date.now()}`);
-    this.alias = options.alias;
-    this.elasticSearchClientWrapper = options.elasticSearchClientWrapper;
-    this.sourceStream = new stream.Readable({ objectMode: true });
-    this.sourceStream._read = () => {
-    };
-    const that = this;
-    this.bulkResult = this.elasticSearchClientWrapper.bulk({
-      datasource: this.sourceStream,
-      onDocument() {
-        that.processed++;
-        return {
-          index: { _index: that.indexName }
-        };
-      },
-      refreshOnCompletion: options.skipRefresh !== true
-    });
-    this.bulkResult.catch((e) => {
-      this.bulkClientError = e;
-    });
-  }
-  async initialize() {
-    this.logger.info(`Started indexing documents for index ${this.type}`);
-    const indices = await this.elasticSearchClientWrapper.listIndices({
-      index: this.constructIndexName("*")
-    });
-    for (const key of Object.keys(indices.body)) {
-      this.removableIndices.push(key);
-    }
-    await this.elasticSearchClientWrapper.createIndex({
-      index: this.indexName
-    });
-  }
-  async index(documents) {
-    await this.isReady();
-    documents.forEach((document) => {
-      this.sourceStream.push(document);
-    });
-  }
-  async finalize() {
-    await this.isReady();
-    this.sourceStream.push(null);
-    const result = await this.bulkResult;
-    if (this.processed === 0) {
-      this.logger.warn(
-        `Index for ${this.type} was not ${this.removableIndices.length ? "replaced" : "created"}: indexer received 0 documents`
-      );
-      try {
-        await this.elasticSearchClientWrapper.deleteIndex({
-          index: this.indexName
-        });
-      } catch (error) {
-        this.logger.error(`Unable to clean up elastic index: ${error}`);
-      }
-      return;
-    }
-    this.logger.info(
-      `Indexing completed for index ${this.type} in ${duration(
-        this.startTimestamp
-      )}`,
-      result
-    );
-    await this.elasticSearchClientWrapper.updateAliases({
-      actions: [
-        {
-          remove: { index: this.constructIndexName("*"), alias: this.alias }
-        },
-        {
-          add: { index: this.indexName, alias: this.alias }
-        }
-      ].filter(Boolean)
-    });
-    if (this.removableIndices.length) {
-      this.logger.info("Removing stale search indices", {
-        removableIndices: this.removableIndices
-      });
-      const chunks = this.removableIndices.reduce(
-        (resultArray, item, index) => {
-          const chunkIndex = Math.floor(index / 50);
-          if (!resultArray[chunkIndex]) {
-            resultArray[chunkIndex] = [];
-          }
-          resultArray[chunkIndex].push(item);
-          return resultArray;
-        },
-        []
-      );
-      for (const chunk of chunks) {
-        try {
-          await this.elasticSearchClientWrapper.deleteIndex({
-            index: chunk
-          });
-        } catch (e) {
-          this.logger.warn(`Failed to remove stale search indices: ${e}`);
-        }
-      }
-    }
-  }
-  /**
-   * Ensures that the number of documents sent over the wire to ES matches the
-   * number of documents this stream has received so far. This helps manage
-   * backpressure in other parts of the indexing pipeline.
-   */
-  isReady() {
-    if (this.bulkClientError) {
-      return Promise.reject(this.bulkClientError);
-    }
-    if (this.sourceStream.readableLength < this.configuredBatchSize) {
-      return Promise.resolve();
-    }
-    return new Promise((isReady, abort) => {
-      let streamLengthChecks = 0;
-      const interval = setInterval(() => {
-        streamLengthChecks++;
-        if (this.sourceStream.readableLength < this.configuredBatchSize) {
-          clearInterval(interval);
-          isReady();
-        }
-        if (streamLengthChecks >= 6e3) {
-          clearInterval(interval);
-          abort(
-            new Error(
-              "Exceeded 5 minutes waiting for elastic to be ready to accept more documents. Check the elastic logs for possible problems."
-            )
-          );
-        }
-      }, 50);
-    });
-  }
-  constructIndexName(postFix) {
-    return `${this.indexPrefix}${this.type}${this.indexSeparator}${postFix}`;
-  }
-}
-
-function isBlank(str) {
-  return lodash.isEmpty(str) && !lodash.isNumber(str) || lodash.isNaN(str);
-}
-const DEFAULT_INDEXER_BATCH_SIZE = 1e3;
-class ElasticSearchSearchEngine {
-  constructor(elasticSearchClientOptions, aliasPostfix, indexPrefix, logger, batchSize, highlightOptions) {
-    this.elasticSearchClientOptions = elasticSearchClientOptions;
-    this.aliasPostfix = aliasPostfix;
-    this.indexPrefix = indexPrefix;
-    this.logger = logger;
-    this.batchSize = batchSize;
-    this.elasticSearchClientWrapper = ElasticSearchClientWrapper.fromClientOptions(elasticSearchClientOptions);
-    const uuidTag = uuid.v4();
-    this.highlightOptions = {
-      preTag: `<${uuidTag}>`,
-      postTag: `</${uuidTag}>`,
-      fragmentSize: 1e3,
-      numFragments: 1,
-      fragmentDelimiter: " ... ",
-      ...highlightOptions
-    };
-  }
-  elasticSearchClientWrapper;
-  highlightOptions;
-  static async fromConfig(options) {
-    const {
-      logger,
-      config,
-      aliasPostfix = `search`,
-      indexPrefix = ``,
-      translator
-    } = options;
-    const credentialProvider = integrationAwsNode.DefaultAwsCredentialsManager.fromConfig(config);
-    const clientOptions = await this.createElasticSearchClientOptions(
-      await credentialProvider?.getCredentialProvider(),
-      config.getConfig("search.elasticsearch")
-    );
-    if (clientOptions.provider === "elastic") {
-      logger.info("Initializing Elastic.co ElasticSearch search engine.");
-    } else if (clientOptions.provider === "aws") {
-      logger.info("Initializing AWS OpenSearch search engine.");
-    } else if (clientOptions.provider === "opensearch") {
-      logger.info("Initializing OpenSearch search engine.");
-    } else {
-      logger.info("Initializing ElasticSearch search engine.");
-    }
-    const engine = new ElasticSearchSearchEngine(
-      clientOptions,
-      aliasPostfix,
-      indexPrefix,
-      logger,
-      config.getOptionalNumber("search.elasticsearch.batchSize") ?? DEFAULT_INDEXER_BATCH_SIZE,
-      config.getOptional(
-        "search.elasticsearch.highlightOptions"
-      )
-    );
-    for (const indexTemplate of this.readIndexTemplateConfig(
-      config.getConfig("search.elasticsearch")
-    )) {
-      await engine.setIndexTemplate(indexTemplate);
-    }
-    if (translator) {
-      await engine.setTranslator(translator);
-    }
-    return engine;
-  }
-  /**
-   * Create a custom search client from the derived search client configuration.
-   * This need not be the same client that the engine uses internally.
-   *
-   * @example Instantiate an instance of an Elasticsearch client.
-   *
-   * ```ts
-   * import { isOpenSearchCompatible } from '@backstage/plugin-search-backend-module-elasticsearch';
-   * import { Client } from '@elastic/elasticsearch';
-   *
-   * const client = searchEngine.newClient<Client>(options => {
-   *   // This type guard ensures options are compatible with either OpenSearch
-   *   // or Elasticsearch client constructors.
-   *   if (!isOpenSearchCompatible(options)) {
-   *     return new Client(options);
-   *   }
-   *   throw new Error('Incompatible options provided');
-   * });
-   * ```
-   */
-  newClient(create) {
-    return create(this.elasticSearchClientOptions);
-  }
-  translator(query, options) {
-    const { term, filters = {}, types, pageCursor } = query;
-    const filter = Object.entries(filters).filter(([_, value]) => Boolean(value)).map(([key, value]) => {
-      if (["string", "number", "boolean"].includes(typeof value)) {
-        const keyword = typeof value === "string" ? `${key}.keyword` : key;
-        return esb__default.default.matchQuery(keyword, value.toString());
-      }
-      if (Array.isArray(value)) {
-        return esb__default.default.boolQuery().should(value.map((it) => esb__default.default.matchQuery(key, it.toString())));
-      }
-      this.logger.error("Failed to query, unrecognized filter type", {
-        key,
-        value
-      });
-      throw new Error(
-        "Failed to add filters to query. Unrecognized filter type"
-      );
-    });
-    const esbQuery = isBlank(term) ? esb__default.default.matchAllQuery() : esb__default.default.multiMatchQuery(["*"], term).fuzziness("auto").minimumShouldMatch(1);
-    const pageSize = query.pageLimit || 25;
-    const { page } = decodePageCursor(pageCursor);
-    let esbRequestBodySearch = esb__default.default.requestBodySearch().query(esb__default.default.boolQuery().filter(filter).must([esbQuery])).from(page * pageSize).size(pageSize);
-    if (options?.highlightOptions) {
-      esbRequestBodySearch = esbRequestBodySearch.highlight(
-        esb__default.default.highlight("*").numberOfFragments(options.highlightOptions.numFragments).fragmentSize(options.highlightOptions.fragmentSize).preTags(options.highlightOptions.preTag).postTags(options.highlightOptions.postTag)
-      );
-    }
-    return {
-      elasticSearchQuery: esbRequestBodySearch.toJSON(),
-      documentTypes: types,
-      pageSize
-    };
-  }
-  setTranslator(translator) {
-    this.translator = translator;
-  }
-  async setIndexTemplate(template) {
-    try {
-      await this.elasticSearchClientWrapper.putIndexTemplate(template);
-      this.logger.info("Custom index template set");
-    } catch (error) {
-      this.logger.error(`Unable to set custom index template: ${error}`);
-    }
-  }
-  async getIndexer(type) {
-    const alias = this.constructSearchAlias(type);
-    const indexerLogger = this.logger.child({ documentType: type });
-    const indexer = new ElasticSearchSearchEngineIndexer({
-      type,
-      indexPrefix: this.indexPrefix,
-      indexSeparator: this.indexSeparator,
-      alias,
-      elasticSearchClientWrapper: this.elasticSearchClientWrapper,
-      logger: indexerLogger,
-      batchSize: this.batchSize,
-      skipRefresh: this.elasticSearchClientOptions?.service === "aoss"
-    });
-    indexer.on("error", async (e) => {
-      indexerLogger.error(`Failed to index documents for type ${type}`, e);
-      let cleanupError;
-      await new Promise(async (done) => {
-        const maxAttempts = 5;
-        let attempts = 0;
-        while (attempts < maxAttempts) {
-          try {
-            await this.elasticSearchClientWrapper.deleteIndex({
-              index: indexer.indexName
-            });
-            attempts = maxAttempts;
-            cleanupError = void 0;
-            done();
-          } catch (err) {
-            cleanupError = err;
-          }
-          await new Promise((okay) => setTimeout(okay, 1e3));
-          attempts++;
-        }
-        done();
-      });
-      if (cleanupError) {
-        indexerLogger.error(
-          `Unable to clean up elastic index ${indexer.indexName}: ${cleanupError}`
-        );
-      } else {
-        indexerLogger.info(
-          `Removed partial, failed index ${indexer.indexName}`
-        );
-      }
-    });
-    return indexer;
-  }
-  async query(query) {
-    const { elasticSearchQuery, documentTypes, pageSize } = this.translator(
-      query,
-      { highlightOptions: this.highlightOptions }
-    );
-    const queryIndices = documentTypes ? documentTypes.map((it) => this.constructSearchAlias(it)) : this.constructSearchAlias("*");
-    try {
-      const result = await this.elasticSearchClientWrapper.search({
-        index: queryIndices,
-        body: elasticSearchQuery
-      });
-      const { page } = decodePageCursor(query.pageCursor);
-      const hasNextPage = result.body.hits.total.value > (page + 1) * pageSize;
-      const hasPreviousPage = page > 0;
-      const nextPageCursor = hasNextPage ? encodePageCursor({ page: page + 1 }) : void 0;
-      const previousPageCursor = hasPreviousPage ? encodePageCursor({ page: page - 1 }) : void 0;
-      return {
-        results: result.body.hits.hits.map(
-          (d, index) => {
-            const resultItem = {
-              type: this.getTypeFromIndex(d._index),
-              document: d._source,
-              rank: pageSize * page + index + 1
-            };
-            if (d.highlight) {
-              resultItem.highlight = {
-                preTag: this.highlightOptions.preTag,
-                postTag: this.highlightOptions.postTag,
-                fields: Object.fromEntries(
-                  Object.entries(d.highlight).map(([field, fragments]) => [
-                    field,
-                    fragments.join(this.highlightOptions.fragmentDelimiter)
-                  ])
-                )
-              };
-            }
-            return resultItem;
-          }
-        ),
-        nextPageCursor,
-        previousPageCursor,
-        numberOfResults: result.body.hits.total.value
-      };
-    } catch (error) {
-      if (error.meta?.body?.error?.type === "index_not_found_exception") {
-        throw new pluginSearchBackendNode.MissingIndexError(
-          `Missing index for ${queryIndices}. This means there are no documents to search through.`,
-          error
-        );
-      }
-      this.logger.error(
-        `Failed to query documents for indices ${queryIndices}`,
-        error
-      );
-      return Promise.reject({ results: [] });
-    }
-  }
-  indexSeparator = "-index__";
-  getTypeFromIndex(index) {
-    return index.substring(this.indexPrefix.length).split(this.indexSeparator)[0];
-  }
-  constructSearchAlias(type) {
-    const postFix = this.aliasPostfix ? `__${this.aliasPostfix}` : "";
-    return `${this.indexPrefix}${type}${postFix}`;
-  }
-  static async createElasticSearchClientOptions(credentialProvider, config) {
-    if (!config) {
-      throw new Error("No elastic search config found");
-    }
-    const clientOptionsConfig = config.getOptionalConfig("clientOptions");
-    const sslConfig = clientOptionsConfig?.getOptionalConfig("ssl");
-    if (config.getOptionalString("provider") === "elastic") {
-      const authConfig2 = config.getConfig("auth");
-      return {
-        provider: "elastic",
-        cloud: {
-          id: config.getString("cloudId")
-        },
-        auth: {
-          username: authConfig2.getString("username"),
-          password: authConfig2.getString("password")
-        },
-        ...sslConfig ? {
-          ssl: {
-            rejectUnauthorized: sslConfig?.getOptionalBoolean("rejectUnauthorized")
-          }
-        } : {}
-      };
-    }
-    if (config.getOptionalString("provider") === "aws") {
-      const requestSigner = new aws4.RequestSigner(config.getString("node"));
-      const service = config.getOptionalString("service") ?? requestSigner.service;
-      if (service !== "es" && service !== "aoss")
-        throw new Error(`Unrecognized serivce type: ${service}`);
-      return {
-        provider: "aws",
-        node: config.getString("node"),
-        region: config.getOptionalString("region"),
-        service,
-        ...sslConfig ? {
-          ssl: {
-            rejectUnauthorized: sslConfig?.getOptionalBoolean("rejectUnauthorized")
-          }
-        } : {},
-        ...aws.AwsSigv4Signer({
-          region: config.getOptionalString("region") ?? requestSigner.region,
-          // for backwards compatibility
-          service,
-          getCredentials: async () => await credentialProvider.sdkCredentialProvider()
-        })
-      };
-    }
-    if (config.getOptionalString("provider") === "opensearch") {
-      const authConfig2 = config.getConfig("auth");
-      return {
-        provider: "opensearch",
-        node: config.getString("node"),
-        auth: {
-          username: authConfig2.getString("username"),
-          password: authConfig2.getString("password")
-        },
-        ...sslConfig ? {
-          ssl: {
-            rejectUnauthorized: sslConfig?.getOptionalBoolean("rejectUnauthorized")
-          }
-        } : {}
-      };
-    }
-    const authConfig = config.getOptionalConfig("auth");
-    const auth = authConfig && (authConfig.has("apiKey") ? {
-      apiKey: authConfig.getString("apiKey")
-    } : {
-      username: authConfig.getString("username"),
-      password: authConfig.getString("password")
-    });
-    return {
-      node: config.getString("node"),
-      auth,
-      ...sslConfig ? {
-        ssl: {
-          rejectUnauthorized: sslConfig?.getOptionalBoolean("rejectUnauthorized")
-        }
-      } : {}
-    };
-  }
-  static readIndexTemplateConfig(config) {
-    return config.getOptionalConfigArray("indexTemplates")?.map((templateConfig) => {
-      const bodyConfig = templateConfig.getConfig("body");
-      return {
-        name: templateConfig.getString("name"),
-        body: {
-          index_patterns: bodyConfig.getStringArray("index_patterns"),
-          composed_of: bodyConfig.getOptionalStringArray("composed_of"),
-          template: bodyConfig.getOptionalConfig("template")?.get()
-        }
-      };
-    }) ?? [];
-  }
-}
-function decodePageCursor(pageCursor) {
-  if (!pageCursor) {
-    return { page: 0 };
-  }
-  return {
-    page: Number(Buffer.from(pageCursor, "base64").toString("utf-8"))
-  };
-}
-function encodePageCursor({ page }) {
-  return Buffer.from(`${page}`, "utf-8").toString("base64");
-}
-
-exports.ElasticSearchSearchEngine = ElasticSearchSearchEngine;
-exports.decodeElasticSearchPageCursor = decodePageCursor;
-exports.isOpenSearchCompatible = isOpenSearchCompatible;
+exports.default = module$1.default;
+exports.elasticsearchTranslatorExtensionPoint = module$1.elasticsearchTranslatorExtensionPoint;
+exports.ElasticSearchSearchEngine = ElasticSearchSearchEngine.ElasticSearchSearchEngine;
+exports.decodeElasticSearchPageCursor = ElasticSearchSearchEngine.decodePageCursor;
+exports.isOpenSearchCompatible = ElasticSearchClientOptions.isOpenSearchCompatible;
 //# sourceMappingURL=index.cjs.js.map