@backstage/plugin-search-backend-module-pg 0.5.48-next.0 → 0.5.49-next.0

package/CHANGELOG.md

@@ -1,5 +1,25 @@
 # @backstage/plugin-search-backend-module-pg
 
+## 0.5.49-next.0
+
+### Patch Changes
+
+- a919ca3: Truncate long docs to fit PG index size limit
+- 8d15a51: Added the < character to the query filter regexp
+- Updated dependencies
+  - @backstage/plugin-search-backend-node@1.3.15
+  - @backstage/backend-plugin-api@1.4.3
+  - @backstage/config@1.3.3
+  - @backstage/plugin-search-common@1.2.19
+
+## 0.5.48
+
+### Patch Changes
+
+- Updated dependencies
+  - @backstage/backend-plugin-api@1.4.3
+  - @backstage/plugin-search-backend-node@1.3.15
+
 ## 0.5.48-next.0
 
 ### Patch Changes

package/dist/PgSearchEngine/PgSearchEngine.cjs.js

@@ -88,7 +88,7 @@ class PgSearchEngine {
     const normalization = options.normalization || 0;
     return {
       pgQuery: {
-        pgTerm: query.term.split(/\s/).map((p) => p.replace(/[\0()|&:*!]/g, "").trim()).filter((p) => p !== "").map((p) => `(${JSON.stringify(p)} | ${JSON.stringify(p)}:*)`).join("&"),
+        pgTerm: query.term.split(/\s/).map((p) => p.replace(/[\0()|&:*!<]/g, "").trim()).filter((p) => p !== "").map((p) => `(${JSON.stringify(p)} | ${JSON.stringify(p)}:*)`).join("&"),
         fields: query.filters,
         types: query.types,
         offset,

package/dist/PgSearchEngine/PgSearchEngine.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"PgSearchEngine.cjs.js","sources":["../../src/PgSearchEngine/PgSearchEngine.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { SearchEngine } from '@backstage/plugin-search-backend-node';\nimport {\n  SearchQuery,\n  IndexableResultSet,\n  IndexableResult,\n} from '@backstage/plugin-search-common';\nimport { PgSearchEngineIndexer } from './PgSearchEngineIndexer';\nimport {\n  DatabaseDocumentStore,\n  DatabaseStore,\n  PgSearchQuery,\n} from '../database';\nimport { v4 as uuid } from 'uuid';\nimport { Config } from '@backstage/config';\nimport { DatabaseService, LoggerService } from '@backstage/backend-plugin-api';\n\n/**\n * Search query that the Postgres search engine understands.\n * @public\n */\nexport type ConcretePgSearchQuery = {\n  pgQuery: PgSearchQuery;\n  pageSize: number;\n};\n\n/**\n * Options available for the Postgres specific query translator.\n * @public\n */\nexport type PgSearchQueryTranslatorOptions = {\n  highlightOptions: PgSearchHighlightOptions;\n  normalization?: number;\n};\n\n/**\n * Postgres specific query translator.\n * @public\n */\nexport type PgSearchQueryTranslator = (\n  query: SearchQuery,\n  options: PgSearchQueryTranslatorOptions,\n) => ConcretePgSearchQuery;\n\n/**\n * Options to instantiate PgSearchEngine\n * @public\n */\nexport type PgSearchOptions = {\n  database: DatabaseService;\n  logger?: LoggerService;\n};\n\n/**\n * Options for highlighting search terms\n * @public\n */\nexport type PgSearchHighlightOptions = {\n  useHighlight?: boolean;\n  maxWords?: number;\n  minWords?: number;\n  shortWord?: number;\n  highlightAll?: boolean;\n  maxFragments?: number;\n  fragmentDelimiter?: string;\n  preTag: string;\n  postTag: string;\n};\n\n/** @public */\nexport class PgSearchEngine implements SearchEngine {\n  private readonly logger?: LoggerService;\n  private readonly highlightOptions: PgSearchHighlightOptions;\n  private readonly indexerBatchSize: number;\n  private readonly normalization: number;\n\n  /**\n   * @deprecated This will be marked as private in a future release, please us fromConfig instead\n   */\n  constructor(\n    private readonly databaseStore: DatabaseStore,\n    config: Config,\n    logger?: LoggerService,\n  ) {\n    const uuidTag = uuid();\n    const highlightConfig = config.getOptionalConfig(\n      'search.pg.highlightOptions',\n    );\n\n    const highlightOptions: PgSearchHighlightOptions = {\n      preTag: `<${uuidTag}>`,\n      postTag: `</${uuidTag}>`,\n      useHighlight: highlightConfig?.getOptionalBoolean('useHighlight') ?? true,\n      maxWords: highlightConfig?.getOptionalNumber('maxWords') ?? 35,\n      minWords: highlightConfig?.getOptionalNumber('minWords') ?? 15,\n      shortWord: highlightConfig?.getOptionalNumber('shortWord') ?? 3,\n      highlightAll:\n        highlightConfig?.getOptionalBoolean('highlightAll') ?? false,\n      maxFragments: highlightConfig?.getOptionalNumber('maxFragments') ?? 0,\n      fragmentDelimiter:\n        highlightConfig?.getOptionalString('fragmentDelimiter') ?? ' ... ',\n    };\n    this.highlightOptions = highlightOptions;\n    this.indexerBatchSize =\n      config.getOptionalNumber('search.pg.indexerBatchSize') ?? 1000;\n\n    this.normalization = this.getNormalizationValue(config);\n    this.logger = logger;\n  }\n\n  private getNormalizationValue(config: Config) {\n    const normalizationConfig =\n      config.getOptional('search.pg.normalization') ?? 0;\n    if (typeof normalizationConfig === 'number') {\n      return normalizationConfig;\n    } else if (typeof normalizationConfig === 'string') {\n      return this.evaluateBitwiseOrExpression(normalizationConfig);\n    }\n    this.logger?.error(\n      `Unknown normalization configuration: ${normalizationConfig}`,\n    );\n\n    return 0;\n  }\n\n  private evaluateBitwiseOrExpression(expression: string) {\n    const tokens = expression.split('|').map(token => token.trim());\n\n    const numbers = tokens.map(token => {\n      const num = parseInt(token, 10);\n      if (isNaN(num)) {\n        this.logger?.error(\n          `Unknown expression for normalization: ${expression}`,\n        );\n        return 0;\n      }\n      return num;\n    });\n    return numbers.reduce((acc, num) => acc | num, 0);\n  }\n\n  /**\n   * @deprecated This will be removed in a future release, please use fromConfig instead\n   */\n  static async from(options: {\n    database: DatabaseService;\n    config: Config;\n    logger?: LoggerService;\n  }): Promise<PgSearchEngine> {\n    return new PgSearchEngine(\n      await DatabaseDocumentStore.create(options.database),\n      options.config,\n      options.logger,\n    );\n  }\n\n  static async fromConfig(config: Config, options: PgSearchOptions) {\n    return new PgSearchEngine(\n      await DatabaseDocumentStore.create(options.database),\n      config,\n      options.logger,\n    );\n  }\n\n  static async supported(database: DatabaseService): Promise<boolean> {\n    return await DatabaseDocumentStore.supported(await database.getClient());\n  }\n\n  translator(\n    query: SearchQuery,\n    options: PgSearchQueryTranslatorOptions,\n  ): ConcretePgSearchQuery {\n    const pageSize = query.pageLimit || 25;\n    const { page } = decodePageCursor(query.pageCursor);\n    const offset = page * pageSize;\n    // We request more result to know whether there is another page\n    const limit = pageSize + 1;\n    const normalization = options.normalization || 0;\n\n    return {\n      pgQuery: {\n        pgTerm: query.term\n          .split(/\\s/)\n          .map(p => p.replace(/[\\0()|&:*!]/g, '').trim())\n          .filter(p => p !== '')\n          .map(p => `(${JSON.stringify(p)} | ${JSON.stringify(p)}:*)`)\n          .join('&'),\n        fields: query.filters as Record<string, string | string[]>,\n        types: query.types,\n        offset,\n        limit,\n        normalization,\n        options: options.highlightOptions,\n      },\n      pageSize,\n    };\n  }\n\n  setTranslator(translator: PgSearchQueryTranslator) {\n    this.translator = translator;\n  }\n\n  async getIndexer(type: string) {\n    return new PgSearchEngineIndexer({\n      batchSize: this.indexerBatchSize,\n      type,\n      databaseStore: this.databaseStore,\n      logger: this.logger?.child({ documentType: type }),\n    });\n  }\n\n  async query(query: SearchQuery): Promise<IndexableResultSet> {\n    const { pgQuery, pageSize } = this.translator(query, {\n      highlightOptions: this.highlightOptions,\n      normalization: this.normalization,\n    });\n\n    const rows = await this.databaseStore.transaction(async tx =>\n      this.databaseStore.query(tx, pgQuery),\n    );\n\n    // We requested one result more than the page size to know whether there is\n    // another page.\n    const { page } = decodePageCursor(query.pageCursor);\n    const hasNextPage = rows.length > pageSize;\n    const hasPreviousPage = page > 0;\n    const pageRows = rows.slice(0, pageSize);\n    const nextPageCursor = hasNextPage\n      ? encodePageCursor({ page: page + 1 })\n      : undefined;\n    const previousPageCursor = hasPreviousPage\n      ? encodePageCursor({ page: page - 1 })\n      : undefined;\n\n    const results = pageRows.map(\n      ({ type, document, highlight }, index): IndexableResult => ({\n        type,\n        document,\n        rank: page * pageSize + index + 1,\n        highlight: {\n          preTag: pgQuery.options.preTag,\n          postTag: pgQuery.options.postTag,\n          fields: highlight\n            ? {\n                text: highlight.text,\n                title: highlight.title,\n                location: highlight.location,\n                path: '',\n              }\n            : {},\n        },\n      }),\n    );\n\n    return { results, nextPageCursor, previousPageCursor };\n  }\n}\n\nexport function decodePageCursor(pageCursor?: string): { page: number } {\n  if (!pageCursor) {\n    return { page: 0 };\n  }\n\n  return {\n    page: Number(Buffer.from(pageCursor, 'base64').toString('utf-8')),\n  };\n}\n\nexport function encodePageCursor({ page }: { page: number }): string {\n  return Buffer.from(`${page}`, 'utf-8').toString('base64');\n}\n"],"names":["uuid","DatabaseDocumentStore","PgSearchEngineIndexer"],"mappings":";;;;;;AAqFO,MAAM,cAAA,CAAuC;AAAA;AAAA;AAAA;AAAA,EASlD,WAAA,CACmB,aAAA,EACjB,MAAA,EACA,MAAA,EACA;AAHiB,IAAA,IAAA,CAAA,aAAA,GAAA,aAAA;AAIjB,IAAA,MAAM,UAAUA,OAAA,EAAK;AACrB,IAAA,MAAM,kBAAkB,MAAA,CAAO,iBAAA;AAAA,MAC7B;AAAA,KACF;AAEA,IAAA,MAAM,gBAAA,GAA6C;AAAA,MACjD,MAAA,EAAQ,IAAI,OAAO,CAAA,CAAA,CAAA;AAAA,MACnB,OAAA,EAAS,KAAK,OAAO,CAAA,CAAA,CAAA;AAAA,MACrB,YAAA,EAAc,eAAA,EAAiB,kBAAA,CAAmB,cAAc,CAAA,IAAK,IAAA;AAAA,MACrE,QAAA,EAAU,eAAA,EAAiB,iBAAA,CAAkB,UAAU,CAAA,IAAK,EAAA;AAAA,MAC5D,QAAA,EAAU,eAAA,EAAiB,iBAAA,CAAkB,UAAU,CAAA,IAAK,EAAA;AAAA,MAC5D,SAAA,EAAW,eAAA,EAAiB,iBAAA,CAAkB,WAAW,CAAA,IAAK,CAAA;AAAA,MAC9D,YAAA,EACE,eAAA,EAAiB,kBAAA,CAAmB,cAAc,CAAA,IAAK,KAAA;AAAA,MACzD,YAAA,EAAc,eAAA,EAAiB,iBAAA,CAAkB,cAAc,CAAA,IAAK,CAAA;AAAA,MACpE,iBAAA,EACE,eAAA,EAAiB,iBAAA,CAAkB,mBAAmB,CAAA,IAAK;AAAA,KAC/D;AACA,IAAA,IAAA,CAAK,gBAAA,GAAmB,gBAAA;AACxB,IAAA,IAAA,CAAK,gBAAA,GACH,MAAA,CAAO,iBAAA,CAAkB,4BAA4B,CAAA,IAAK,GAAA;AAE5D,IAAA,IAAA,CAAK,aAAA,GAAgB,IAAA,CAAK,qBAAA,CAAsB,MAAM,CAAA;AACtD,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AAAA,EAChB;AAAA,EArCiB,MAAA;AAAA,EACA,gBAAA;AAAA,EACA,gBAAA;AAAA,EACA,aAAA;AAAA,EAoCT,sBAAsB,MAAA,EAAgB;AAC5C,IAAA,MAAM,mBAAA,GACJ,MAAA,CAAO,WAAA,CAAY,yBAAyB,CAAA,IAAK,CAAA;AACnD,IAAA,IAAI,OAAO,wBAAwB,QAAA,EAAU;AAC3C,MAAA,OAAO,mBAAA;AAAA,IACT,CAAA,MAAA,IAAW,OAAO,mBAAA,KAAwB,QAAA,EAAU;AAClD,MAAA,OAAO,IAAA,CAAK,4BAA4B,mBAAmB,CAAA;AAAA,IAC7D;AACA,IAAA,IAAA,CAAK,MAAA,EAAQ,KAAA;AAAA,MACX,wCAAwC,mBAAmB,CAAA;AAAA,KAC7D;AAEA,IAAA,OAAO,CAAA;AAAA,EACT;AAAA,EAEQ,4BAA4B,UAAA,EAAoB;AACtD,IAAA,MAAM,MAAA,GAAS,WAAW,KAAA,CAAM,GAAG,EAAE,GAAA,CAAI,CAAA,KAAA,KAAS,KAAA,CAAM,IAAA,EAAM,CAAA;AAE9D,IAAA,MAAM,OAAA,GAAU,MAAA,CAAO,GAAA,CAAI,CAAA,KAAA,KAAS;AAClC,MAAA,MAAM,GAAA,GAAM,QAAA,CAAS,KAAA,EAAO,EAAE,CAAA;AAC9B,MAAA,IAAI,KAAA,CAAM,GAAG,CAAA,EAAG;AACd,QAAA,IAAA,CAAK,MAAA,EAAQ,KAAA;AAAA,UACX,yCAAyC,UAAU,CAAA;AAAA,SACrD;AACA,QAAA,OAAO,CAAA;AAAA,MACT;AACA,MAAA,OAAO,GAAA;AAAA,IACT,CAAC,CAAA;AACD,IAAA,OAAO,QAAQ,MAAA,CAAO,CAAC,KAAK,GAAA,KAAQ,GAAA,GAAM,KAAK,CAAC,CAAA;AAAA,EAClD;AAAA;AAAA;AAAA;AAAA,EAKA,aAAa,KAAK,OAAA,EAIU;AAC1B,IAAA,OAAO,IAAI,cAAA;AAAA,MACT,MAAMC,2CAAA,CAAsB,MAAA,CAAO,OAAA,CAAQ,QAAQ,CAAA;AAAA,MACnD,OAAA,CAAQ,MAAA;AAAA,MACR,OAAA,CAAQ;AAAA,KACV;AAAA,EACF;AAAA,EAEA,aAAa,UAAA,CAAW,MAAA,EAAgB,OAAA,EAA0B;AAChE,IAAA,OAAO,IAAI,cAAA;AAAA,MACT,MAAMA,2CAAA,CAAsB,MAAA,CAAO,OAAA,CAAQ,QAAQ,CAAA;AAAA,MACnD,MAAA;AAAA,MACA,OAAA,CAAQ;AAAA,KACV;AAAA,EACF;AAAA,EAEA,aAAa,UAAU,QAAA,EAA6C;AAClE,IAAA,OAAO,MAAMA,2CAAA,CAAsB,SAAA,CAAU,MAAM,QAAA,CAAS,WAAW,CAAA;AAAA,EACzE;AAAA,EAEA,UAAA,CACE,OACA,OAAA,EACuB;AACvB,IAAA,MAAM,QAAA,GAAW,MAAM,SAAA,IAAa,EAAA;AACpC,IAAA,MAAM,EAAE,IAAA,EAAK,GAAI,gBAAA,CAAiB,MAAM,UAAU,CAAA;AAClD,IAAA,MAAM,SAAS,IAAA,GAAO,QAAA;AAEtB,IAAA,MAAM,QAAQ,QAAA,GAAW,CAAA;AACzB,IAAA,MAAM,aAAA,GAAgB,QAAQ,aAAA,IAAiB,CAAA;AAE/C,IAAA,OAAO;AAAA,MACL,OAAA,EAAS;AAAA,QACP,QAAQ,KAAA,CAAM,IAAA,CACX,KAAA,CAAM,IAAI,EACV,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,OAAA,CAAQ,gBAAgB,EAAE,CAAA,CAAE,IAAA,EAAM,EAC7C,MAAA,CAAO,CAAA,CAAA,KAAK,CAAA,KAAM,EAAE,EACpB,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAA,EAAI,IAAA,CAAK,UAAU,CAAC,CAAC,CAAA,GAAA,EAAM,IAAA,CAAK,UAAU,CAAC,CAAC,CAAA,GAAA,CAAK,CAAA,CAC1D,KAAK,GAAG,CAAA;AAAA,QACX,QAAQ,KAAA,CAAM,OAAA;AAAA,QACd,OAAO,KAAA,CAAM,KAAA;AAAA,QACb,MAAA;AAAA,QACA,KAAA;AAAA,QACA,aAAA;AAAA,QACA,SAAS,OAAA,CAAQ;AAAA,OACnB;AAAA,MACA;AAAA,KACF;AAAA,EACF;AAAA,EAEA,cAAc,UAAA,EAAqC;AACjD,IAAA,IAAA,CAAK,UAAA,GAAa,UAAA;AAAA,EACpB;AAAA,EAEA,MAAM,WAAW,IAAA,EAAc;AAC7B,IAAA,OAAO,IAAIC,2CAAA,CAAsB;AAAA,MAC/B,WAAW,IAAA,CAAK,gBAAA;AAAA,MAChB,IAAA;AAAA,MACA,eAAe,IAAA,CAAK,aAAA;AAAA,MACpB,QAAQ,IAAA,CAAK,MAAA,EAAQ,MAAM,EAAE,YAAA,EAAc,MAAM;AAAA,KAClD,CAAA;AAAA,EACH;AAAA,EAEA,MAAM,MAAM,KAAA,EAAiD;AAC3D,IAAA,MAAM,EAAE,OAAA,EAAS,QAAA,EAAS,GAAI,IAAA,CAAK,WAAW,KAAA,EAAO;AAAA,MACnD,kBAAkB,IAAA,CAAK,gBAAA;AAAA,MACvB,eAAe,IAAA,CAAK;AAAA,KACrB,CAAA;AAED,IAAA,MAAM,IAAA,GAAO,MAAM,IAAA,CAAK,aAAA,CAAc,WAAA;AAAA,MAAY,OAAM,EAAA,KACtD,IAAA,CAAK,aAAA,CAAc,KAAA,CAAM,IAAI,OAAO;AAAA,KACtC;AAIA,IAAA,MAAM,EAAE,IAAA,EAAK,GAAI,gBAAA,CAAiB,MAAM,UAAU,CAAA;AAClD,IAAA,MAAM,WAAA,GAAc,KAAK,MAAA,GAAS,QAAA;AAClC,IAAA,MAAM,kBAAkB,IAAA,GAAO,CAAA;AAC/B,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAA;AACvC,IAAA,MAAM,cAAA,GAAiB,cACnB,gBAAA,CAAiB,EAAE,MAAM,IAAA,GAAO,CAAA,EAAG,CAAA,GACnC,MAAA;AACJ,IAAA,MAAM,kBAAA,GAAqB,kBACvB,gBAAA,CAAiB,EAAE,MAAM,IAAA,GAAO,CAAA,EAAG,CAAA,GACnC,MAAA;AAEJ,IAAA,MAAM,UAAU,QAAA,CAAS,GAAA;AAAA,MACvB,CAAC,EAAE,IAAA,EAAM,QAAA,EAAU,SAAA,IAAa,KAAA,MAA4B;AAAA,QAC1D,IAAA;AAAA,QACA,QAAA;AAAA,QACA,IAAA,EAAM,IAAA,GAAO,QAAA,GAAW,KAAA,GAAQ,CAAA;AAAA,QAChC,SAAA,EAAW;AAAA,UACT,MAAA,EAAQ,QAAQ,OAAA,CAAQ,MAAA;AAAA,UACxB,OAAA,EAAS,QAAQ,OAAA,CAAQ,OAAA;AAAA,UACzB,QAAQ,SAAA,GACJ;AAAA,YACE,MAAM,SAAA,CAAU,IAAA;AAAA,YAChB,OAAO,SAAA,CAAU,KAAA;AAAA,YACjB,UAAU,SAAA,CAAU,QAAA;AAAA,YACpB,IAAA,EAAM;AAAA,cAER;AAAC;AACP,OACF;AAAA,KACF;AAEA,IAAA,OAAO,EAAE,OAAA,EAAS,cAAA,EAAgB,kBAAA,EAAmB;AAAA,EACvD;AACF;AAEO,SAAS,iBAAiB,UAAA,EAAuC;AACtE,EAAA,IAAI,CAAC,UAAA,EAAY;AACf,IAAA,OAAO,EAAE,MAAM,CAAA,EAAE;AAAA,EACnB;AAEA,EAAA,OAAO;AAAA,IACL,IAAA,EAAM,OAAO,MAAA,CAAO,IAAA,CAAK,YAAY,QAAQ,CAAA,CAAE,QAAA,CAAS,OAAO,CAAC;AAAA,GAClE;AACF;AAEO,SAAS,gBAAA,CAAiB,EAAE,IAAA,EAAK,EAA6B;AACnE,EAAA,OAAO,MAAA,CAAO,KAAK,CAAA,EAAG,IAAI,IAAI,OAAO,CAAA,CAAE,SAAS,QAAQ,CAAA;AAC1D;;;;;;"}
+{"version":3,"file":"PgSearchEngine.cjs.js","sources":["../../src/PgSearchEngine/PgSearchEngine.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { SearchEngine } from '@backstage/plugin-search-backend-node';\nimport {\n  SearchQuery,\n  IndexableResultSet,\n  IndexableResult,\n} from '@backstage/plugin-search-common';\nimport { PgSearchEngineIndexer } from './PgSearchEngineIndexer';\nimport {\n  DatabaseDocumentStore,\n  DatabaseStore,\n  PgSearchQuery,\n} from '../database';\nimport { v4 as uuid } from 'uuid';\nimport { Config } from '@backstage/config';\nimport { DatabaseService, LoggerService } from '@backstage/backend-plugin-api';\n\n/**\n * Search query that the Postgres search engine understands.\n * @public\n */\nexport type ConcretePgSearchQuery = {\n  pgQuery: PgSearchQuery;\n  pageSize: number;\n};\n\n/**\n * Options available for the Postgres specific query translator.\n * @public\n */\nexport type PgSearchQueryTranslatorOptions = {\n  highlightOptions: PgSearchHighlightOptions;\n  normalization?: number;\n};\n\n/**\n * Postgres specific query translator.\n * @public\n */\nexport type PgSearchQueryTranslator = (\n  query: SearchQuery,\n  options: PgSearchQueryTranslatorOptions,\n) => ConcretePgSearchQuery;\n\n/**\n * Options to instantiate PgSearchEngine\n * @public\n */\nexport type PgSearchOptions = {\n  database: DatabaseService;\n  logger?: LoggerService;\n};\n\n/**\n * Options for highlighting search terms\n * @public\n */\nexport type PgSearchHighlightOptions = {\n  useHighlight?: boolean;\n  maxWords?: number;\n  minWords?: number;\n  shortWord?: number;\n  highlightAll?: boolean;\n  maxFragments?: number;\n  fragmentDelimiter?: string;\n  preTag: string;\n  postTag: string;\n};\n\n/** @public */\nexport class PgSearchEngine implements SearchEngine {\n  private readonly logger?: LoggerService;\n  private readonly highlightOptions: PgSearchHighlightOptions;\n  private readonly indexerBatchSize: number;\n  private readonly normalization: number;\n\n  /**\n   * @deprecated This will be marked as private in a future release, please us fromConfig instead\n   */\n  constructor(\n    private readonly databaseStore: DatabaseStore,\n    config: Config,\n    logger?: LoggerService,\n  ) {\n    const uuidTag = uuid();\n    const highlightConfig = config.getOptionalConfig(\n      'search.pg.highlightOptions',\n    );\n\n    const highlightOptions: PgSearchHighlightOptions = {\n      preTag: `<${uuidTag}>`,\n      postTag: `</${uuidTag}>`,\n      useHighlight: highlightConfig?.getOptionalBoolean('useHighlight') ?? true,\n      maxWords: highlightConfig?.getOptionalNumber('maxWords') ?? 35,\n      minWords: highlightConfig?.getOptionalNumber('minWords') ?? 15,\n      shortWord: highlightConfig?.getOptionalNumber('shortWord') ?? 3,\n      highlightAll:\n        highlightConfig?.getOptionalBoolean('highlightAll') ?? false,\n      maxFragments: highlightConfig?.getOptionalNumber('maxFragments') ?? 0,\n      fragmentDelimiter:\n        highlightConfig?.getOptionalString('fragmentDelimiter') ?? ' ... ',\n    };\n    this.highlightOptions = highlightOptions;\n    this.indexerBatchSize =\n      config.getOptionalNumber('search.pg.indexerBatchSize') ?? 1000;\n\n    this.normalization = this.getNormalizationValue(config);\n    this.logger = logger;\n  }\n\n  private getNormalizationValue(config: Config) {\n    const normalizationConfig =\n      config.getOptional('search.pg.normalization') ?? 0;\n    if (typeof normalizationConfig === 'number') {\n      return normalizationConfig;\n    } else if (typeof normalizationConfig === 'string') {\n      return this.evaluateBitwiseOrExpression(normalizationConfig);\n    }\n    this.logger?.error(\n      `Unknown normalization configuration: ${normalizationConfig}`,\n    );\n\n    return 0;\n  }\n\n  private evaluateBitwiseOrExpression(expression: string) {\n    const tokens = expression.split('|').map(token => token.trim());\n\n    const numbers = tokens.map(token => {\n      const num = parseInt(token, 10);\n      if (isNaN(num)) {\n        this.logger?.error(\n          `Unknown expression for normalization: ${expression}`,\n        );\n        return 0;\n      }\n      return num;\n    });\n    return numbers.reduce((acc, num) => acc | num, 0);\n  }\n\n  /**\n   * @deprecated This will be removed in a future release, please use fromConfig instead\n   */\n  static async from(options: {\n    database: DatabaseService;\n    config: Config;\n    logger?: LoggerService;\n  }): Promise<PgSearchEngine> {\n    return new PgSearchEngine(\n      await DatabaseDocumentStore.create(options.database),\n      options.config,\n      options.logger,\n    );\n  }\n\n  static async fromConfig(config: Config, options: PgSearchOptions) {\n    return new PgSearchEngine(\n      await DatabaseDocumentStore.create(options.database),\n      config,\n      options.logger,\n    );\n  }\n\n  static async supported(database: DatabaseService): Promise<boolean> {\n    return await DatabaseDocumentStore.supported(await database.getClient());\n  }\n\n  translator(\n    query: SearchQuery,\n    options: PgSearchQueryTranslatorOptions,\n  ): ConcretePgSearchQuery {\n    const pageSize = query.pageLimit || 25;\n    const { page } = decodePageCursor(query.pageCursor);\n    const offset = page * pageSize;\n    // We request more result to know whether there is another page\n    const limit = pageSize + 1;\n    const normalization = options.normalization || 0;\n\n    return {\n      pgQuery: {\n        pgTerm: query.term\n          .split(/\\s/)\n          .map(p => p.replace(/[\\0()|&:*!<]/g, '').trim())\n          .filter(p => p !== '')\n          .map(p => `(${JSON.stringify(p)} | ${JSON.stringify(p)}:*)`)\n          .join('&'),\n        fields: query.filters as Record<string, string | string[]>,\n        types: query.types,\n        offset,\n        limit,\n        normalization,\n        options: options.highlightOptions,\n      },\n      pageSize,\n    };\n  }\n\n  setTranslator(translator: PgSearchQueryTranslator) {\n    this.translator = translator;\n  }\n\n  async getIndexer(type: string) {\n    return new PgSearchEngineIndexer({\n      batchSize: this.indexerBatchSize,\n      type,\n      databaseStore: this.databaseStore,\n      logger: this.logger?.child({ documentType: type }),\n    });\n  }\n\n  async query(query: SearchQuery): Promise<IndexableResultSet> {\n    const { pgQuery, pageSize } = this.translator(query, {\n      highlightOptions: this.highlightOptions,\n      normalization: this.normalization,\n    });\n\n    const rows = await this.databaseStore.transaction(async tx =>\n      this.databaseStore.query(tx, pgQuery),\n    );\n\n    // We requested one result more than the page size to know whether there is\n    // another page.\n    const { page } = decodePageCursor(query.pageCursor);\n    const hasNextPage = rows.length > pageSize;\n    const hasPreviousPage = page > 0;\n    const pageRows = rows.slice(0, pageSize);\n    const nextPageCursor = hasNextPage\n      ? encodePageCursor({ page: page + 1 })\n      : undefined;\n    const previousPageCursor = hasPreviousPage\n      ? encodePageCursor({ page: page - 1 })\n      : undefined;\n\n    const results = pageRows.map(\n      ({ type, document, highlight }, index): IndexableResult => ({\n        type,\n        document,\n        rank: page * pageSize + index + 1,\n        highlight: {\n          preTag: pgQuery.options.preTag,\n          postTag: pgQuery.options.postTag,\n          fields: highlight\n            ? {\n                text: highlight.text,\n                title: highlight.title,\n                location: highlight.location,\n                path: '',\n              }\n            : {},\n        },\n      }),\n    );\n\n    return { results, nextPageCursor, previousPageCursor };\n  }\n}\n\nexport function decodePageCursor(pageCursor?: string): { page: number } {\n  if (!pageCursor) {\n    return { page: 0 };\n  }\n\n  return {\n    page: Number(Buffer.from(pageCursor, 'base64').toString('utf-8')),\n  };\n}\n\nexport function encodePageCursor({ page }: { page: number }): string {\n  return Buffer.from(`${page}`, 'utf-8').toString('base64');\n}\n"],"names":["uuid","DatabaseDocumentStore","PgSearchEngineIndexer"],"mappings":";;;;;;AAqFO,MAAM,cAAA,CAAuC;AAAA;AAAA;AAAA;AAAA,EASlD,WAAA,CACmB,aAAA,EACjB,MAAA,EACA,MAAA,EACA;AAHiB,IAAA,IAAA,CAAA,aAAA,GAAA,aAAA;AAIjB,IAAA,MAAM,UAAUA,OAAA,EAAK;AACrB,IAAA,MAAM,kBAAkB,MAAA,CAAO,iBAAA;AAAA,MAC7B;AAAA,KACF;AAEA,IAAA,MAAM,gBAAA,GAA6C;AAAA,MACjD,MAAA,EAAQ,IAAI,OAAO,CAAA,CAAA,CAAA;AAAA,MACnB,OAAA,EAAS,KAAK,OAAO,CAAA,CAAA,CAAA;AAAA,MACrB,YAAA,EAAc,eAAA,EAAiB,kBAAA,CAAmB,cAAc,CAAA,IAAK,IAAA;AAAA,MACrE,QAAA,EAAU,eAAA,EAAiB,iBAAA,CAAkB,UAAU,CAAA,IAAK,EAAA;AAAA,MAC5D,QAAA,EAAU,eAAA,EAAiB,iBAAA,CAAkB,UAAU,CAAA,IAAK,EAAA;AAAA,MAC5D,SAAA,EAAW,eAAA,EAAiB,iBAAA,CAAkB,WAAW,CAAA,IAAK,CAAA;AAAA,MAC9D,YAAA,EACE,eAAA,EAAiB,kBAAA,CAAmB,cAAc,CAAA,IAAK,KAAA;AAAA,MACzD,YAAA,EAAc,eAAA,EAAiB,iBAAA,CAAkB,cAAc,CAAA,IAAK,CAAA;AAAA,MACpE,iBAAA,EACE,eAAA,EAAiB,iBAAA,CAAkB,mBAAmB,CAAA,IAAK;AAAA,KAC/D;AACA,IAAA,IAAA,CAAK,gBAAA,GAAmB,gBAAA;AACxB,IAAA,IAAA,CAAK,gBAAA,GACH,MAAA,CAAO,iBAAA,CAAkB,4BAA4B,CAAA,IAAK,GAAA;AAE5D,IAAA,IAAA,CAAK,aAAA,GAAgB,IAAA,CAAK,qBAAA,CAAsB,MAAM,CAAA;AACtD,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AAAA,EAChB;AAAA,EArCiB,MAAA;AAAA,EACA,gBAAA;AAAA,EACA,gBAAA;AAAA,EACA,aAAA;AAAA,EAoCT,sBAAsB,MAAA,EAAgB;AAC5C,IAAA,MAAM,mBAAA,GACJ,MAAA,CAAO,WAAA,CAAY,yBAAyB,CAAA,IAAK,CAAA;AACnD,IAAA,IAAI,OAAO,wBAAwB,QAAA,EAAU;AAC3C,MAAA,OAAO,mBAAA;AAAA,IACT,CAAA,MAAA,IAAW,OAAO,mBAAA,KAAwB,QAAA,EAAU;AAClD,MAAA,OAAO,IAAA,CAAK,4BAA4B,mBAAmB,CAAA;AAAA,IAC7D;AACA,IAAA,IAAA,CAAK,MAAA,EAAQ,KAAA;AAAA,MACX,wCAAwC,mBAAmB,CAAA;AAAA,KAC7D;AAEA,IAAA,OAAO,CAAA;AAAA,EACT;AAAA,EAEQ,4BAA4B,UAAA,EAAoB;AACtD,IAAA,MAAM,MAAA,GAAS,WAAW,KAAA,CAAM,GAAG,EAAE,GAAA,CAAI,CAAA,KAAA,KAAS,KAAA,CAAM,IAAA,EAAM,CAAA;AAE9D,IAAA,MAAM,OAAA,GAAU,MAAA,CAAO,GAAA,CAAI,CAAA,KAAA,KAAS;AAClC,MAAA,MAAM,GAAA,GAAM,QAAA,CAAS,KAAA,EAAO,EAAE,CAAA;AAC9B,MAAA,IAAI,KAAA,CAAM,GAAG,CAAA,EAAG;AACd,QAAA,IAAA,CAAK,MAAA,EAAQ,KAAA;AAAA,UACX,yCAAyC,UAAU,CAAA;AAAA,SACrD;AACA,QAAA,OAAO,CAAA;AAAA,MACT;AACA,MAAA,OAAO,GAAA;AAAA,IACT,CAAC,CAAA;AACD,IAAA,OAAO,QAAQ,MAAA,CAAO,CAAC,KAAK,GAAA,KAAQ,GAAA,GAAM,KAAK,CAAC,CAAA;AAAA,EAClD;AAAA;AAAA;AAAA;AAAA,EAKA,aAAa,KAAK,OAAA,EAIU;AAC1B,IAAA,OAAO,IAAI,cAAA;AAAA,MACT,MAAMC,2CAAA,CAAsB,MAAA,CAAO,OAAA,CAAQ,QAAQ,CAAA;AAAA,MACnD,OAAA,CAAQ,MAAA;AAAA,MACR,OAAA,CAAQ;AAAA,KACV;AAAA,EACF;AAAA,EAEA,aAAa,UAAA,CAAW,MAAA,EAAgB,OAAA,EAA0B;AAChE,IAAA,OAAO,IAAI,cAAA;AAAA,MACT,MAAMA,2CAAA,CAAsB,MAAA,CAAO,OAAA,CAAQ,QAAQ,CAAA;AAAA,MACnD,MAAA;AAAA,MACA,OAAA,CAAQ;AAAA,KACV;AAAA,EACF;AAAA,EAEA,aAAa,UAAU,QAAA,EAA6C;AAClE,IAAA,OAAO,MAAMA,2CAAA,CAAsB,SAAA,CAAU,MAAM,QAAA,CAAS,WAAW,CAAA;AAAA,EACzE;AAAA,EAEA,UAAA,CACE,OACA,OAAA,EACuB;AACvB,IAAA,MAAM,QAAA,GAAW,MAAM,SAAA,IAAa,EAAA;AACpC,IAAA,MAAM,EAAE,IAAA,EAAK,GAAI,gBAAA,CAAiB,MAAM,UAAU,CAAA;AAClD,IAAA,MAAM,SAAS,IAAA,GAAO,QAAA;AAEtB,IAAA,MAAM,QAAQ,QAAA,GAAW,CAAA;AACzB,IAAA,MAAM,aAAA,GAAgB,QAAQ,aAAA,IAAiB,CAAA;AAE/C,IAAA,OAAO;AAAA,MACL,OAAA,EAAS;AAAA,QACP,QAAQ,KAAA,CAAM,IAAA,CACX,KAAA,CAAM,IAAI,EACV,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,OAAA,CAAQ,iBAAiB,EAAE,CAAA,CAAE,IAAA,EAAM,EAC9C,MAAA,CAAO,CAAA,CAAA,KAAK,CAAA,KAAM,EAAE,EACpB,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAA,EAAI,IAAA,CAAK,UAAU,CAAC,CAAC,CAAA,GAAA,EAAM,IAAA,CAAK,UAAU,CAAC,CAAC,CAAA,GAAA,CAAK,CAAA,CAC1D,KAAK,GAAG,CAAA;AAAA,QACX,QAAQ,KAAA,CAAM,OAAA;AAAA,QACd,OAAO,KAAA,CAAM,KAAA;AAAA,QACb,MAAA;AAAA,QACA,KAAA;AAAA,QACA,aAAA;AAAA,QACA,SAAS,OAAA,CAAQ;AAAA,OACnB;AAAA,MACA;AAAA,KACF;AAAA,EACF;AAAA,EAEA,cAAc,UAAA,EAAqC;AACjD,IAAA,IAAA,CAAK,UAAA,GAAa,UAAA;AAAA,EACpB;AAAA,EAEA,MAAM,WAAW,IAAA,EAAc;AAC7B,IAAA,OAAO,IAAIC,2CAAA,CAAsB;AAAA,MAC/B,WAAW,IAAA,CAAK,gBAAA;AAAA,MAChB,IAAA;AAAA,MACA,eAAe,IAAA,CAAK,aAAA;AAAA,MACpB,QAAQ,IAAA,CAAK,MAAA,EAAQ,MAAM,EAAE,YAAA,EAAc,MAAM;AAAA,KAClD,CAAA;AAAA,EACH;AAAA,EAEA,MAAM,MAAM,KAAA,EAAiD;AAC3D,IAAA,MAAM,EAAE,OAAA,EAAS,QAAA,EAAS,GAAI,IAAA,CAAK,WAAW,KAAA,EAAO;AAAA,MACnD,kBAAkB,IAAA,CAAK,gBAAA;AAAA,MACvB,eAAe,IAAA,CAAK;AAAA,KACrB,CAAA;AAED,IAAA,MAAM,IAAA,GAAO,MAAM,IAAA,CAAK,aAAA,CAAc,WAAA;AAAA,MAAY,OAAM,EAAA,KACtD,IAAA,CAAK,aAAA,CAAc,KAAA,CAAM,IAAI,OAAO;AAAA,KACtC;AAIA,IAAA,MAAM,EAAE,IAAA,EAAK,GAAI,gBAAA,CAAiB,MAAM,UAAU,CAAA;AAClD,IAAA,MAAM,WAAA,GAAc,KAAK,MAAA,GAAS,QAAA;AAClC,IAAA,MAAM,kBAAkB,IAAA,GAAO,CAAA;AAC/B,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAA;AACvC,IAAA,MAAM,cAAA,GAAiB,cACnB,gBAAA,CAAiB,EAAE,MAAM,IAAA,GAAO,CAAA,EAAG,CAAA,GACnC,MAAA;AACJ,IAAA,MAAM,kBAAA,GAAqB,kBACvB,gBAAA,CAAiB,EAAE,MAAM,IAAA,GAAO,CAAA,EAAG,CAAA,GACnC,MAAA;AAEJ,IAAA,MAAM,UAAU,QAAA,CAAS,GAAA;AAAA,MACvB,CAAC,EAAE,IAAA,EAAM,QAAA,EAAU,SAAA,IAAa,KAAA,MAA4B;AAAA,QAC1D,IAAA;AAAA,QACA,QAAA;AAAA,QACA,IAAA,EAAM,IAAA,GAAO,QAAA,GAAW,KAAA,GAAQ,CAAA;AAAA,QAChC,SAAA,EAAW;AAAA,UACT,MAAA,EAAQ,QAAQ,OAAA,CAAQ,MAAA;AAAA,UACxB,OAAA,EAAS,QAAQ,OAAA,CAAQ,OAAA;AAAA,UACzB,QAAQ,SAAA,GACJ;AAAA,YACE,MAAM,SAAA,CAAU,IAAA;AAAA,YAChB,OAAO,SAAA,CAAU,KAAA;AAAA,YACjB,UAAU,SAAA,CAAU,QAAA;AAAA,YACpB,IAAA,EAAM;AAAA,cAER;AAAC;AACP,OACF;AAAA,KACF;AAEA,IAAA,OAAO,EAAE,OAAA,EAAS,cAAA,EAAgB,kBAAA,EAAmB;AAAA,EACvD;AACF;AAEO,SAAS,iBAAiB,UAAA,EAAuC;AACtE,EAAA,IAAI,CAAC,UAAA,EAAY;AACf,IAAA,OAAO,EAAE,MAAM,CAAA,EAAE;AAAA,EACnB;AAEA,EAAA,OAAO;AAAA,IACL,IAAA,EAAM,OAAO,MAAA,CAAO,IAAA,CAAK,YAAY,QAAQ,CAAA,CAAE,QAAA,CAAS,OAAO,CAAC;AAAA,GAClE;AACF;AAEO,SAAS,gBAAA,CAAiB,EAAE,IAAA,EAAK,EAA6B;AACnE,EAAA,OAAO,MAAA,CAAO,KAAK,CAAA,EAAG,IAAI,IAAI,OAAO,CAAA,CAAE,SAAS,QAAQ,CAAA;AAC1D;;;;;;"}

package/dist/PgSearchEngine/PgSearchEngineIndexer.cjs.js

@@ -44,13 +44,21 @@ class PgSearchEngineIndexer extends pluginSearchBackendNode.BatchSearchEngineInd
       this.tx.rollback();
       return;
     }
-    try {
-      await this.store.completeInsert(this.tx, this.type);
-      this.tx.commit();
-    } catch (e) {
-      this.tx.rollback(e);
-      throw e;
-    }
+    let retryTruncated = false;
+    do {
+      try {
+        await this.store.completeInsert(this.tx, this.type, retryTruncated);
+        this.tx.commit();
+        retryTruncated = false;
+      } catch (e) {
+        this.tx.rollback(e);
+        if (e instanceof Error && e.message.includes("string is too long for tsvector") && !retryTruncated) {
+          retryTruncated = true;
+        } else {
+          throw e;
+        }
+      }
+    } while (retryTruncated);
   }
   /**
    * Custom handler covering the case where an error occurred somewhere else in

package/dist/PgSearchEngine/PgSearchEngineIndexer.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"PgSearchEngineIndexer.cjs.js","sources":["../../src/PgSearchEngine/PgSearchEngineIndexer.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node';\nimport { IndexableDocument } from '@backstage/plugin-search-common';\nimport { Knex } from 'knex';\nimport { DatabaseStore } from '../database';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\n/** @public */\nexport type PgSearchEngineIndexerOptions = {\n  batchSize: number;\n  type: string;\n  databaseStore: DatabaseStore;\n  logger?: LoggerService;\n};\n\n/** @public */\nexport class PgSearchEngineIndexer extends BatchSearchEngineIndexer {\n  private logger?: LoggerService;\n  private store: DatabaseStore;\n  private type: string;\n  private tx: Knex.Transaction | undefined;\n  private numRecords = 0;\n\n  constructor(options: PgSearchEngineIndexerOptions) {\n    super({ batchSize: options.batchSize });\n    this.store = options.databaseStore;\n    this.type = options.type;\n    this.logger = options.logger;\n  }\n\n  async initialize(): Promise<void> {\n    this.tx = await this.store.getTransaction();\n    try {\n      await this.store.prepareInsert(this.tx);\n    } catch (e) {\n      // In case of error, rollback the transaction and re-throw the error so\n      // that the stream can be closed and destroyed properly.\n      this.tx.rollback(e);\n      throw e;\n    }\n  }\n\n  async index(documents: IndexableDocument[]): Promise<void> {\n    this.numRecords += documents.length;\n\n    const refs = [...new Set(documents.map(d => d.authorization?.resourceRef))];\n    this.logger?.debug(\n      `Attempting to index the following entities: ${refs.toString()}`,\n    );\n\n    try {\n      await this.store.insertDocuments(this.tx!, this.type, documents);\n    } catch (e) {\n      // In case of error, rollback the transaction and re-throw the error so\n      // that the stream can be closed and destroyed properly.\n      this.tx!.rollback(e);\n      throw e;\n    }\n  }\n\n  async finalize(): Promise<void> {\n    // If no documents were indexed, rollback the transaction, log a warning,\n    // and do not continue. This ensures that collators that return empty sets\n    // of documents do not cause the index to be deleted.\n    if (this.numRecords === 0) {\n      this.logger?.warn(\n        `Index for ${this.type} was not replaced: indexer received 0 documents`,\n      );\n      this.tx!.rollback!();\n      return;\n    }\n\n    // Attempt to complete and commit the transaction.\n    try {\n      await this.store.completeInsert(this.tx!, this.type);\n      this.tx!.commit();\n    } catch (e) {\n      // Otherwise, rollback the transaction and re-throw the error so that the\n      // stream can be closed and destroyed properly.\n      this.tx!.rollback!(e);\n      throw e;\n    }\n  }\n\n  /**\n   * Custom handler covering the case where an error occurred somewhere else in\n   * the indexing pipeline (e.g. a collator or decorator). In such cases, the\n   * finalize method is not called, which leaves a dangling transaction and\n   * therefore an open connection to PG. This handler ensures we close the\n   * transaction and associated connection.\n   *\n   * todo(@backstage/search-maintainers): Consider introducing a more\n   * formal mechanism for handling such errors in BatchSearchEngineIndexer and\n   * replacing this method with it. See: #17291\n   *\n   * @internal\n   */\n  async _destroy(error: Error | null, done: (error?: Error | null) => void) {\n    // Ignore situations where there was no error.\n    if (!error) {\n      done();\n      return;\n    }\n\n    if (!this.tx!.isCompleted()) {\n      await this.tx!.rollback(error);\n    }\n\n    done(error);\n  }\n}\n"],"names":["BatchSearchEngineIndexer"],"mappings":";;;;AA+BO,MAAM,8BAA8BA,gDAAA,CAAyB;AAAA,EAC1D,MAAA;AAAA,EACA,KAAA;AAAA,EACA,IAAA;AAAA,EACA,EAAA;AAAA,EACA,UAAA,GAAa,CAAA;AAAA,EAErB,YAAY,OAAA,EAAuC;AACjD,IAAA,KAAA,CAAM,EAAE,SAAA,EAAW,OAAA,CAAQ,SAAA,EAAW,CAAA;AACtC,IAAA,IAAA,CAAK,QAAQ,OAAA,CAAQ,aAAA;AACrB,IAAA,IAAA,CAAK,OAAO,OAAA,CAAQ,IAAA;AACpB,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AAAA,EACxB;AAAA,EAEA,MAAM,UAAA,GAA4B;AAChC,IAAA,IAAA,CAAK,EAAA,GAAK,MAAM,IAAA,CAAK,KAAA,CAAM,cAAA,EAAe;AAC1C,IAAA,IAAI;AACF,MAAA,MAAM,IAAA,CAAK,KAAA,CAAM,aAAA,CAAc,IAAA,CAAK,EAAE,CAAA;AAAA,IACxC,SAAS,CAAA,EAAG;AAGV,MAAA,IAAA,CAAK,EAAA,CAAG,SAAS,CAAC,CAAA;AAClB,MAAA,MAAM,CAAA;AAAA,IACR;AAAA,EACF;AAAA,EAEA,MAAM,MAAM,SAAA,EAA+C;AACzD,IAAA,IAAA,CAAK,cAAc,SAAA,CAAU,MAAA;AAE7B,IAAA,MAAM,IAAA,GAAO,CAAC,GAAG,IAAI,GAAA,CAAI,SAAA,CAAU,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,aAAA,EAAe,WAAW,CAAC,CAAC,CAAA;AAC1E,IAAA,IAAA,CAAK,MAAA,EAAQ,KAAA;AAAA,MACX,CAAA,4CAAA,EAA+C,IAAA,CAAK,QAAA,EAAU,CAAA;AAAA,KAChE;AAEA,IAAA,IAAI;AACF,MAAA,MAAM,KAAK,KAAA,CAAM,eAAA,CAAgB,KAAK,EAAA,EAAK,IAAA,CAAK,MAAM,SAAS,CAAA;AAAA,IACjE,SAAS,CAAA,EAAG;AAGV,MAAA,IAAA,CAAK,EAAA,CAAI,SAAS,CAAC,CAAA;AACnB,MAAA,MAAM,CAAA;AAAA,IACR;AAAA,EACF;AAAA,EAEA,MAAM,QAAA,GAA0B;AAI9B,IAAA,IAAI,IAAA,CAAK,eAAe,CAAA,EAAG;AACzB,MAAA,IAAA,CAAK,MAAA,EAAQ,IAAA;AAAA,QACX,CAAA,UAAA,EAAa,KAAK,IAAI,CAAA,+CAAA;AAAA,OACxB;AACA,MAAA,IAAA,CAAK,GAAI,QAAA,EAAU;AACnB,MAAA;AAAA,IACF;AAGA,IAAA,IAAI;AACF,MAAA,MAAM,KAAK,KAAA,CAAM,cAAA,CAAe,IAAA,CAAK,EAAA,EAAK,KAAK,IAAI,CAAA;AACnD,MAAA,IAAA,CAAK,GAAI,MAAA,EAAO;AAAA,IAClB,SAAS,CAAA,EAAG;AAGV,MAAA,IAAA,CAAK,EAAA,CAAI,SAAU,CAAC,CAAA;AACpB,MAAA,MAAM,CAAA;AAAA,IACR;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAeA,MAAM,QAAA,CAAS,KAAA,EAAqB,IAAA,EAAsC;AAExE,IAAA,IAAI,CAAC,KAAA,EAAO;AACV,MAAA,IAAA,EAAK;AACL,MAAA;AAAA,IACF;AAEA,IAAA,IAAI,CAAC,IAAA,CAAK,EAAA,CAAI,WAAA,EAAY,EAAG;AAC3B,MAAA,MAAM,IAAA,CAAK,EAAA,CAAI,QAAA,CAAS,KAAK,CAAA;AAAA,IAC/B;AAEA,IAAA,IAAA,CAAK,KAAK,CAAA;AAAA,EACZ;AACF;;;;"}
+{"version":3,"file":"PgSearchEngineIndexer.cjs.js","sources":["../../src/PgSearchEngine/PgSearchEngineIndexer.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node';\nimport { IndexableDocument } from '@backstage/plugin-search-common';\nimport { Knex } from 'knex';\nimport { DatabaseStore } from '../database';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\n/** @public */\nexport type PgSearchEngineIndexerOptions = {\n  batchSize: number;\n  type: string;\n  databaseStore: DatabaseStore;\n  logger?: LoggerService;\n};\n\n/** @public */\nexport class PgSearchEngineIndexer extends BatchSearchEngineIndexer {\n  private logger?: LoggerService;\n  private store: DatabaseStore;\n  private type: string;\n  private tx: Knex.Transaction | undefined;\n  private numRecords = 0;\n\n  constructor(options: PgSearchEngineIndexerOptions) {\n    super({ batchSize: options.batchSize });\n    this.store = options.databaseStore;\n    this.type = options.type;\n    this.logger = options.logger;\n  }\n\n  async initialize(): Promise<void> {\n    this.tx = await this.store.getTransaction();\n    try {\n      await this.store.prepareInsert(this.tx);\n    } catch (e) {\n      // In case of error, rollback the transaction and re-throw the error so\n      // that the stream can be closed and destroyed properly.\n      this.tx.rollback(e);\n      throw e;\n    }\n  }\n\n  async index(documents: IndexableDocument[]): Promise<void> {\n    this.numRecords += documents.length;\n\n    const refs = [...new Set(documents.map(d => d.authorization?.resourceRef))];\n    this.logger?.debug(\n      `Attempting to index the following entities: ${refs.toString()}`,\n    );\n\n    try {\n      await this.store.insertDocuments(this.tx!, this.type, documents);\n    } catch (e) {\n      // In case of error, rollback the transaction and re-throw the error so\n      // that the stream can be closed and destroyed properly.\n      this.tx!.rollback(e);\n      throw e;\n    }\n  }\n\n  async finalize(): Promise<void> {\n    // If no documents were indexed, rollback the transaction, log a warning,\n    // and do not continue. This ensures that collators that return empty sets\n    // of documents do not cause the index to be deleted.\n    if (this.numRecords === 0) {\n      this.logger?.warn(\n        `Index for ${this.type} was not replaced: indexer received 0 documents`,\n      );\n      this.tx!.rollback!();\n      return;\n    }\n\n    // Do not truncate text by default\n    let retryTruncated = false;\n\n    do {\n      // Attempt to complete and commit the transaction.\n      try {\n        await this.store.completeInsert(this.tx!, this.type, retryTruncated);\n        this.tx!.commit();\n        retryTruncated = false;\n      } catch (e) {\n        // Otherwise, rollback the transaction and re-throw the error so that the\n        // stream can be closed and destroyed properly.\n        this.tx!.rollback!(e);\n\n        if (\n          e instanceof Error &&\n          e.message.includes('string is too long for tsvector') &&\n          !retryTruncated\n        ) {\n          // If the error is due to a string being too long for tsvector, we\n          // retry the operation with truncated text.\n          retryTruncated = true;\n        } else {\n          throw e;\n        }\n      }\n    } while (retryTruncated);\n  }\n\n  /**\n   * Custom handler covering the case where an error occurred somewhere else in\n   * the indexing pipeline (e.g. a collator or decorator). In such cases, the\n   * finalize method is not called, which leaves a dangling transaction and\n   * therefore an open connection to PG. This handler ensures we close the\n   * transaction and associated connection.\n   *\n   * todo(@backstage/search-maintainers): Consider introducing a more\n   * formal mechanism for handling such errors in BatchSearchEngineIndexer and\n   * replacing this method with it. See: #17291\n   *\n   * @internal\n   */\n  async _destroy(error: Error | null, done: (error?: Error | null) => void) {\n    // Ignore situations where there was no error.\n    if (!error) {\n      done();\n      return;\n    }\n\n    if (!this.tx!.isCompleted()) {\n      await this.tx!.rollback(error);\n    }\n\n    done(error);\n  }\n}\n"],"names":["BatchSearchEngineIndexer"],"mappings":";;;;AA+BO,MAAM,8BAA8BA,gDAAA,CAAyB;AAAA,EAC1D,MAAA;AAAA,EACA,KAAA;AAAA,EACA,IAAA;AAAA,EACA,EAAA;AAAA,EACA,UAAA,GAAa,CAAA;AAAA,EAErB,YAAY,OAAA,EAAuC;AACjD,IAAA,KAAA,CAAM,EAAE,SAAA,EAAW,OAAA,CAAQ,SAAA,EAAW,CAAA;AACtC,IAAA,IAAA,CAAK,QAAQ,OAAA,CAAQ,aAAA;AACrB,IAAA,IAAA,CAAK,OAAO,OAAA,CAAQ,IAAA;AACpB,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AAAA,EACxB;AAAA,EAEA,MAAM,UAAA,GAA4B;AAChC,IAAA,IAAA,CAAK,EAAA,GAAK,MAAM,IAAA,CAAK,KAAA,CAAM,cAAA,EAAe;AAC1C,IAAA,IAAI;AACF,MAAA,MAAM,IAAA,CAAK,KAAA,CAAM,aAAA,CAAc,IAAA,CAAK,EAAE,CAAA;AAAA,IACxC,SAAS,CAAA,EAAG;AAGV,MAAA,IAAA,CAAK,EAAA,CAAG,SAAS,CAAC,CAAA;AAClB,MAAA,MAAM,CAAA;AAAA,IACR;AAAA,EACF;AAAA,EAEA,MAAM,MAAM,SAAA,EAA+C;AACzD,IAAA,IAAA,CAAK,cAAc,SAAA,CAAU,MAAA;AAE7B,IAAA,MAAM,IAAA,GAAO,CAAC,GAAG,IAAI,GAAA,CAAI,SAAA,CAAU,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,aAAA,EAAe,WAAW,CAAC,CAAC,CAAA;AAC1E,IAAA,IAAA,CAAK,MAAA,EAAQ,KAAA;AAAA,MACX,CAAA,4CAAA,EAA+C,IAAA,CAAK,QAAA,EAAU,CAAA;AAAA,KAChE;AAEA,IAAA,IAAI;AACF,MAAA,MAAM,KAAK,KAAA,CAAM,eAAA,CAAgB,KAAK,EAAA,EAAK,IAAA,CAAK,MAAM,SAAS,CAAA;AAAA,IACjE,SAAS,CAAA,EAAG;AAGV,MAAA,IAAA,CAAK,EAAA,CAAI,SAAS,CAAC,CAAA;AACnB,MAAA,MAAM,CAAA;AAAA,IACR;AAAA,EACF;AAAA,EAEA,MAAM,QAAA,GAA0B;AAI9B,IAAA,IAAI,IAAA,CAAK,eAAe,CAAA,EAAG;AACzB,MAAA,IAAA,CAAK,MAAA,EAAQ,IAAA;AAAA,QACX,CAAA,UAAA,EAAa,KAAK,IAAI,CAAA,+CAAA;AAAA,OACxB;AACA,MAAA,IAAA,CAAK,GAAI,QAAA,EAAU;AACnB,MAAA;AAAA,IACF;AAGA,IAAA,IAAI,cAAA,GAAiB,KAAA;AAErB,IAAA,GAAG;AAED,MAAA,IAAI;AACF,QAAA,MAAM,KAAK,KAAA,CAAM,cAAA,CAAe,KAAK,EAAA,EAAK,IAAA,CAAK,MAAM,cAAc,CAAA;AACnE,QAAA,IAAA,CAAK,GAAI,MAAA,EAAO;AAChB,QAAA,cAAA,GAAiB,KAAA;AAAA,MACnB,SAAS,CAAA,EAAG;AAGV,QAAA,IAAA,CAAK,EAAA,CAAI,SAAU,CAAC,CAAA;AAEpB,QAAA,IACE,CAAA,YAAa,SACb,CAAA,CAAE,OAAA,CAAQ,SAAS,iCAAiC,CAAA,IACpD,CAAC,cAAA,EACD;AAGA,UAAA,cAAA,GAAiB,IAAA;AAAA,QACnB,CAAA,MAAO;AACL,UAAA,MAAM,CAAA;AAAA,QACR;AAAA,MACF;AAAA,IACF,CAAA,QAAS,cAAA;AAAA,EACX;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAeA,MAAM,QAAA,CAAS,KAAA,EAAqB,IAAA,EAAsC;AAExE,IAAA,IAAI,CAAC,KAAA,EAAO;AACV,MAAA,IAAA,EAAK;AACL,MAAA;AAAA,IACF;AAEA,IAAA,IAAI,CAAC,IAAA,CAAK,EAAA,CAAI,WAAA,EAAY,EAAG;AAC3B,MAAA,MAAM,IAAA,CAAK,EAAA,CAAI,QAAA,CAAS,KAAK,CAAA;AAAA,IAC/B;AAEA,IAAA,IAAA,CAAK,KAAK,CAAA;AAAA,EACZ;AACF;;;;"}

package/dist/database/DatabaseDocumentStore.cjs.js

@@ -51,11 +51,14 @@ class DatabaseDocumentStore {
       "CREATE TEMP TABLE documents_to_insert (type text NOT NULL, document jsonb NOT NULL, hash bytea NOT NULL GENERATED ALWAYS AS (sha256(replace(document::text || type, '\\', '\\\\')::bytea)) STORED) ON COMMIT DROP"
     );
   }
-  async completeInsert(tx, type) {
+  async completeInsert(tx, type, truncate = false) {
+    const documentSelect = truncate ? tx.raw(
+      `jsonb_set(document, '{text}', to_jsonb(LEFT(document->>'text', 50000))) as document`
+    ) : "document";
     await tx.insert(
       tx("documents_to_insert").select(
         "type",
-        "document",
+        documentSelect,
         "hash"
       )
     ).into(tx.raw("documents (type, document, hash)")).onConflict("hash").ignore();
@@ -68,7 +71,10 @@ class DatabaseDocumentStore {
     await tx("documents_to_insert").insert(
       documents.map((document) => ({
         type,
-        document
+        document: {
+          ...document
+          // text: truncateDocsText(document.text),
+        }
       }))
     );
   }

package/dist/database/DatabaseDocumentStore.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"DatabaseDocumentStore.cjs.js","sources":["../../src/database/DatabaseDocumentStore.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nimport {\n  DatabaseService,\n  resolvePackagePath,\n} from '@backstage/backend-plugin-api';\nimport { IndexableDocument } from '@backstage/plugin-search-common';\nimport { Knex } from 'knex';\nimport {\n  DatabaseStore,\n  DocumentResultRow,\n  PgSearchQuery,\n  RawDocumentRow,\n} from './types';\nimport { queryPostgresMajorVersion } from './util';\n\nconst migrationsDir = resolvePackagePath(\n  '@backstage/plugin-search-backend-module-pg',\n  'migrations',\n);\n\n/** @public */\nexport class DatabaseDocumentStore implements DatabaseStore {\n  static async create(\n    database: DatabaseService,\n  ): Promise<DatabaseDocumentStore> {\n    const knex = await database.getClient();\n    try {\n      const majorVersion = await queryPostgresMajorVersion(knex);\n\n      if (majorVersion < 12) {\n        // We are using some features (like generated columns) that aren't\n        // available in older postgres versions.\n        throw new Error(\n          `The PgSearchEngine requires at least postgres version 12 (but is running on ${majorVersion})`,\n        );\n      }\n    } catch {\n      // Actually both mysql and sqlite have a full text search, too. We could\n      // implement them separately or add them here.\n      throw new Error(\n        'The PgSearchEngine is only supported when using a postgres database (>=12.x)',\n      );\n    }\n\n    if (!database.migrations?.skip) {\n      await knex.migrate.latest({\n        directory: migrationsDir,\n      });\n    }\n\n    return new DatabaseDocumentStore(knex);\n  }\n\n  static async supported(knex: Knex): Promise<boolean> {\n    try {\n      const majorVersion = await queryPostgresMajorVersion(knex);\n\n      return majorVersion >= 12;\n    } catch {\n      return false;\n    }\n  }\n\n  constructor(private readonly db: Knex) {}\n\n  async transaction<T>(fn: (tx: Knex.Transaction) => Promise<T>): Promise<T> {\n    return await this.db.transaction(fn);\n  }\n\n  async getTransaction(): Promise<Knex.Transaction> {\n    return this.db.transaction();\n  }\n\n  async prepareInsert(tx: Knex.Transaction): Promise<void> {\n    // We create a temporary table to collect the hashes of the documents that\n    // we expect to be in the documents table at the end. The table is deleted\n    // at the end of the transaction.\n    // The hash makes sure that we generate a new row for every change.\n    await tx.raw(\n      'CREATE TEMP TABLE documents_to_insert (' +\n        'type text NOT NULL, ' +\n        'document jsonb NOT NULL, ' +\n        // Generating the hash requires a trick, as the text to bytea\n        // conversation runs into errors in case the text contains a backslash.\n        // Therefore we have to escape them.\n        \"hash bytea NOT NULL GENERATED ALWAYS AS (sha256(replace(document::text || type, '\\\\', '\\\\\\\\')::bytea)) STORED\" +\n        ') ON COMMIT DROP',\n    );\n  }\n\n  async completeInsert(tx: Knex.Transaction, type: string): Promise<void> {\n    // Copy all new rows into the documents table\n    await tx\n      .insert(\n        tx<RawDocumentRow>('documents_to_insert').select(\n          'type',\n          'document',\n          'hash',\n        ),\n      )\n      .into(tx.raw('documents (type, document, hash)'))\n      .onConflict('hash')\n      .ignore();\n\n    // Delete all documents that we don't expect (deleted and changed)\n    const rowsToDelete = tx<RawDocumentRow>('documents')\n      .select('documents.hash')\n      .leftJoin<RawDocumentRow>('documents_to_insert', {\n        'documents.hash': 'documents_to_insert.hash',\n      })\n      .whereNull('documents_to_insert.hash');\n\n    await tx<RawDocumentRow>('documents')\n      .where({ type })\n      .whereIn('hash', rowsToDelete)\n      .delete();\n  }\n\n  async insertDocuments(\n    tx: Knex.Transaction,\n    type: string,\n    documents: IndexableDocument[],\n  ): Promise<void> {\n    // Insert all documents into the temporary table to process them later\n    await tx<DocumentResultRow>('documents_to_insert').insert(\n      documents.map(document => ({\n        type,\n        document,\n      })),\n    );\n  }\n\n  async query(\n    tx: Knex.Transaction,\n    searchQuery: PgSearchQuery,\n  ): Promise<DocumentResultRow[]> {\n    const {\n      types,\n      pgTerm,\n      fields,\n      offset,\n      limit,\n      normalization = 0,\n      options,\n    } = searchQuery;\n    // TODO(awanlin): We should make the language a parameter so that we can support more then just english\n    // Builds a query like:\n    // SELECT ts_rank_cd(body, query, 0) AS rank, type, document,\n    // ts_headline('english', document, query) AS highlight\n    // FROM documents, to_tsquery('english', 'consent') query\n    // WHERE query @@ body AND (document @> '{\"kind\": \"API\"}')\n    // ORDER BY rank DESC\n    // LIMIT 10;\n    const query = tx<DocumentResultRow>('documents');\n\n    if (pgTerm) {\n      query\n        .from(tx.raw(\"documents, to_tsquery('english', ?) query\", pgTerm))\n        .whereRaw('query @@ body');\n    } else {\n      query.from('documents');\n    }\n\n    if (types) {\n      query.whereIn('type', types);\n    }\n\n    if (fields) {\n      Object.keys(fields).forEach(key => {\n        const value = fields[key];\n        const valueArray = Array.isArray(value) ? value : [value];\n        const fieldValueCompare = valueArray\n          .map(v => ({ [key]: v }))\n          .map(v => JSON.stringify(v));\n        const arrayValueCompare = valueArray\n          .map(v => ({ [key]: [v] }))\n          .map(v => JSON.stringify(v));\n        const valueCompare = [...fieldValueCompare, ...arrayValueCompare];\n        query.whereRaw(\n          `(${valueCompare.map(() => 'document @> ?').join(' OR ')})`,\n          valueCompare,\n        );\n      });\n    }\n\n    query.select('type', 'document');\n\n    if (pgTerm && options.useHighlight) {\n      const headlineOptions = `MaxWords=${options.maxWords}, MinWords=${options.minWords}, ShortWord=${options.shortWord}, HighlightAll=${options.highlightAll}, MaxFragments=${options.maxFragments}, FragmentDelimiter=${options.fragmentDelimiter}, StartSel=${options.preTag}, StopSel=${options.postTag}`;\n      query\n        .select(tx.raw(`ts_rank_cd(body, query, ${normalization}) AS \"rank\"`))\n        .select(\n          tx.raw(\n            `ts_headline(\\'english\\', document, query, '${headlineOptions}') as \"highlight\"`,\n          ),\n        )\n        .orderBy('rank', 'desc');\n    } else if (pgTerm && !options.useHighlight) {\n      query\n        .select(tx.raw(`ts_rank_cd(body, query, ${normalization}) AS \"rank\"`))\n        .orderBy('rank', 'desc');\n    } else {\n      query.select(tx.raw('1 as rank'));\n    }\n\n    return await query.offset(offset).limit(limit);\n  }\n}\n"],"names":["resolvePackagePath","queryPostgresMajorVersion"],"mappings":";;;;;AA6BA,MAAM,aAAA,GAAgBA,mCAAA;AAAA,EACpB,4CAAA;AAAA,EACA;AACF,CAAA;AAGO,MAAM,qBAAA,CAA+C;AAAA,EA0C1D,YAA6B,EAAA,EAAU;AAAV,IAAA,IAAA,CAAA,EAAA,GAAA,EAAA;AAAA,EAAW;AAAA,EAzCxC,aAAa,OACX,QAAA,EACgC;AAChC,IAAA,MAAM,IAAA,GAAO,MAAM,QAAA,CAAS,SAAA,EAAU;AACtC,IAAA,IAAI;AACF,MAAA,MAAM,YAAA,GAAe,MAAMC,8BAAA,CAA0B,IAAI,CAAA;AAEzD,MAAA,IAAI,eAAe,EAAA,EAAI;AAGrB,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,+EAA+E,YAAY,CAAA,CAAA;AAAA,SAC7F;AAAA,MACF;AAAA,IACF,CAAA,CAAA,MAAQ;AAGN,MAAA,MAAM,IAAI,KAAA;AAAA,QACR;AAAA,OACF;AAAA,IACF;AAEA,IAAA,IAAI,CAAC,QAAA,CAAS,UAAA,EAAY,IAAA,EAAM;AAC9B,MAAA,MAAM,IAAA,CAAK,QAAQ,MAAA,CAAO;AAAA,QACxB,SAAA,EAAW;AAAA,OACZ,CAAA;AAAA,IACH;AAEA,IAAA,OAAO,IAAI,sBAAsB,IAAI,CAAA;AAAA,EACvC;AAAA,EAEA,aAAa,UAAU,IAAA,EAA8B;AACnD,IAAA,IAAI;AACF,MAAA,MAAM,YAAA,GAAe,MAAMA,8BAAA,CAA0B,IAAI,CAAA;AAEzD,MAAA,OAAO,YAAA,IAAgB,EAAA;AAAA,IACzB,CAAA,CAAA,MAAQ;AACN,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAAA,EAIA,MAAM,YAAe,EAAA,EAAsD;AACzE,IAAA,OAAO,MAAM,IAAA,CAAK,EAAA,CAAG,WAAA,CAAY,EAAE,CAAA;AAAA,EACrC;AAAA,EAEA,MAAM,cAAA,GAA4C;AAChD,IAAA,OAAO,IAAA,CAAK,GAAG,WAAA,EAAY;AAAA,EAC7B;AAAA,EAEA,MAAM,cAAc,EAAA,EAAqC;AAKvD,IAAA,MAAM,EAAA,CAAG,GAAA;AAAA,MACP;AAAA,KAQF;AAAA,EACF;AAAA,EAEA,MAAM,cAAA,CAAe,EAAA,EAAsB,IAAA,EAA6B;AAEtE,IAAA,MAAM,EAAA,CACH,MAAA;AAAA,MACC,EAAA,CAAmB,qBAAqB,CAAA,CAAE,MAAA;AAAA,QACxC,MAAA;AAAA,QACA,UAAA;AAAA,QACA;AAAA;AACF,KACF,CACC,IAAA,CAAK,EAAA,CAAG,GAAA,CAAI,kCAAkC,CAAC,CAAA,CAC/C,UAAA,CAAW,MAAM,CAAA,CACjB,MAAA,EAAO;AAGV,IAAA,MAAM,YAAA,GAAe,GAAmB,WAAW,CAAA,CAChD,OAAO,gBAAgB,CAAA,CACvB,SAAyB,qBAAA,EAAuB;AAAA,MAC/C,gBAAA,EAAkB;AAAA,KACnB,CAAA,CACA,SAAA,CAAU,0BAA0B,CAAA;AAEvC,IAAA,MAAM,EAAA,CAAmB,WAAW,CAAA,CACjC,KAAA,CAAM,EAAE,IAAA,EAAM,CAAA,CACd,OAAA,CAAQ,MAAA,EAAQ,YAAY,CAAA,CAC5B,MAAA,EAAO;AAAA,EACZ;AAAA,EAEA,MAAM,eAAA,CACJ,EAAA,EACA,IAAA,EACA,SAAA,EACe;AAEf,IAAA,MAAM,EAAA,CAAsB,qBAAqB,CAAA,CAAE,MAAA;AAAA,MACjD,SAAA,CAAU,IAAI,CAAA,QAAA,MAAa;AAAA,QACzB,IAAA;AAAA,QACA;AAAA,OACF,CAAE;AAAA,KACJ;AAAA,EACF;AAAA,EAEA,MAAM,KAAA,CACJ,EAAA,EACA,WAAA,EAC8B;AAC9B,IAAA,MAAM;AAAA,MACJ,KAAA;AAAA,MACA,MAAA;AAAA,MACA,MAAA;AAAA,MACA,MAAA;AAAA,MACA,KAAA;AAAA,MACA,aAAA,GAAgB,CAAA;AAAA,MAChB;AAAA,KACF,GAAI,WAAA;AASJ,IAAA,MAAM,KAAA,GAAQ,GAAsB,WAAW,CAAA;AAE/C,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,KAAA,CACG,IAAA,CAAK,GAAG,GAAA,CAAI,2CAAA,EAA6C,MAAM,CAAC,CAAA,CAChE,SAAS,eAAe,CAAA;AAAA,IAC7B,CAAA,MAAO;AACL,MAAA,KAAA,CAAM,KAAK,WAAW,CAAA;AAAA,IACxB;AAEA,IAAA,IAAI,KAAA,EAAO;AACT,MAAA,KAAA,CAAM,OAAA,CAAQ,QAAQ,KAAK,CAAA;AAAA,IAC7B;AAEA,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,MAAA,CAAO,IAAA,CAAK,MAAM,CAAA,CAAE,OAAA,CAAQ,CAAA,GAAA,KAAO;AACjC,QAAA,MAAM,KAAA,GAAQ,OAAO,GAAG,CAAA;AACxB,QAAA,MAAM,aAAa,KAAA,CAAM,OAAA,CAAQ,KAAK,CAAA,GAAI,KAAA,GAAQ,CAAC,KAAK,CAAA;AACxD,QAAA,MAAM,oBAAoB,UAAA,CACvB,GAAA,CAAI,CAAA,CAAA,MAAM,EAAE,CAAC,GAAG,GAAG,CAAA,EAAE,CAAE,EACvB,GAAA,CAAI,CAAA,CAAA,KAAK,IAAA,CAAK,SAAA,CAAU,CAAC,CAAC,CAAA;AAC7B,QAAA,MAAM,oBAAoB,UAAA,CACvB,GAAA,CAAI,QAAM,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAA,GAAI,CAAA,CACzB,GAAA,CAAI,OAAK,IAAA,CAAK,SAAA,CAAU,CAAC,CAAC,CAAA;AAC7B,QAAA,MAAM,YAAA,GAAe,CAAC,GAAG,iBAAA,EAAmB,GAAG,iBAAiB,CAAA;AAChE,QAAA,KAAA,CAAM,QAAA;AAAA,UACJ,CAAA,CAAA,EAAI,aAAa,GAAA,CAAI,MAAM,eAAe,CAAA,CAAE,IAAA,CAAK,MAAM,CAAC,CAAA,CAAA,CAAA;AAAA,UACxD;AAAA,SACF;AAAA,MACF,CAAC,CAAA;AAAA,IACH;AAEA,IAAA,KAAA,CAAM,MAAA,CAAO,QAAQ,UAAU,CAAA;AAE/B,IAAA,IAAI,MAAA,IAAU,QAAQ,YAAA,EAAc;AAClC,MAAA,MAAM,eAAA,GAAkB,CAAA,SAAA,EAAY,OAAA,CAAQ,QAAQ,CAAA,WAAA,EAAc,QAAQ,QAAQ,CAAA,YAAA,EAAe,OAAA,CAAQ,SAAS,CAAA,eAAA,EAAkB,OAAA,CAAQ,YAAY,CAAA,eAAA,EAAkB,OAAA,CAAQ,YAAY,CAAA,oBAAA,EAAuB,OAAA,CAAQ,iBAAiB,cAAc,OAAA,CAAQ,MAAM,CAAA,UAAA,EAAa,OAAA,CAAQ,OAAO,CAAA,CAAA;AACtS,MAAA,KAAA,CACG,OAAO,EAAA,CAAG,GAAA,CAAI,2BAA2B,aAAa,CAAA,WAAA,CAAa,CAAC,CAAA,CACpE,MAAA;AAAA,QACC,EAAA,CAAG,GAAA;AAAA,UACD,4CAA8C,eAAe,CAAA,iBAAA;AAAA;AAC/D,OACF,CACC,OAAA,CAAQ,MAAA,EAAQ,MAAM,CAAA;AAAA,IAC3B,CAAA,MAAA,IAAW,MAAA,IAAU,CAAC,OAAA,CAAQ,YAAA,EAAc;AAC1C,MAAA,KAAA,CACG,MAAA,CAAO,EAAA,CAAG,GAAA,CAAI,CAAA,wBAAA,EAA2B,aAAa,aAAa,CAAC,CAAA,CACpE,OAAA,CAAQ,MAAA,EAAQ,MAAM,CAAA;AAAA,IAC3B,CAAA,MAAO;AACL,MAAA,KAAA,CAAM,MAAA,CAAO,EAAA,CAAG,GAAA,CAAI,WAAW,CAAC,CAAA;AAAA,IAClC;AAEA,IAAA,OAAO,MAAM,KAAA,CAAM,MAAA,CAAO,MAAM,CAAA,CAAE,MAAM,KAAK,CAAA;AAAA,EAC/C;AACF;;;;"}
+{"version":3,"file":"DatabaseDocumentStore.cjs.js","sources":["../../src/database/DatabaseDocumentStore.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nimport {\n  DatabaseService,\n  resolvePackagePath,\n} from '@backstage/backend-plugin-api';\nimport { IndexableDocument } from '@backstage/plugin-search-common';\nimport { Knex } from 'knex';\nimport {\n  DatabaseStore,\n  DocumentResultRow,\n  PgSearchQuery,\n  RawDocumentRow,\n} from './types';\nimport { queryPostgresMajorVersion } from './util';\n\nconst migrationsDir = resolvePackagePath(\n  '@backstage/plugin-search-backend-module-pg',\n  'migrations',\n);\n\n/** @public */\nexport class DatabaseDocumentStore implements DatabaseStore {\n  static async create(\n    database: DatabaseService,\n  ): Promise<DatabaseDocumentStore> {\n    const knex = await database.getClient();\n    try {\n      const majorVersion = await queryPostgresMajorVersion(knex);\n\n      if (majorVersion < 12) {\n        // We are using some features (like generated columns) that aren't\n        // available in older postgres versions.\n        throw new Error(\n          `The PgSearchEngine requires at least postgres version 12 (but is running on ${majorVersion})`,\n        );\n      }\n    } catch {\n      // Actually both mysql and sqlite have a full text search, too. We could\n      // implement them separately or add them here.\n      throw new Error(\n        'The PgSearchEngine is only supported when using a postgres database (>=12.x)',\n      );\n    }\n\n    if (!database.migrations?.skip) {\n      await knex.migrate.latest({\n        directory: migrationsDir,\n      });\n    }\n\n    return new DatabaseDocumentStore(knex);\n  }\n\n  static async supported(knex: Knex): Promise<boolean> {\n    try {\n      const majorVersion = await queryPostgresMajorVersion(knex);\n\n      return majorVersion >= 12;\n    } catch {\n      return false;\n    }\n  }\n\n  constructor(private readonly db: Knex) {}\n\n  async transaction<T>(fn: (tx: Knex.Transaction) => Promise<T>): Promise<T> {\n    return await this.db.transaction(fn);\n  }\n\n  async getTransaction(): Promise<Knex.Transaction> {\n    return this.db.transaction();\n  }\n\n  async prepareInsert(tx: Knex.Transaction): Promise<void> {\n    // We create a temporary table to collect the hashes of the documents that\n    // we expect to be in the documents table at the end. The table is deleted\n    // at the end of the transaction.\n    // The hash makes sure that we generate a new row for every change.\n    await tx.raw(\n      'CREATE TEMP TABLE documents_to_insert (' +\n        'type text NOT NULL, ' +\n        'document jsonb NOT NULL, ' +\n        // Generating the hash requires a trick, as the text to bytea\n        // conversation runs into errors in case the text contains a backslash.\n        // Therefore we have to escape them.\n        \"hash bytea NOT NULL GENERATED ALWAYS AS (sha256(replace(document::text || type, '\\\\', '\\\\\\\\')::bytea)) STORED\" +\n        ') ON COMMIT DROP',\n    );\n  }\n\n  async completeInsert(\n    tx: Knex.Transaction,\n    type: string,\n    truncate = false,\n  ): Promise<void> {\n    const documentSelect = truncate\n      ? tx.raw(\n          `jsonb_set(document, '{text}', to_jsonb(LEFT(document->>'text', 50000))) as document`,\n        )\n      : 'document';\n\n    // Copy all new rows into the documents table\n    await tx\n      .insert(\n        tx<RawDocumentRow>('documents_to_insert').select(\n          'type',\n          documentSelect,\n          'hash',\n        ),\n      )\n      .into(tx.raw('documents (type, document, hash)'))\n      .onConflict('hash')\n      .ignore();\n\n    // Delete all documents that we don't expect (deleted and changed)\n    const rowsToDelete = tx<RawDocumentRow>('documents')\n      .select('documents.hash')\n      .leftJoin<RawDocumentRow>('documents_to_insert', {\n        'documents.hash': 'documents_to_insert.hash',\n      })\n      .whereNull('documents_to_insert.hash');\n\n    await tx<RawDocumentRow>('documents')\n      .where({ type })\n      .whereIn('hash', rowsToDelete)\n      .delete();\n  }\n\n  async insertDocuments(\n    tx: Knex.Transaction,\n    type: string,\n    documents: IndexableDocument[],\n  ): Promise<void> {\n    // Insert all documents into the temporary table to process them later\n    await tx<DocumentResultRow>('documents_to_insert').insert(\n      documents.map(document => ({\n        type,\n        document: {\n          ...document,\n          // text: truncateDocsText(document.text),\n        },\n      })),\n    );\n  }\n\n  async query(\n    tx: Knex.Transaction,\n    searchQuery: PgSearchQuery,\n  ): Promise<DocumentResultRow[]> {\n    const {\n      types,\n      pgTerm,\n      fields,\n      offset,\n      limit,\n      normalization = 0,\n      options,\n    } = searchQuery;\n    // TODO(awanlin): We should make the language a parameter so that we can support more then just english\n    // Builds a query like:\n    // SELECT ts_rank_cd(body, query, 0) AS rank, type, document,\n    // ts_headline('english', document, query) AS highlight\n    // FROM documents, to_tsquery('english', 'consent') query\n    // WHERE query @@ body AND (document @> '{\"kind\": \"API\"}')\n    // ORDER BY rank DESC\n    // LIMIT 10;\n    const query = tx<DocumentResultRow>('documents');\n\n    if (pgTerm) {\n      query\n        .from(tx.raw(\"documents, to_tsquery('english', ?) query\", pgTerm))\n        .whereRaw('query @@ body');\n    } else {\n      query.from('documents');\n    }\n\n    if (types) {\n      query.whereIn('type', types);\n    }\n\n    if (fields) {\n      Object.keys(fields).forEach(key => {\n        const value = fields[key];\n        const valueArray = Array.isArray(value) ? value : [value];\n        const fieldValueCompare = valueArray\n          .map(v => ({ [key]: v }))\n          .map(v => JSON.stringify(v));\n        const arrayValueCompare = valueArray\n          .map(v => ({ [key]: [v] }))\n          .map(v => JSON.stringify(v));\n        const valueCompare = [...fieldValueCompare, ...arrayValueCompare];\n        query.whereRaw(\n          `(${valueCompare.map(() => 'document @> ?').join(' OR ')})`,\n          valueCompare,\n        );\n      });\n    }\n\n    query.select('type', 'document');\n\n    if (pgTerm && options.useHighlight) {\n      const headlineOptions = `MaxWords=${options.maxWords}, MinWords=${options.minWords}, ShortWord=${options.shortWord}, HighlightAll=${options.highlightAll}, MaxFragments=${options.maxFragments}, FragmentDelimiter=${options.fragmentDelimiter}, StartSel=${options.preTag}, StopSel=${options.postTag}`;\n      query\n        .select(tx.raw(`ts_rank_cd(body, query, ${normalization}) AS \"rank\"`))\n        .select(\n          tx.raw(\n            `ts_headline(\\'english\\', document, query, '${headlineOptions}') as \"highlight\"`,\n          ),\n        )\n        .orderBy('rank', 'desc');\n    } else if (pgTerm && !options.useHighlight) {\n      query\n        .select(tx.raw(`ts_rank_cd(body, query, ${normalization}) AS \"rank\"`))\n        .orderBy('rank', 'desc');\n    } else {\n      query.select(tx.raw('1 as rank'));\n    }\n\n    return await query.offset(offset).limit(limit);\n  }\n}\n"],"names":["resolvePackagePath","queryPostgresMajorVersion"],"mappings":";;;;;AA6BA,MAAM,aAAA,GAAgBA,mCAAA;AAAA,EACpB,4CAAA;AAAA,EACA;AACF,CAAA;AAGO,MAAM,qBAAA,CAA+C;AAAA,EA0C1D,YAA6B,EAAA,EAAU;AAAV,IAAA,IAAA,CAAA,EAAA,GAAA,EAAA;AAAA,EAAW;AAAA,EAzCxC,aAAa,OACX,QAAA,EACgC;AAChC,IAAA,MAAM,IAAA,GAAO,MAAM,QAAA,CAAS,SAAA,EAAU;AACtC,IAAA,IAAI;AACF,MAAA,MAAM,YAAA,GAAe,MAAMC,8BAAA,CAA0B,IAAI,CAAA;AAEzD,MAAA,IAAI,eAAe,EAAA,EAAI;AAGrB,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,+EAA+E,YAAY,CAAA,CAAA;AAAA,SAC7F;AAAA,MACF;AAAA,IACF,CAAA,CAAA,MAAQ;AAGN,MAAA,MAAM,IAAI,KAAA;AAAA,QACR;AAAA,OACF;AAAA,IACF;AAEA,IAAA,IAAI,CAAC,QAAA,CAAS,UAAA,EAAY,IAAA,EAAM;AAC9B,MAAA,MAAM,IAAA,CAAK,QAAQ,MAAA,CAAO;AAAA,QACxB,SAAA,EAAW;AAAA,OACZ,CAAA;AAAA,IACH;AAEA,IAAA,OAAO,IAAI,sBAAsB,IAAI,CAAA;AAAA,EACvC;AAAA,EAEA,aAAa,UAAU,IAAA,EAA8B;AACnD,IAAA,IAAI;AACF,MAAA,MAAM,YAAA,GAAe,MAAMA,8BAAA,CAA0B,IAAI,CAAA;AAEzD,MAAA,OAAO,YAAA,IAAgB,EAAA;AAAA,IACzB,CAAA,CAAA,MAAQ;AACN,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAAA,EAIA,MAAM,YAAe,EAAA,EAAsD;AACzE,IAAA,OAAO,MAAM,IAAA,CAAK,EAAA,CAAG,WAAA,CAAY,EAAE,CAAA;AAAA,EACrC;AAAA,EAEA,MAAM,cAAA,GAA4C;AAChD,IAAA,OAAO,IAAA,CAAK,GAAG,WAAA,EAAY;AAAA,EAC7B;AAAA,EAEA,MAAM,cAAc,EAAA,EAAqC;AAKvD,IAAA,MAAM,EAAA,CAAG,GAAA;AAAA,MACP;AAAA,KAQF;AAAA,EACF;AAAA,EAEA,MAAM,cAAA,CACJ,EAAA,EACA,IAAA,EACA,WAAW,KAAA,EACI;AACf,IAAA,MAAM,cAAA,GAAiB,WACnB,EAAA,CAAG,GAAA;AAAA,MACD,CAAA,mFAAA;AAAA,KACF,GACA,UAAA;AAGJ,IAAA,MAAM,EAAA,CACH,MAAA;AAAA,MACC,EAAA,CAAmB,qBAAqB,CAAA,CAAE,MAAA;AAAA,QACxC,MAAA;AAAA,QACA,cAAA;AAAA,QACA;AAAA;AACF,KACF,CACC,IAAA,CAAK,EAAA,CAAG,GAAA,CAAI,kCAAkC,CAAC,CAAA,CAC/C,UAAA,CAAW,MAAM,CAAA,CACjB,MAAA,EAAO;AAGV,IAAA,MAAM,YAAA,GAAe,GAAmB,WAAW,CAAA,CAChD,OAAO,gBAAgB,CAAA,CACvB,SAAyB,qBAAA,EAAuB;AAAA,MAC/C,gBAAA,EAAkB;AAAA,KACnB,CAAA,CACA,SAAA,CAAU,0BAA0B,CAAA;AAEvC,IAAA,MAAM,EAAA,CAAmB,WAAW,CAAA,CACjC,KAAA,CAAM,EAAE,IAAA,EAAM,CAAA,CACd,OAAA,CAAQ,MAAA,EAAQ,YAAY,CAAA,CAC5B,MAAA,EAAO;AAAA,EACZ;AAAA,EAEA,MAAM,eAAA,CACJ,EAAA,EACA,IAAA,EACA,SAAA,EACe;AAEf,IAAA,MAAM,EAAA,CAAsB,qBAAqB,CAAA,CAAE,MAAA;AAAA,MACjD,SAAA,CAAU,IAAI,CAAA,QAAA,MAAa;AAAA,QACzB,IAAA;AAAA,QACA,QAAA,EAAU;AAAA,UACR,GAAG;AAAA;AAAA;AAEL,OACF,CAAE;AAAA,KACJ;AAAA,EACF;AAAA,EAEA,MAAM,KAAA,CACJ,EAAA,EACA,WAAA,EAC8B;AAC9B,IAAA,MAAM;AAAA,MACJ,KAAA;AAAA,MACA,MAAA;AAAA,MACA,MAAA;AAAA,MACA,MAAA;AAAA,MACA,KAAA;AAAA,MACA,aAAA,GAAgB,CAAA;AAAA,MAChB;AAAA,KACF,GAAI,WAAA;AASJ,IAAA,MAAM,KAAA,GAAQ,GAAsB,WAAW,CAAA;AAE/C,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,KAAA,CACG,IAAA,CAAK,GAAG,GAAA,CAAI,2CAAA,EAA6C,MAAM,CAAC,CAAA,CAChE,SAAS,eAAe,CAAA;AAAA,IAC7B,CAAA,MAAO;AACL,MAAA,KAAA,CAAM,KAAK,WAAW,CAAA;AAAA,IACxB;AAEA,IAAA,IAAI,KAAA,EAAO;AACT,MAAA,KAAA,CAAM,OAAA,CAAQ,QAAQ,KAAK,CAAA;AAAA,IAC7B;AAEA,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,MAAA,CAAO,IAAA,CAAK,MAAM,CAAA,CAAE,OAAA,CAAQ,CAAA,GAAA,KAAO;AACjC,QAAA,MAAM,KAAA,GAAQ,OAAO,GAAG,CAAA;AACxB,QAAA,MAAM,aAAa,KAAA,CAAM,OAAA,CAAQ,KAAK,CAAA,GAAI,KAAA,GAAQ,CAAC,KAAK,CAAA;AACxD,QAAA,MAAM,oBAAoB,UAAA,CACvB,GAAA,CAAI,CAAA,CAAA,MAAM,EAAE,CAAC,GAAG,GAAG,CAAA,EAAE,CAAE,EACvB,GAAA,CAAI,CAAA,CAAA,KAAK,IAAA,CAAK,SAAA,CAAU,CAAC,CAAC,CAAA;AAC7B,QAAA,MAAM,oBAAoB,UAAA,CACvB,GAAA,CAAI,QAAM,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAA,GAAI,CAAA,CACzB,GAAA,CAAI,OAAK,IAAA,CAAK,SAAA,CAAU,CAAC,CAAC,CAAA;AAC7B,QAAA,MAAM,YAAA,GAAe,CAAC,GAAG,iBAAA,EAAmB,GAAG,iBAAiB,CAAA;AAChE,QAAA,KAAA,CAAM,QAAA;AAAA,UACJ,CAAA,CAAA,EAAI,aAAa,GAAA,CAAI,MAAM,eAAe,CAAA,CAAE,IAAA,CAAK,MAAM,CAAC,CAAA,CAAA,CAAA;AAAA,UACxD;AAAA,SACF;AAAA,MACF,CAAC,CAAA;AAAA,IACH;AAEA,IAAA,KAAA,CAAM,MAAA,CAAO,QAAQ,UAAU,CAAA;AAE/B,IAAA,IAAI,MAAA,IAAU,QAAQ,YAAA,EAAc;AAClC,MAAA,MAAM,eAAA,GAAkB,CAAA,SAAA,EAAY,OAAA,CAAQ,QAAQ,CAAA,WAAA,EAAc,QAAQ,QAAQ,CAAA,YAAA,EAAe,OAAA,CAAQ,SAAS,CAAA,eAAA,EAAkB,OAAA,CAAQ,YAAY,CAAA,eAAA,EAAkB,OAAA,CAAQ,YAAY,CAAA,oBAAA,EAAuB,OAAA,CAAQ,iBAAiB,cAAc,OAAA,CAAQ,MAAM,CAAA,UAAA,EAAa,OAAA,CAAQ,OAAO,CAAA,CAAA;AACtS,MAAA,KAAA,CACG,OAAO,EAAA,CAAG,GAAA,CAAI,2BAA2B,aAAa,CAAA,WAAA,CAAa,CAAC,CAAA,CACpE,MAAA;AAAA,QACC,EAAA,CAAG,GAAA;AAAA,UACD,4CAA8C,eAAe,CAAA,iBAAA;AAAA;AAC/D,OACF,CACC,OAAA,CAAQ,MAAA,EAAQ,MAAM,CAAA;AAAA,IAC3B,CAAA,MAAA,IAAW,MAAA,IAAU,CAAC,OAAA,CAAQ,YAAA,EAAc;AAC1C,MAAA,KAAA,CACG,MAAA,CAAO,EAAA,CAAG,GAAA,CAAI,CAAA,wBAAA,EAA2B,aAAa,aAAa,CAAC,CAAA,CACpE,OAAA,CAAQ,MAAA,EAAQ,MAAM,CAAA;AAAA,IAC3B,CAAA,MAAO;AACL,MAAA,KAAA,CAAM,MAAA,CAAO,EAAA,CAAG,GAAA,CAAI,WAAW,CAAC,CAAA;AAAA,IAClC;AAEA,IAAA,OAAO,MAAM,KAAA,CAAM,MAAA,CAAO,MAAM,CAAA,CAAE,MAAM,KAAK,CAAA;AAAA,EAC/C;AACF;;;;"}

package/dist/index.d.ts

@@ -120,7 +120,7 @@ interface DatabaseStore {
     getTransaction(): Promise<Knex.Transaction>;
     prepareInsert(tx: Knex.Transaction): Promise<void>;
     insertDocuments(tx: Knex.Transaction, type: string, documents: IndexableDocument[]): Promise<void>;
-    completeInsert(tx: Knex.Transaction, type: string): Promise<void>;
+    completeInsert(tx: Knex.Transaction, type: string, truncate?: boolean): Promise<void>;
     query(tx: Knex.Transaction, pgQuery: PgSearchQuery): Promise<DocumentResultRow[]>;
 }
 /** @public */
@@ -145,7 +145,7 @@ declare class DatabaseDocumentStore implements DatabaseStore {
     transaction<T>(fn: (tx: Knex.Transaction) => Promise<T>): Promise<T>;
     getTransaction(): Promise<Knex.Transaction>;
     prepareInsert(tx: Knex.Transaction): Promise<void>;
-    completeInsert(tx: Knex.Transaction, type: string): Promise<void>;
+    completeInsert(tx: Knex.Transaction, type: string, truncate?: boolean): Promise<void>;
     insertDocuments(tx: Knex.Transaction, type: string, documents: IndexableDocument[]): Promise<void>;
     query(tx: Knex.Transaction, searchQuery: PgSearchQuery): Promise<DocumentResultRow[]>;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-search-backend-module-pg",
-  "version": "0.5.48-next.0",
+  "version": "0.5.49-next.0",
   "description": "A module for the search backend that implements search using PostgreSQL",
   "backstage": {
     "role": "backend-plugin-module",
@@ -63,17 +63,17 @@
     "test": "backstage-cli package test"
   },
   "dependencies": {
-    "@backstage/backend-plugin-api": "1.4.3-next.0",
+    "@backstage/backend-plugin-api": "1.4.3",
     "@backstage/config": "1.3.3",
-    "@backstage/plugin-search-backend-node": "1.3.15-next.0",
+    "@backstage/plugin-search-backend-node": "1.3.15",
     "@backstage/plugin-search-common": "1.2.19",
     "knex": "^3.0.0",
     "lodash": "^4.17.21",
     "uuid": "^11.0.0"
   },
   "devDependencies": {
-    "@backstage/backend-test-utils": "1.9.0-next.1",
-    "@backstage/cli": "0.34.2-next.1"
+    "@backstage/backend-test-utils": "1.9.1-next.0",
+    "@backstage/cli": "0.34.4-next.0"
   },
   "configSchema": "config.d.ts"
 }