@backstage/plugin-search-backend-module-pg 0.5.54-next.2 → 0.5.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
# @backstage/plugin-search-backend-module-pg
|
|
2
2
|
|
|
3
|
+
## 0.5.54
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- aa08b7f: Fix a bug in large document indexing logic by using sub-transaction rollbacks
|
|
8
|
+
- Updated dependencies
|
|
9
|
+
- @backstage/backend-plugin-api@1.9.0
|
|
10
|
+
- @backstage/plugin-search-backend-node@1.4.3
|
|
11
|
+
- @backstage/config@1.3.7
|
|
12
|
+
- @backstage/plugin-search-common@1.2.23
|
|
13
|
+
|
|
3
14
|
## 0.5.54-next.2
|
|
4
15
|
|
|
5
16
|
### Patch Changes
|
|
@@ -46,15 +46,24 @@ class PgSearchEngineIndexer extends pluginSearchBackendNode.BatchSearchEngineInd
|
|
|
46
46
|
}
|
|
47
47
|
let retryTruncated = false;
|
|
48
48
|
do {
|
|
49
|
+
let subTx;
|
|
49
50
|
try {
|
|
50
|
-
await this.
|
|
51
|
-
this.tx.commit();
|
|
52
|
-
retryTruncated = false;
|
|
51
|
+
subTx = await this.tx.transaction();
|
|
53
52
|
} catch (e) {
|
|
54
53
|
this.tx.rollback(e);
|
|
54
|
+
throw e;
|
|
55
|
+
}
|
|
56
|
+
try {
|
|
57
|
+
await this.store.completeInsert(subTx, this.type, retryTruncated);
|
|
58
|
+
await subTx.commit();
|
|
59
|
+
await this.tx.commit();
|
|
60
|
+
retryTruncated = false;
|
|
61
|
+
} catch (e) {
|
|
62
|
+
await subTx.rollback(e);
|
|
55
63
|
if (e instanceof Error && e.message.includes("string is too long for tsvector") && !retryTruncated) {
|
|
56
64
|
retryTruncated = true;
|
|
57
65
|
} else {
|
|
66
|
+
this.tx.rollback(e);
|
|
58
67
|
throw e;
|
|
59
68
|
}
|
|
60
69
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PgSearchEngineIndexer.cjs.js","sources":["../../src/PgSearchEngine/PgSearchEngineIndexer.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node';\nimport { IndexableDocument } from '@backstage/plugin-search-common';\nimport { Knex } from 'knex';\nimport { DatabaseStore } from '../database';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\n/** @public */\nexport type PgSearchEngineIndexerOptions = {\n batchSize: number;\n type: string;\n databaseStore: DatabaseStore;\n logger?: LoggerService;\n};\n\n/** @public */\nexport class PgSearchEngineIndexer extends BatchSearchEngineIndexer {\n private logger?: LoggerService;\n private store: DatabaseStore;\n private type: string;\n private tx: Knex.Transaction | undefined;\n private numRecords = 0;\n\n constructor(options: PgSearchEngineIndexerOptions) {\n super({ batchSize: options.batchSize });\n this.store = options.databaseStore;\n this.type = options.type;\n this.logger = options.logger;\n }\n\n async initialize(): Promise<void> {\n this.tx = await this.store.getTransaction();\n try {\n await this.store.prepareInsert(this.tx);\n } catch (e) {\n // In case of error, rollback the transaction and re-throw the error so\n // that the stream can be closed and destroyed properly.\n this.tx.rollback(e);\n throw e;\n }\n }\n\n async index(documents: IndexableDocument[]): Promise<void> {\n this.numRecords += documents.length;\n\n const refs = [...new Set(documents.map(d => d.authorization?.resourceRef))];\n this.logger?.debug(\n `Attempting to index the following entities: ${refs.toString()}`,\n );\n\n try {\n await this.store.insertDocuments(this.tx!, this.type, documents);\n } catch (e) {\n // In case of error, rollback the transaction and re-throw the error so\n // that the stream can be closed and destroyed properly.\n this.tx!.rollback(e);\n throw e;\n }\n }\n\n async finalize(): Promise<void> {\n // If no documents were indexed, rollback the transaction, log a warning,\n // and do not continue. This ensures that collators that return empty sets\n // of documents do not cause the index to be deleted.\n if (this.numRecords === 0) {\n this.logger?.warn(\n `Index for ${this.type} was not replaced: indexer received 0 documents`,\n );\n this.tx!.rollback!();\n return;\n }\n\n // Do not truncate text by default\n let retryTruncated = false;\n\n do {\n //
|
|
1
|
+
{"version":3,"file":"PgSearchEngineIndexer.cjs.js","sources":["../../src/PgSearchEngine/PgSearchEngineIndexer.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node';\nimport { IndexableDocument } from '@backstage/plugin-search-common';\nimport { Knex } from 'knex';\nimport { DatabaseStore } from '../database';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\n/** @public */\nexport type PgSearchEngineIndexerOptions = {\n batchSize: number;\n type: string;\n databaseStore: DatabaseStore;\n logger?: LoggerService;\n};\n\n/** @public */\nexport class PgSearchEngineIndexer extends BatchSearchEngineIndexer {\n private logger?: LoggerService;\n private store: DatabaseStore;\n private type: string;\n private tx: Knex.Transaction | undefined;\n private numRecords = 0;\n\n constructor(options: PgSearchEngineIndexerOptions) {\n super({ batchSize: options.batchSize });\n this.store = options.databaseStore;\n this.type = options.type;\n this.logger = options.logger;\n }\n\n async initialize(): Promise<void> {\n this.tx = await this.store.getTransaction();\n try {\n await this.store.prepareInsert(this.tx);\n } catch (e) {\n // In case of error, rollback the transaction and re-throw the error so\n // that the stream can be closed and destroyed properly.\n this.tx.rollback(e);\n throw e;\n }\n }\n\n async index(documents: IndexableDocument[]): Promise<void> {\n this.numRecords += documents.length;\n\n const refs = [...new Set(documents.map(d => d.authorization?.resourceRef))];\n this.logger?.debug(\n `Attempting to index the following entities: ${refs.toString()}`,\n );\n\n try {\n await this.store.insertDocuments(this.tx!, this.type, documents);\n } catch (e) {\n // In case of error, rollback the transaction and re-throw the error so\n // that the stream can be closed and destroyed properly.\n this.tx!.rollback(e);\n throw e;\n }\n }\n\n async finalize(): Promise<void> {\n // If no documents were indexed, rollback the transaction, log a warning,\n // and do not continue. This ensures that collators that return empty sets\n // of documents do not cause the index to be deleted.\n if (this.numRecords === 0) {\n this.logger?.warn(\n `Index for ${this.type} was not replaced: indexer received 0 documents`,\n );\n this.tx!.rollback!();\n return;\n }\n\n // Do not truncate text by default\n let retryTruncated = false;\n\n do {\n // Sub-transaction allows a partial rollback if a truncated retry is needed\n let subTx: Knex.Transaction;\n try {\n subTx = await this.tx!.transaction();\n } catch (e) {\n // If sub-transaction creation fails, rollback the outer transaction\n // and re-throw the error so that the stream can be closed and\n // destroyed properly.\n this.tx!.rollback!(e);\n throw e;\n }\n\n // Attempt to complete and commit the transaction.\n try {\n await this.store.completeInsert(subTx, this.type, retryTruncated);\n\n // Commit both transactions if successful\n await subTx.commit();\n await this.tx!.commit();\n retryTruncated = false;\n } catch (e) {\n await subTx.rollback(e); // Rollback the first completeInsert attempt\n\n if (\n e instanceof Error &&\n e.message.includes('string is too long for tsvector') &&\n !retryTruncated\n ) {\n // If the error is due to a string being too long for tsvector, we\n // retry the operation with truncated text.\n retryTruncated = true;\n } else {\n // Otherwise, rollback the outer transaction and re-throw the error so that the\n // stream can be closed and destroyed properly.\n this.tx!.rollback!(e);\n throw e;\n }\n }\n } while (retryTruncated);\n }\n\n /**\n * Custom handler covering the case where an error occurred somewhere else in\n * the indexing pipeline (e.g. a collator or decorator). In such cases, the\n * finalize method is not called, which leaves a dangling transaction and\n * therefore an open connection to PG. This handler ensures we close the\n * transaction and associated connection.\n *\n * todo(@backstage/search-maintainers): Consider introducing a more\n * formal mechanism for handling such errors in BatchSearchEngineIndexer and\n * replacing this method with it. See: #17291\n *\n * @internal\n */\n async _destroy(error: Error | null, done: (error?: Error | null) => void) {\n // Ignore situations where there was no error.\n if (!error) {\n done();\n return;\n }\n\n if (!this.tx!.isCompleted()) {\n await this.tx!.rollback(error);\n }\n\n done(error);\n }\n}\n"],"names":["BatchSearchEngineIndexer"],"mappings":";;;;AA+BO,MAAM,8BAA8BA,gDAAA,CAAyB;AAAA,EAC1D,MAAA;AAAA,EACA,KAAA;AAAA,EACA,IAAA;AAAA,EACA,EAAA;AAAA,EACA,UAAA,GAAa,CAAA;AAAA,EAErB,YAAY,OAAA,EAAuC;AACjD,IAAA,KAAA,CAAM,EAAE,SAAA,EAAW,OAAA,CAAQ,SAAA,EAAW,CAAA;AACtC,IAAA,IAAA,CAAK,QAAQ,OAAA,CAAQ,aAAA;AACrB,IAAA,IAAA,CAAK,OAAO,OAAA,CAAQ,IAAA;AACpB,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AAAA,EACxB;AAAA,EAEA,MAAM,UAAA,GAA4B;AAChC,IAAA,IAAA,CAAK,EAAA,GAAK,MAAM,IAAA,CAAK,KAAA,CAAM,cAAA,EAAe;AAC1C,IAAA,IAAI;AACF,MAAA,MAAM,IAAA,CAAK,KAAA,CAAM,aAAA,CAAc,IAAA,CAAK,EAAE,CAAA;AAAA,IACxC,SAAS,CAAA,EAAG;AAGV,MAAA,IAAA,CAAK,EAAA,CAAG,SAAS,CAAC,CAAA;AAClB,MAAA,MAAM,CAAA;AAAA,IACR;AAAA,EACF;AAAA,EAEA,MAAM,MAAM,SAAA,EAA+C;AACzD,IAAA,IAAA,CAAK,cAAc,SAAA,CAAU,MAAA;AAE7B,IAAA,MAAM,IAAA,GAAO,CAAC,GAAG,IAAI,GAAA,CAAI,SAAA,CAAU,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,aAAA,EAAe,WAAW,CAAC,CAAC,CAAA;AAC1E,IAAA,IAAA,CAAK,MAAA,EAAQ,KAAA;AAAA,MACX,CAAA,4CAAA,EAA+C,IAAA,CAAK,QAAA,EAAU,CAAA;AAAA,KAChE;AAEA,IAAA,IAAI;AACF,MAAA,MAAM,KAAK,KAAA,CAAM,eAAA,CAAgB,KAAK,EAAA,EAAK,IAAA,CAAK,MAAM,SAAS,CAAA;AAAA,IACjE,SAAS,CAAA,EAAG;AAGV,MAAA,IAAA,CAAK,EAAA,CAAI,SAAS,CAAC,CAAA;AACnB,MAAA,MAAM,CAAA;AAAA,IACR;AAAA,EACF;AAAA,EAEA,MAAM,QAAA,GAA0B;AAI9B,IAAA,IAAI,IAAA,CAAK,eAAe,CAAA,EAAG;AACzB,MAAA,IAAA,CAAK,MAAA,EAAQ,IAAA;AAAA,QACX,CAAA,UAAA,EAAa,KAAK,IAAI,CAAA,+CAAA;AAAA,OACxB;AACA,MAAA,IAAA,CAAK,GAAI,QAAA,EAAU;AACnB,MAAA;AAAA,IACF;AAGA,IAAA,IAAI,cAAA,GAAiB,KAAA;AAErB,IAAA,GAAG;AAED,MAAA,IAAI,KAAA;AACJ,MAAA,IAAI;AACF,QAAA,KAAA,GAAQ,MAAM,IAAA,CAAK,EAAA,CAAI,WAAA,EAAY;AAAA,MACrC,SAAS,CAAA,EAAG;AAIV,QAAA,IAAA,CAAK,EAAA,CAAI,SAAU,CAAC,CAAA;AACpB,QAAA,MAAM,CAAA;AAAA,MACR;AAGA,MAAA,IAAI;AACF,QAAA,MAAM,KAAK,KAAA,CAAM,cAAA,CAAe,KAAA,EAAO,IAAA,CAAK,MAAM,cAAc,CAAA;AAGhE,QAAA,MAAM,MAAM,MAAA,EAAO;AACnB,QAAA,MAAM,IAAA,CAAK,GAAI,MAAA,EAAO;AACtB,QAAA,cAAA,GAAiB,KAAA;AAAA,MACnB,SAAS,CAAA,EAAG;AACV,QAAA,MAAM,KAAA,CAAM,SAAS,CAAC,CAAA;AAEtB,QAAA,IACE,CAAA,YAAa,SACb,CAAA,CAAE,OAAA,CAAQ,SAAS,iCAAiC,CAAA,IACpD,CAAC,cAAA,EACD;AAGA,UAAA,cAAA,GAAiB,IAAA;AAAA,QACnB,CAAA,MAAO;AAGL,UAAA,IAAA,CAAK,EAAA,CAAI,SAAU,CAAC,CAAA;AACpB,UAAA,MAAM,CAAA;AAAA,QACR;AAAA,MACF;AAAA,IACF,CAAA,QAAS,cAAA;AAAA,EACX;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAeA,MAAM,QAAA,CAAS,KAAA,EAAqB,IAAA,EAAsC;AAExE,IAAA,IAAI,CAAC,KAAA,EAAO;AACV,MAAA,IAAA,EAAK;AACL,MAAA;AAAA,IACF;AAEA,IAAA,IAAI,CAAC,IAAA,CAAK,EAAA,CAAI,WAAA,EAAY,EAAG;AAC3B,MAAA,MAAM,IAAA,CAAK,EAAA,CAAI,QAAA,CAAS,KAAK,CAAA;AAAA,IAC/B;AAEA,IAAA,IAAA,CAAK,KAAK,CAAA;AAAA,EACZ;AACF;;;;"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@backstage/plugin-search-backend-module-pg",
|
|
3
|
-
"version": "0.5.54
|
|
3
|
+
"version": "0.5.54",
|
|
4
4
|
"description": "A module for the search backend that implements search using PostgreSQL",
|
|
5
5
|
"backstage": {
|
|
6
6
|
"role": "backend-plugin-module",
|
|
@@ -63,17 +63,17 @@
|
|
|
63
63
|
"test": "backstage-cli package test"
|
|
64
64
|
},
|
|
65
65
|
"dependencies": {
|
|
66
|
-
"@backstage/backend-plugin-api": "1.9.0
|
|
67
|
-
"@backstage/config": "1.3.7
|
|
68
|
-
"@backstage/plugin-search-backend-node": "1.4.3
|
|
69
|
-
"@backstage/plugin-search-common": "1.2.23
|
|
66
|
+
"@backstage/backend-plugin-api": "^1.9.0",
|
|
67
|
+
"@backstage/config": "^1.3.7",
|
|
68
|
+
"@backstage/plugin-search-backend-node": "^1.4.3",
|
|
69
|
+
"@backstage/plugin-search-common": "^1.2.23",
|
|
70
70
|
"knex": "^3.0.0",
|
|
71
71
|
"lodash": "^4.17.21",
|
|
72
72
|
"uuid": "^11.0.0"
|
|
73
73
|
},
|
|
74
74
|
"devDependencies": {
|
|
75
|
-
"@backstage/backend-test-utils": "1.11.2
|
|
76
|
-
"@backstage/cli": "0.36.1
|
|
75
|
+
"@backstage/backend-test-utils": "^1.11.2",
|
|
76
|
+
"@backstage/cli": "^0.36.1"
|
|
77
77
|
},
|
|
78
78
|
"configSchema": "config.d.ts"
|
|
79
79
|
}
|