@backstage/plugin-catalog-backend 3.0.2-next.0 → 3.0.2-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
# @backstage/plugin-catalog-backend
|
|
2
2
|
|
|
3
|
+
## 3.0.2-next.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 2204f5b: Prevent deadlock in catalog deferred stitching
|
|
8
|
+
- Updated dependencies
|
|
9
|
+
- @backstage/catalog-client@1.12.0-next.0
|
|
10
|
+
- @backstage/plugin-catalog-node@1.19.0-next.1
|
|
11
|
+
- @backstage/integration@1.18.0-next.0
|
|
12
|
+
|
|
3
13
|
## 3.0.2-next.0
|
|
4
14
|
|
|
5
15
|
### Patch Changes
|
|
@@ -2,14 +2,26 @@
|
|
|
2
2
|
|
|
3
3
|
var splitToChunks = require('lodash/chunk');
|
|
4
4
|
var uuid = require('uuid');
|
|
5
|
+
var errors = require('@backstage/errors');
|
|
6
|
+
var promises = require('timers/promises');
|
|
5
7
|
|
|
6
8
|
function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e : { default: e }; }
|
|
7
9
|
|
|
8
10
|
var splitToChunks__default = /*#__PURE__*/_interopDefaultCompat(splitToChunks);
|
|
9
11
|
|
|
12
|
+
const UPDATE_CHUNK_SIZE = 100;
|
|
13
|
+
const DEADLOCK_RETRY_ATTEMPTS = 3;
|
|
14
|
+
const DEADLOCK_BASE_DELAY_MS = 25;
|
|
15
|
+
const POSTGRES_DEADLOCK_SQLSTATE = "40P01";
|
|
16
|
+
function isDeadlockError(knex, e) {
|
|
17
|
+
if (knex.client.config.client.includes("pg")) {
|
|
18
|
+
return errors.isError(e) && e.code === POSTGRES_DEADLOCK_SQLSTATE;
|
|
19
|
+
}
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
10
22
|
async function markForStitching(options) {
|
|
11
|
-
const entityRefs =
|
|
12
|
-
const entityIds =
|
|
23
|
+
const entityRefs = sortSplit(options.entityRefs);
|
|
24
|
+
const entityIds = sortSplit(options.entityIds);
|
|
13
25
|
const knex = options.knex;
|
|
14
26
|
const mode = options.strategy.mode;
|
|
15
27
|
if (mode === "immediate") {
|
|
@@ -20,43 +32,68 @@ async function markForStitching(options) {
|
|
|
20
32
|
"entity_id",
|
|
21
33
|
knex("refresh_state").select("entity_id").whereIn("entity_ref", chunk)
|
|
22
34
|
);
|
|
23
|
-
await
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
35
|
+
await retryOnDeadlock(async () => {
|
|
36
|
+
await knex.table("refresh_state").update({
|
|
37
|
+
result_hash: "force-stitching",
|
|
38
|
+
next_update_at: knex.fn.now()
|
|
39
|
+
}).whereIn("entity_ref", chunk);
|
|
40
|
+
}, knex);
|
|
27
41
|
}
|
|
28
42
|
for (const chunk of entityIds) {
|
|
29
43
|
await knex.table("final_entities").update({
|
|
30
44
|
hash: "force-stitching"
|
|
31
45
|
}).whereIn("entity_id", chunk);
|
|
32
|
-
await
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
46
|
+
await retryOnDeadlock(async () => {
|
|
47
|
+
await knex.table("refresh_state").update({
|
|
48
|
+
result_hash: "force-stitching",
|
|
49
|
+
next_update_at: knex.fn.now()
|
|
50
|
+
}).whereIn("entity_id", chunk);
|
|
51
|
+
}, knex);
|
|
36
52
|
}
|
|
37
53
|
} else if (mode === "deferred") {
|
|
38
54
|
const ticket = uuid.v4();
|
|
39
55
|
for (const chunk of entityRefs) {
|
|
40
|
-
await
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
56
|
+
await retryOnDeadlock(async () => {
|
|
57
|
+
await knex("refresh_state").update({
|
|
58
|
+
next_stitch_at: knex.fn.now(),
|
|
59
|
+
next_stitch_ticket: ticket
|
|
60
|
+
}).whereIn("entity_ref", chunk);
|
|
61
|
+
}, knex);
|
|
44
62
|
}
|
|
45
63
|
for (const chunk of entityIds) {
|
|
46
|
-
await
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
64
|
+
await retryOnDeadlock(async () => {
|
|
65
|
+
await knex("refresh_state").update({
|
|
66
|
+
next_stitch_at: knex.fn.now(),
|
|
67
|
+
next_stitch_ticket: ticket
|
|
68
|
+
}).whereIn("entity_id", chunk);
|
|
69
|
+
}, knex);
|
|
50
70
|
}
|
|
51
71
|
} else {
|
|
52
72
|
throw new Error(`Unknown stitching strategy mode ${mode}`);
|
|
53
73
|
}
|
|
54
74
|
}
|
|
55
|
-
function
|
|
75
|
+
function sortSplit(input) {
|
|
56
76
|
if (!input) {
|
|
57
77
|
return [];
|
|
58
78
|
}
|
|
59
|
-
|
|
79
|
+
const array = Array.isArray(input) ? input.slice() : [...input];
|
|
80
|
+
array.sort();
|
|
81
|
+
return splitToChunks__default.default(array, UPDATE_CHUNK_SIZE);
|
|
82
|
+
}
|
|
83
|
+
async function retryOnDeadlock(fn, knex, retries = DEADLOCK_RETRY_ATTEMPTS, baseMs = DEADLOCK_BASE_DELAY_MS) {
|
|
84
|
+
let attempt = 0;
|
|
85
|
+
for (; ; ) {
|
|
86
|
+
try {
|
|
87
|
+
return await fn();
|
|
88
|
+
} catch (e) {
|
|
89
|
+
if (isDeadlockError(knex, e) && attempt < retries) {
|
|
90
|
+
await promises.setTimeout(baseMs * Math.pow(2, attempt));
|
|
91
|
+
attempt++;
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
throw e;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
60
97
|
}
|
|
61
98
|
|
|
62
99
|
exports.markForStitching = markForStitching;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"markForStitching.cjs.js","sources":["../../../../src/database/operations/stitcher/markForStitching.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport splitToChunks from 'lodash/chunk';\nimport { v4 as uuid } from 'uuid';\nimport { StitchingStrategy } from '../../../stitching/types';\nimport { DbFinalEntitiesRow, DbRefreshStateRow } from '../../tables';\n\n/**\n * Marks a number of entities for stitching some time in the near\n * future.\n *\n * @remarks\n */\nexport async function markForStitching(options: {\n knex: Knex | Knex.Transaction;\n strategy: StitchingStrategy;\n entityRefs?: Iterable<string>;\n entityIds?: Iterable<string>;\n}): Promise<void> {\n
|
|
1
|
+
{"version":3,"file":"markForStitching.cjs.js","sources":["../../../../src/database/operations/stitcher/markForStitching.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport splitToChunks from 'lodash/chunk';\nimport { v4 as uuid } from 'uuid';\nimport { ErrorLike, isError } from '@backstage/errors';\nimport { StitchingStrategy } from '../../../stitching/types';\nimport { setTimeout as sleep } from 'timers/promises';\nimport { DbFinalEntitiesRow, DbRefreshStateRow } from '../../tables';\n\nconst UPDATE_CHUNK_SIZE = 100; // Smaller chunks reduce contention\nconst DEADLOCK_RETRY_ATTEMPTS = 3;\nconst DEADLOCK_BASE_DELAY_MS = 25;\n\n// PostgreSQL deadlock error code\nconst POSTGRES_DEADLOCK_SQLSTATE = '40P01';\n\n/**\n * Checks if the given error is a deadlock error for the database engine in use.\n */\nfunction isDeadlockError(\n knex: Knex | Knex.Transaction,\n e: unknown,\n): e is ErrorLike {\n if (knex.client.config.client.includes('pg')) {\n // PostgreSQL deadlock detection\n return isError(e) && e.code === POSTGRES_DEADLOCK_SQLSTATE;\n }\n\n // Add more database engine checks here as needed\n return false;\n}\n\n/**\n * Marks a number of entities for stitching some time in the near\n * future.\n *\n * @remarks\n */\nexport async function markForStitching(options: {\n knex: Knex | Knex.Transaction;\n strategy: StitchingStrategy;\n entityRefs?: Iterable<string>;\n entityIds?: Iterable<string>;\n}): Promise<void> {\n const entityRefs = sortSplit(options.entityRefs);\n const entityIds = sortSplit(options.entityIds);\n const knex = options.knex;\n const mode = options.strategy.mode;\n\n if (mode === 'immediate') {\n for (const chunk of entityRefs) {\n await knex\n .table<DbFinalEntitiesRow>('final_entities')\n .update({\n hash: 'force-stitching',\n })\n .whereIn(\n 'entity_id',\n knex<DbRefreshStateRow>('refresh_state')\n .select('entity_id')\n .whereIn('entity_ref', chunk),\n );\n await retryOnDeadlock(async () => {\n await knex\n .table<DbRefreshStateRow>('refresh_state')\n .update({\n result_hash: 'force-stitching',\n next_update_at: knex.fn.now(),\n })\n .whereIn('entity_ref', chunk);\n }, knex);\n }\n\n for (const chunk of entityIds) {\n await knex\n .table<DbFinalEntitiesRow>('final_entities')\n .update({\n hash: 'force-stitching',\n })\n .whereIn('entity_id', chunk);\n await retryOnDeadlock(async () => {\n await knex\n .table<DbRefreshStateRow>('refresh_state')\n .update({\n result_hash: 'force-stitching',\n next_update_at: knex.fn.now(),\n })\n .whereIn('entity_id', chunk);\n }, knex);\n }\n } else if (mode === 'deferred') {\n // It's OK that this is shared across refresh state rows; it just needs to\n // be uniquely generated for every new stitch request.\n const ticket = uuid();\n\n // Update by primary key in deterministic order to avoid deadlocks\n for (const chunk of entityRefs) {\n await retryOnDeadlock(async () => {\n await knex<DbRefreshStateRow>('refresh_state')\n .update({\n next_stitch_at: knex.fn.now(),\n next_stitch_ticket: ticket,\n })\n .whereIn('entity_ref', chunk);\n }, knex);\n }\n\n for (const chunk of entityIds) {\n await retryOnDeadlock(async () => {\n await knex<DbRefreshStateRow>('refresh_state')\n .update({\n next_stitch_at: knex.fn.now(),\n next_stitch_ticket: ticket,\n })\n .whereIn('entity_id', chunk);\n }, knex);\n }\n } else {\n throw new Error(`Unknown stitching strategy mode ${mode}`);\n }\n}\n\nfunction sortSplit(input: Iterable<string> | undefined): string[][] {\n if (!input) {\n return [];\n }\n const array = Array.isArray(input) ? input.slice() : [...input];\n array.sort();\n return splitToChunks(array, UPDATE_CHUNK_SIZE);\n}\n\nasync function retryOnDeadlock<T>(\n fn: () => Promise<T>,\n knex: Knex | Knex.Transaction,\n retries = DEADLOCK_RETRY_ATTEMPTS,\n baseMs = DEADLOCK_BASE_DELAY_MS,\n): Promise<T> {\n let attempt = 0;\n for (;;) {\n try {\n return await fn();\n } catch (e: unknown) {\n if (isDeadlockError(knex, e) && attempt < retries) {\n await sleep(baseMs * Math.pow(2, attempt));\n attempt++;\n continue;\n }\n throw e;\n }\n }\n}\n"],"names":["isError","uuid","splitToChunks","sleep"],"mappings":";;;;;;;;;;;AAwBA,MAAM,iBAAA,GAAoB,GAAA;AAC1B,MAAM,uBAAA,GAA0B,CAAA;AAChC,MAAM,sBAAA,GAAyB,EAAA;AAG/B,MAAM,0BAAA,GAA6B,OAAA;AAKnC,SAAS,eAAA,CACP,MACA,CAAA,EACgB;AAChB,EAAA,IAAI,KAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,IAAI,CAAA,EAAG;AAE5C,IAAA,OAAOA,cAAA,CAAQ,CAAC,CAAA,IAAK,CAAA,CAAE,IAAA,KAAS,0BAAA;AAAA,EAClC;AAGA,EAAA,OAAO,KAAA;AACT;AAQA,eAAsB,iBAAiB,OAAA,EAKrB;AAChB,EAAA,MAAM,UAAA,GAAa,SAAA,CAAU,OAAA,CAAQ,UAAU,CAAA;AAC/C,EAAA,MAAM,SAAA,GAAY,SAAA,CAAU,OAAA,CAAQ,SAAS,CAAA;AAC7C,EAAA,MAAM,OAAO,OAAA,CAAQ,IAAA;AACrB,EAAA,MAAM,IAAA,GAAO,QAAQ,QAAA,CAAS,IAAA;AAE9B,EAAA,IAAI,SAAS,WAAA,EAAa;AACxB,IAAA,KAAA,MAAW,SAAS,UAAA,EAAY;AAC9B,MAAA,MAAM,IAAA,CACH,KAAA,CAA0B,gBAAgB,CAAA,CAC1C,MAAA,CAAO;AAAA,QACN,IAAA,EAAM;AAAA,OACP,CAAA,CACA,OAAA;AAAA,QACC,WAAA;AAAA,QACA,IAAA,CAAwB,eAAe,CAAA,CACpC,MAAA,CAAO,WAAW,CAAA,CAClB,OAAA,CAAQ,cAAc,KAAK;AAAA,OAChC;AACF,MAAA,MAAM,gBAAgB,YAAY;AAChC,QAAA,MAAM,IAAA,CACH,KAAA,CAAyB,eAAe,CAAA,CACxC,MAAA,CAAO;AAAA,UACN,WAAA,EAAa,iBAAA;AAAA,UACb,cAAA,EAAgB,IAAA,CAAK,EAAA,CAAG,GAAA;AAAI,SAC7B,CAAA,CACA,OAAA,CAAQ,YAAA,EAAc,KAAK,CAAA;AAAA,MAChC,GAAG,IAAI,CAAA;AAAA,IACT;AAEA,IAAA,KAAA,MAAW,SAAS,SAAA,EAAW;AAC7B,MAAA,MAAM,IAAA,CACH,KAAA,CAA0B,gBAAgB,CAAA,CAC1C,MAAA,CAAO;AAAA,QACN,IAAA,EAAM;AAAA,OACP,CAAA,CACA,OAAA,CAAQ,WAAA,EAAa,KAAK,CAAA;AAC7B,MAAA,MAAM,gBAAgB,YAAY;AAChC,QAAA,MAAM,IAAA,CACH,KAAA,CAAyB,eAAe,CAAA,CACxC,MAAA,CAAO;AAAA,UACN,WAAA,EAAa,iBAAA;AAAA,UACb,cAAA,EAAgB,IAAA,CAAK,EAAA,CAAG,GAAA;AAAI,SAC7B,CAAA,CACA,OAAA,CAAQ,WAAA,EAAa,KAAK,CAAA;AAAA,MAC/B,GAAG,IAAI,CAAA;AAAA,IACT;AAAA,EACF,CAAA,MAAA,IAAW,SAAS,UAAA,EAAY;AAG9B,IAAA,MAAM,SAASC,OAAA,EAAK;AAGpB,IAAA,KAAA,MAAW,SAAS,UAAA,EAAY;AAC9B,MAAA,MAAM,gBAAgB,YAAY;AAChC,QAAA,MAAM,IAAA,CAAwB,eAAe,CAAA,CAC1C,MAAA,CAAO;AAAA,UACN,cAAA,EAAgB,IAAA,CAAK,EAAA,CAAG,GAAA,EAAI;AAAA,UAC5B,kBAAA,EAAoB;AAAA,SACrB,CAAA,CACA,OAAA,CAAQ,YAAA,EAAc,KAAK,CAAA;AAAA,MAChC,GAAG,IAAI,CAAA;AAAA,IACT;AAEA,IAAA,KAAA,MAAW,SAAS,SAAA,EAAW;AAC7B,MAAA,MAAM,gBAAgB,YAAY;AAChC,QAAA,MAAM,IAAA,CAAwB,eAAe,CAAA,CAC1C,MAAA,CAAO;AAAA,UACN,cAAA,EAAgB,IAAA,CAAK,EAAA,CAAG,GAAA,EAAI;AAAA,UAC5B,kBAAA,EAAoB;AAAA,SACrB,CAAA,CACA,OAAA,CAAQ,WAAA,EAAa,KAAK,CAAA;AAAA,MAC/B,GAAG,IAAI,CAAA;AAAA,IACT;AAAA,EACF,CAAA,MAAO;AACL,IAAA,MAAM,IAAI,KAAA,CAAM,CAAA,gCAAA,EAAmC,IAAI,CAAA,CAAE,CAAA;AAAA,EAC3D;AACF;AAEA,SAAS,UAAU,KAAA,EAAiD;AAClE,EAAA,IAAI,CAAC,KAAA,EAAO;AACV,IAAA,OAAO,EAAC;AAAA,EACV;AACA,EAAA,MAAM,KAAA,GAAQ,KAAA,CAAM,OAAA,CAAQ,KAAK,CAAA,GAAI,MAAM,KAAA,EAAM,GAAI,CAAC,GAAG,KAAK,CAAA;AAC9D,EAAA,KAAA,CAAM,IAAA,EAAK;AACX,EAAA,OAAOC,8BAAA,CAAc,OAAO,iBAAiB,CAAA;AAC/C;AAEA,eAAe,gBACb,EAAA,EACA,IAAA,EACA,OAAA,GAAU,uBAAA,EACV,SAAS,sBAAA,EACG;AACZ,EAAA,IAAI,OAAA,GAAU,CAAA;AACd,EAAA,WAAS;AACP,IAAA,IAAI;AACF,MAAA,OAAO,MAAM,EAAA,EAAG;AAAA,IAClB,SAAS,CAAA,EAAY;AACnB,MAAA,IAAI,eAAA,CAAgB,IAAA,EAAM,CAAC,CAAA,IAAK,UAAU,OAAA,EAAS;AACjD,QAAA,MAAMC,oBAAM,MAAA,GAAS,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,OAAO,CAAC,CAAA;AACzC,QAAA,OAAA,EAAA;AACA,QAAA;AAAA,MACF;AACA,MAAA,MAAM,CAAA;AAAA,IACR;AAAA,EACF;AACF;;;;"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@backstage/plugin-catalog-backend",
|
|
3
|
-
"version": "3.0.2-next.
|
|
3
|
+
"version": "3.0.2-next.1",
|
|
4
4
|
"description": "The Backstage backend plugin that provides the Backstage catalog",
|
|
5
5
|
"backstage": {
|
|
6
6
|
"role": "backend-plugin",
|
|
@@ -75,13 +75,13 @@
|
|
|
75
75
|
"dependencies": {
|
|
76
76
|
"@backstage/backend-openapi-utils": "0.6.1-next.0",
|
|
77
77
|
"@backstage/backend-plugin-api": "1.4.3-next.0",
|
|
78
|
-
"@backstage/catalog-client": "1.
|
|
78
|
+
"@backstage/catalog-client": "1.12.0-next.0",
|
|
79
79
|
"@backstage/catalog-model": "1.7.5",
|
|
80
80
|
"@backstage/config": "1.3.3",
|
|
81
81
|
"@backstage/errors": "1.2.7",
|
|
82
82
|
"@backstage/integration": "1.18.0-next.0",
|
|
83
83
|
"@backstage/plugin-catalog-common": "1.1.5",
|
|
84
|
-
"@backstage/plugin-catalog-node": "1.
|
|
84
|
+
"@backstage/plugin-catalog-node": "1.19.0-next.1",
|
|
85
85
|
"@backstage/plugin-events-node": "0.4.15-next.0",
|
|
86
86
|
"@backstage/plugin-permission-common": "0.9.1",
|
|
87
87
|
"@backstage/plugin-permission-node": "0.10.4-next.0",
|
|
@@ -106,11 +106,11 @@
|
|
|
106
106
|
"zod": "^3.22.4"
|
|
107
107
|
},
|
|
108
108
|
"devDependencies": {
|
|
109
|
-
"@backstage/backend-defaults": "0.12.1-next.
|
|
109
|
+
"@backstage/backend-defaults": "0.12.1-next.1",
|
|
110
110
|
"@backstage/backend-test-utils": "1.9.0-next.1",
|
|
111
|
-
"@backstage/cli": "0.34.2-next.
|
|
111
|
+
"@backstage/cli": "0.34.2-next.2",
|
|
112
112
|
"@backstage/plugin-permission-common": "0.9.1",
|
|
113
|
-
"@backstage/repo-tools": "0.15.2-next.
|
|
113
|
+
"@backstage/repo-tools": "0.15.2-next.1",
|
|
114
114
|
"@types/core-js": "^2.5.4",
|
|
115
115
|
"@types/express": "^4.17.6",
|
|
116
116
|
"@types/git-url-parse": "^9.0.0",
|