backstage-plugin-catalog-backend-module-history 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +31 -2
- package/bin/reconcile-catalog-history.js +16 -0
- package/config.schema.json +121 -0
- package/dist/index.cjs.js +44 -0
- package/dist/index.cjs.js.map +1 -0
- package/dist/index.d.ts +227 -0
- package/dist/mapping/entityToRow.cjs.js +111 -0
- package/dist/mapping/entityToRow.cjs.js.map +1 -0
- package/dist/module/catalogModuleHistory.cjs.js +107 -0
- package/dist/module/catalogModuleHistory.cjs.js.map +1 -0
- package/dist/processor/HistoryRecordingCatalogProcessor.cjs.js +182 -0
- package/dist/processor/HistoryRecordingCatalogProcessor.cjs.js.map +1 -0
- package/dist/provider/HistoryRecordingEntityProvider.cjs.js +147 -0
- package/dist/provider/HistoryRecordingEntityProvider.cjs.js.map +1 -0
- package/dist/reconcile-cli.cjs.js +86 -0
- package/dist/reconcile-cli.cjs.js.map +1 -0
- package/dist/reconcile-cli.d.ts +14 -0
- package/dist/reconciler/CatalogServiceEntityFetcher.cjs.js +21 -0
- package/dist/reconciler/CatalogServiceEntityFetcher.cjs.js.map +1 -0
- package/dist/reconciler/paginateEntities.cjs.js +16 -0
- package/dist/reconciler/paginateEntities.cjs.js.map +1 -0
- package/dist/reconciler/reconcile.cjs.js +58 -0
- package/dist/reconciler/reconcile.cjs.js.map +1 -0
- package/package.json +105 -3
package/LICENSE
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright [yyyy] [name of copyright owner]
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|
package/README.md
CHANGED
|
@@ -1,3 +1,32 @@
|
|
|
1
|
-
#
|
|
1
|
+
# backstage-plugin-catalog-backend-module-history
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Backstage backend module that records catalog entity changes into a versioned history store.
|
|
4
|
+
|
|
5
|
+
The default backend feature loader installs both the catalog backend module and the default Postgres `historyStoreServiceRef` factory from `@kunickiaj/catalog-history-backend`.
|
|
6
|
+
|
|
7
|
+
```ts
|
|
8
|
+
backend.add(import('backstage-plugin-catalog-backend-module-history'));
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Advanced storage overrides should register their own `historyStoreServiceRef` factory and import the named module instead of the default loader:
|
|
12
|
+
|
|
13
|
+
```ts
|
|
14
|
+
backend.add(catalogModuleHistory);
|
|
15
|
+
backend.add(myHistoryStoreServiceFactory);
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Provider wrappers should be configured in the same catalog module where the provider is created: depend on `historyStoreServiceRef`, then pass the resolved store into `HistoryRecordingEntityProvider`. The legacy `catalog.history.database` setting remains as a deprecated compatibility path honored by the default store factory, so all consumers of the injected store share the same connection.
|
|
19
|
+
|
|
20
|
+
## Optional full-to-delta conversion
|
|
21
|
+
|
|
22
|
+
`HistoryRecordingEntityProvider` defaults to forwarding provider mutations unchanged. With `convertFullToDelta: true`, full snapshots with an existing history baseline become deltas: every incoming entry goes into `added`, with its `locationKey` intact, and history refs missing from the snapshot go into `removed`. The catalog compares each entry's full content and key; the wrapper does not use history etags to filter catalog writes. This preserves key-only changes and content changes even when a provider reuses an etag, but does not avoid comparing unchanged entries.
|
|
23
|
+
|
|
24
|
+
History still classifies changes using entity etags, so its counts can report an entry as unchanged even when the catalog sees a key or content change. History is best-effort, not a deletion safety check: a missing history ref cannot produce an inferred removal. An empty baseline or a failed baseline read forwards the original full mutation instead.
|
|
25
|
+
|
|
26
|
+
With `forceFullEvery` configured, the first full after the wrapper starts is forwarded unchanged even when history already has a baseline. Each successfully forwarded full starts the interval, including a fallback full after a history-read failure; rejected catalog writes and native deltas do not reset it. The first full received at or after the interval expires is forwarded unchanged to reconcile against the catalog's own provider refs. This is not a background timer. Keep conversion disabled when deletions must always use the catalog's current provider refs.
|
|
27
|
+
|
|
28
|
+
### Catalog regression tests
|
|
29
|
+
|
|
30
|
+
The provider tests include an in-memory SQLite integration with the pinned catalog backend development dependency. Run them from the workspace root with `env CI=1 yarn test packages/catalog-backend-module-history/src/provider --watch=false --runInBand`. They do not require a running catalog or an external database.
|
|
31
|
+
|
|
32
|
+
For current usage and productization roadmap details, see the [workspace README](../../README.md).
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Thin shim so npm can wire this up via package.json `bin`. The real work
|
|
3
|
+
// lives in src/bin/reconcile.ts and is built into
|
|
4
|
+
// dist/reconcile-cli.cjs.js by @backstage/cli (declared as an entry point
|
|
5
|
+
// in the package.json `exports` field).
|
|
6
|
+
const { main } = require('../dist/reconcile-cli.cjs.js');
|
|
7
|
+
|
|
8
|
+
main().catch(err => {
|
|
9
|
+
process.stderr.write(
|
|
10
|
+
`error ${err instanceof Error ? err.message : String(err)}\n`,
|
|
11
|
+
);
|
|
12
|
+
if (err && err.stack) {
|
|
13
|
+
process.stderr.write(`${err.stack}\n`);
|
|
14
|
+
}
|
|
15
|
+
process.exit(1);
|
|
16
|
+
});
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
3
|
+
"type": "object",
|
|
4
|
+
"properties": {
|
|
5
|
+
"catalog": {
|
|
6
|
+
"type": "object",
|
|
7
|
+
"properties": {
|
|
8
|
+
"history": {
|
|
9
|
+
"type": "object",
|
|
10
|
+
"properties": {
|
|
11
|
+
"enabled": {
|
|
12
|
+
"type": "boolean",
|
|
13
|
+
"description": "Master switch for the catalog history module. Set to false to skip schema bootstrap entirely."
|
|
14
|
+
},
|
|
15
|
+
"database": {
|
|
16
|
+
"type": "object",
|
|
17
|
+
"properties": {
|
|
18
|
+
"client": {
|
|
19
|
+
"type": "string",
|
|
20
|
+
"description": "Knex database client name. Defaults to pg."
|
|
21
|
+
},
|
|
22
|
+
"connection": {
|
|
23
|
+
"anyOf": [
|
|
24
|
+
{
|
|
25
|
+
"type": "string"
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
"type": "object",
|
|
29
|
+
"additionalProperties": {}
|
|
30
|
+
}
|
|
31
|
+
],
|
|
32
|
+
"description": "Knex connection string or connection object for the history database. Required when the `database` block is present — the default store factory uses it to open the override connection.",
|
|
33
|
+
"visibility": "secret",
|
|
34
|
+
"deepVisibility": "secret"
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
"required": [
|
|
38
|
+
"connection"
|
|
39
|
+
],
|
|
40
|
+
"description": "Deprecated compatibility database override for history storage.\n\nPrefer registering a custom `historyStoreServiceRef` factory in your backend instead. That lets provider wrappers, processor capture, and scheduled reconciliation all receive the same store through Backstage DI at creation time.\n\nThe default `historyStoreServiceFactory` honors this override for all consumers of `historyStoreServiceRef`, including provider wrappers that inject the same store in their own catalog module."
|
|
41
|
+
},
|
|
42
|
+
"provider": {
|
|
43
|
+
"type": "object",
|
|
44
|
+
"properties": {
|
|
45
|
+
"enabled": {
|
|
46
|
+
"type": "boolean",
|
|
47
|
+
"description": "Enables provider-layer history recording. Defaults to true."
|
|
48
|
+
}
|
|
49
|
+
},
|
|
50
|
+
"description": "Provider-layer capture settings.\n\nIMPORTANT: this key is advisory — the backend module cannot enforce it, because provider wrapping happens in your own backend wiring. You must thread it into the wrapper yourself along with the injected history store, e.g. `new HistoryRecordingEntityProvider({ ..., enabled: config.getOptionalBoolean('catalog.history.provider.enabled') ?? true })`. Setting it in app-config without that wiring has no effect beyond a startup log line."
|
|
51
|
+
},
|
|
52
|
+
"processing": {
|
|
53
|
+
"type": "object",
|
|
54
|
+
"properties": {
|
|
55
|
+
"enabled": {
|
|
56
|
+
"type": "boolean",
|
|
57
|
+
"description": "Enables processor-layer history capture. Defaults to false."
|
|
58
|
+
}
|
|
59
|
+
},
|
|
60
|
+
"description": "Processor-layer capture settings. Defaults to false because processor capture runs once per entity per processing cycle."
|
|
61
|
+
},
|
|
62
|
+
"reconciler": {
|
|
63
|
+
"type": "object",
|
|
64
|
+
"properties": {
|
|
65
|
+
"enabled": {
|
|
66
|
+
"type": "boolean",
|
|
67
|
+
"description": "Enables scheduled in-process reconciliation. Defaults to false."
|
|
68
|
+
},
|
|
69
|
+
"schedule": {
|
|
70
|
+
"type": "object",
|
|
71
|
+
"properties": {
|
|
72
|
+
"frequency": {
|
|
73
|
+
"anyOf": [
|
|
74
|
+
{
|
|
75
|
+
"type": "string"
|
|
76
|
+
},
|
|
77
|
+
{
|
|
78
|
+
"type": "object",
|
|
79
|
+
"additionalProperties": {}
|
|
80
|
+
}
|
|
81
|
+
],
|
|
82
|
+
"description": "Reconciler schedule frequency. Uses Backstage scheduler duration configuration objects, string durations, cron objects, or manual trigger config supported by SchedulerServiceTaskScheduleDefinition."
|
|
83
|
+
},
|
|
84
|
+
"timeout": {
|
|
85
|
+
"anyOf": [
|
|
86
|
+
{
|
|
87
|
+
"type": "string"
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
"type": "object",
|
|
91
|
+
"additionalProperties": {}
|
|
92
|
+
}
|
|
93
|
+
],
|
|
94
|
+
"description": "Reconciler task timeout. Uses Backstage scheduler duration configuration objects or string durations."
|
|
95
|
+
},
|
|
96
|
+
"initialDelay": {
|
|
97
|
+
"anyOf": [
|
|
98
|
+
{
|
|
99
|
+
"type": "string"
|
|
100
|
+
},
|
|
101
|
+
{
|
|
102
|
+
"type": "object",
|
|
103
|
+
"additionalProperties": {}
|
|
104
|
+
}
|
|
105
|
+
],
|
|
106
|
+
"description": "Reconciler initial delay. Uses Backstage scheduler duration configuration objects or string durations."
|
|
107
|
+
}
|
|
108
|
+
},
|
|
109
|
+
"description": "Scheduler configuration for in-process reconciliation. The shape follows Backstage's SchedulerServiceTaskScheduleDefinition config format; omit it to use the default hourly schedule, 10 minute timeout, and 30 second initial delay.\n\nWhen set, `frequency` and `timeout` are both required (Backstage's schedule parser rejects partial definitions at startup); the defaults above apply only when the whole `schedule` key is omitted."
|
|
110
|
+
}
|
|
111
|
+
},
|
|
112
|
+
"description": "In-process reconciler settings. Defaults to disabled. When enabled without an explicit schedule, the task runs hourly with a 10 minute timeout and a 30 second initial delay."
|
|
113
|
+
}
|
|
114
|
+
},
|
|
115
|
+
"description": "Catalog history module configuration."
|
|
116
|
+
}
|
|
117
|
+
},
|
|
118
|
+
"description": "Configuration for catalog history capture and storage."
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
Object.defineProperty(exports, '__esModule', { value: true });
|
|
4
|
+
|
|
5
|
+
var backendPluginApi = require('@backstage/backend-plugin-api');
|
|
6
|
+
var catalogHistoryBackend = require('@kunickiaj/catalog-history-backend');
|
|
7
|
+
var catalogModuleHistory = require('./module/catalogModuleHistory.cjs.js');
|
|
8
|
+
var catalogHistoryNode = require('@kunickiaj/catalog-history-node');
|
|
9
|
+
var entityToRow = require('./mapping/entityToRow.cjs.js');
|
|
10
|
+
var HistoryRecordingEntityProvider = require('./provider/HistoryRecordingEntityProvider.cjs.js');
|
|
11
|
+
var HistoryRecordingCatalogProcessor = require('./processor/HistoryRecordingCatalogProcessor.cjs.js');
|
|
12
|
+
var reconcile = require('./reconciler/reconcile.cjs.js');
|
|
13
|
+
var CatalogServiceEntityFetcher = require('./reconciler/CatalogServiceEntityFetcher.cjs.js');
|
|
14
|
+
|
|
15
|
+
var index = backendPluginApi.createBackendFeatureLoader({
|
|
16
|
+
loader() {
|
|
17
|
+
return [catalogHistoryBackend.historyStoreServiceFactory, catalogModuleHistory.catalogModuleHistory];
|
|
18
|
+
}
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
Object.defineProperty(exports, "PostgresHistoryStore", {
|
|
22
|
+
enumerable: true,
|
|
23
|
+
get: function () { return catalogHistoryBackend.PostgresHistoryStore; }
|
|
24
|
+
});
|
|
25
|
+
Object.defineProperty(exports, "ensureSchema", {
|
|
26
|
+
enumerable: true,
|
|
27
|
+
get: function () { return catalogHistoryBackend.ensureSchema; }
|
|
28
|
+
});
|
|
29
|
+
Object.defineProperty(exports, "historyStoreServiceFactory", {
|
|
30
|
+
enumerable: true,
|
|
31
|
+
get: function () { return catalogHistoryBackend.historyStoreServiceFactory; }
|
|
32
|
+
});
|
|
33
|
+
exports.catalogModuleHistory = catalogModuleHistory.catalogModuleHistory;
|
|
34
|
+
Object.defineProperty(exports, "RECONCILER_PROVIDER", {
|
|
35
|
+
enumerable: true,
|
|
36
|
+
get: function () { return catalogHistoryNode.RECONCILER_PROVIDER; }
|
|
37
|
+
});
|
|
38
|
+
exports.entityToRow = entityToRow.entityToRow;
|
|
39
|
+
exports.HistoryRecordingEntityProvider = HistoryRecordingEntityProvider.HistoryRecordingEntityProvider;
|
|
40
|
+
exports.HistoryRecordingCatalogProcessor = HistoryRecordingCatalogProcessor.HistoryRecordingCatalogProcessor;
|
|
41
|
+
exports.reconcile = reconcile.reconcile;
|
|
42
|
+
exports.CatalogServiceEntityFetcher = CatalogServiceEntityFetcher.CatalogServiceEntityFetcher;
|
|
43
|
+
exports.default = index;
|
|
44
|
+
//# sourceMappingURL=index.cjs.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.cjs.js","sources":["../src/index.ts"],"sourcesContent":["import { createBackendFeatureLoader } from '@backstage/backend-plugin-api';\nimport { historyStoreServiceFactory as defaultHistoryStoreServiceFactory } from '@kunickiaj/catalog-history-backend';\nimport { catalogModuleHistory } from './module/catalogModuleHistory';\n\nexport default createBackendFeatureLoader({\n loader() {\n return [defaultHistoryStoreServiceFactory, catalogModuleHistory];\n },\n});\n\n// Store contracts moved to @kunickiaj/catalog-history-node; re-exported here\n// for compatibility with existing imports.\n/** @deprecated Import from `@kunickiaj/catalog-history-node` instead; these re-exports will be removed in a future release. */\nexport type {\n CaptureSource,\n CurrentEtag,\n CycleInput,\n EntityRow,\n HistoryStore,\n MutationType,\n} from '@kunickiaj/catalog-history-node';\n/** @deprecated Import from `@kunickiaj/catalog-history-node` instead; this re-export will be removed in a future release. */\nexport { RECONCILER_PROVIDER } from '@kunickiaj/catalog-history-node';\n// Storage moved to @kunickiaj/catalog-history-backend; re-exported here for\n// compatibility with existing imports.\n/** @deprecated Import from `@kunickiaj/catalog-history-backend` instead; these re-exports will be removed in a future release. */\nexport {\n ensureSchema,\n historyStoreServiceFactory,\n PostgresHistoryStore,\n} from '@kunickiaj/catalog-history-backend';\nexport { entityToRow } from './mapping/entityToRow';\nexport {\n HistoryRecordingEntityProvider,\n type HistoryRecordingEntityProviderOptions,\n} from './provider/HistoryRecordingEntityProvider';\nexport {\n HistoryRecordingCatalogProcessor,\n type HistoryRecordingCatalogProcessorOptions,\n} from './processor/HistoryRecordingCatalogProcessor';\nexport { reconcile, type ReconcileOptions } from './reconciler/reconcile';\nexport type { EntityFetcher } from './reconciler/EntityFetcher';\nexport { CatalogServiceEntityFetcher } from './reconciler/CatalogServiceEntityFetcher';\nexport { catalogModuleHistory } from './module/catalogModuleHistory';\n"],"names":["createBackendFeatureLoader","defaultHistoryStoreServiceFactory","catalogModuleHistory"],"mappings":";;;;;;;;;;;;;;AAIA,YAAeA,2CAAA,CAA2B;AAAA,EACxC,MAAA,GAAS;AACP,IAAA,OAAO,CAACC,kDAAmCC,yCAAoB,CAAA;AAAA,EACjE;AACF,CAAC,CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
import * as _backstage_backend_plugin_api from '@backstage/backend-plugin-api';
|
|
2
|
+
import { LoggerService, AuthService } from '@backstage/backend-plugin-api';
|
|
3
|
+
import { EntityRow, HistoryStore } from '@kunickiaj/catalog-history-node';
|
|
4
|
+
export { CaptureSource, CurrentEtag, CycleInput, EntityRow, HistoryStore, MutationType, RECONCILER_PROVIDER } from '@kunickiaj/catalog-history-node';
|
|
5
|
+
export { PostgresHistoryStore, ensureSchema, historyStoreServiceFactory } from '@kunickiaj/catalog-history-backend';
|
|
6
|
+
import { Entity } from '@backstage/catalog-model';
|
|
7
|
+
import { EntityProvider, EntityProviderConnection, CatalogProcessor, LocationSpec, CatalogProcessorEmit, CatalogProcessorCache, CatalogService } from '@backstage/plugin-catalog-node';
|
|
8
|
+
|
|
9
|
+
declare function entityToRow(entity: Entity): EntityRow;
|
|
10
|
+
|
|
11
|
+
type ForceFullEveryDuration = {
|
|
12
|
+
days?: number;
|
|
13
|
+
hours?: number;
|
|
14
|
+
minutes?: number;
|
|
15
|
+
seconds?: number;
|
|
16
|
+
};
|
|
17
|
+
type HistoryRecordingEntityProviderOptions = {
|
|
18
|
+
inner: EntityProvider;
|
|
19
|
+
store: HistoryStore;
|
|
20
|
+
logger: LoggerService;
|
|
21
|
+
/**
|
|
22
|
+
* When true, full mutations with a history baseline are forwarded as deltas.
|
|
23
|
+
* Every incoming deferred entry is forwarded unchanged; the catalog compares
|
|
24
|
+
* its full entity content and location key. History refs are used only to
|
|
25
|
+
* infer removals, never to filter incoming entries by entity etag.
|
|
26
|
+
*
|
|
27
|
+
* Off by default. Combine with {@link forceFullEvery} when you want a
|
|
28
|
+
* periodic re-convergence safety valve.
|
|
29
|
+
*
|
|
30
|
+
* History records the original mutation type using its existing entity-etag
|
|
31
|
+
* comparison. Its counts can differ from catalog changes. Conversion does not
|
|
32
|
+
* avoid catalog comparisons for unchanged entries or provide deletion safety.
|
|
33
|
+
*/
|
|
34
|
+
convertFullToDelta?: boolean;
|
|
35
|
+
/**
|
|
36
|
+
* When {@link convertFullToDelta} is on, this controls how often a real
|
|
37
|
+
* full mutation is allowed through unchanged. Acts as a safety valve so
|
|
38
|
+
* the catalog can periodically reconcile removals against its own provider
|
|
39
|
+
* refs when history refs drift. The first full after this wrapper starts is
|
|
40
|
+
* also forwarded unchanged, even when history already has a baseline.
|
|
41
|
+
*
|
|
42
|
+
* If unset, full mutations with a nonempty history baseline are converted.
|
|
43
|
+
* An empty history baseline always forwards the full mutation as-is.
|
|
44
|
+
*/
|
|
45
|
+
forceFullEvery?: ForceFullEveryDuration;
|
|
46
|
+
/**
|
|
47
|
+
* When false, the wrapper becomes a pure passthrough: mutations are
|
|
48
|
+
* forwarded to the catalog untouched and nothing is recorded to history.
|
|
49
|
+
* Lets consumers wire the flag from `catalog.history.provider.enabled`
|
|
50
|
+
* at backend setup without needing config access inside the wrapper.
|
|
51
|
+
*
|
|
52
|
+
* Defaults to true.
|
|
53
|
+
*/
|
|
54
|
+
enabled?: boolean;
|
|
55
|
+
};
|
|
56
|
+
/**
|
|
57
|
+
* Wraps an EntityProvider so every applyMutation call is mirrored to a
|
|
58
|
+
* HistoryStore. The inner provider's normal write to Backstage's catalog
|
|
59
|
+
* happens first and is unaffected by store failures; history recording is
|
|
60
|
+
* best-effort and its errors are swallowed.
|
|
61
|
+
*
|
|
62
|
+
* Both `full` and `delta` mutations are recorded, one cycle per
|
|
63
|
+
* applyMutation call. `delta` mutations record the explicit added/removed
|
|
64
|
+
* lists; full mutations additionally infer deletes for refs present in
|
|
65
|
+
* history but missing from the incoming entity set.
|
|
66
|
+
*
|
|
67
|
+
* Bursty delta coalescing (debounce a burst of webhook events into one
|
|
68
|
+
* cycle) is deferred to a later phase; today every applyMutation
|
|
69
|
+
* invocation produces its own cycle.
|
|
70
|
+
*
|
|
71
|
+
* When {@link HistoryRecordingEntityProviderOptions.convertFullToDelta} is
|
|
72
|
+
* set, full mutations are converted to deltas before forwarding to the
|
|
73
|
+
* catalog, using history refs to infer removals. Incoming entries are never
|
|
74
|
+
* filtered by history etags, which cannot describe location-key-only changes
|
|
75
|
+
* or every content change that the catalog detects.
|
|
76
|
+
*/
|
|
77
|
+
declare class HistoryRecordingEntityProvider implements EntityProvider {
|
|
78
|
+
private readonly inner;
|
|
79
|
+
private readonly store;
|
|
80
|
+
private readonly logger;
|
|
81
|
+
private readonly convertFullToDelta;
|
|
82
|
+
private readonly forceFullEveryMs?;
|
|
83
|
+
private readonly enabled;
|
|
84
|
+
private lastForwardedFullAt?;
|
|
85
|
+
constructor(options: HistoryRecordingEntityProviderOptions);
|
|
86
|
+
getProviderName(): string;
|
|
87
|
+
connect(connection: EntityProviderConnection): Promise<void>;
|
|
88
|
+
private maybeConvertToDelta;
|
|
89
|
+
private shouldForceFull;
|
|
90
|
+
private recordMutation;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
type HistoryRecordingCatalogProcessorOptions = {
|
|
94
|
+
store: HistoryStore;
|
|
95
|
+
logger: LoggerService;
|
|
96
|
+
maxBatchSize?: number;
|
|
97
|
+
flushIntervalMs?: number;
|
|
98
|
+
};
|
|
99
|
+
/**
|
|
100
|
+
* Records processor-layer catalog history without changing catalog behavior.
|
|
101
|
+
*
|
|
102
|
+
* Catalog processors cannot observe deletes; deletion truth comes from the
|
|
103
|
+
* provider and reconciler layers. Registration order across independently-added
|
|
104
|
+
* backend modules is also not enforceable, so this layer may observe pre-final
|
|
105
|
+
* content. The reconciler layer remains the backstop; see
|
|
106
|
+
* docs/adr/2026-07-01-entity-capture-layers.md.
|
|
107
|
+
*
|
|
108
|
+
* The per-cycle `n_unchanged` aggregate is approximate: unchanged
|
|
109
|
+
* observations never trigger a flush by themselves, so their count is
|
|
110
|
+
* attributed to the next cycle that records actual changes.
|
|
111
|
+
*/
|
|
112
|
+
declare class HistoryRecordingCatalogProcessor implements CatalogProcessor {
|
|
113
|
+
private readonly options;
|
|
114
|
+
private readonly maxBatchSize;
|
|
115
|
+
private readonly flushIntervalMs;
|
|
116
|
+
private currentEtags;
|
|
117
|
+
private currentEtagVersions;
|
|
118
|
+
private seedPromise;
|
|
119
|
+
private batchOriginalEtags;
|
|
120
|
+
private batchOriginalEtagVersions;
|
|
121
|
+
private inserts;
|
|
122
|
+
private updates;
|
|
123
|
+
private unchangedCount;
|
|
124
|
+
private batchStartedAt;
|
|
125
|
+
private flushTimer;
|
|
126
|
+
private flushPromise;
|
|
127
|
+
constructor(options: HistoryRecordingCatalogProcessorOptions);
|
|
128
|
+
getProcessorName(): string;
|
|
129
|
+
postProcessEntity(entity: Entity, _location: LocationSpec, _emit: CatalogProcessorEmit, _cache: CatalogProcessorCache): Promise<Entity>;
|
|
130
|
+
flush(): Promise<void>;
|
|
131
|
+
stop(): Promise<void>;
|
|
132
|
+
private getCurrentEtags;
|
|
133
|
+
private ensureBatchStarted;
|
|
134
|
+
private flushBuffered;
|
|
135
|
+
private bufferLength;
|
|
136
|
+
private clearFlushTimer;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Minimal interface for fetching the current catalog snapshot. Used by the
|
|
141
|
+
* reconciler so it can be driven either by Backstage's in-process catalog
|
|
142
|
+
* service (the default deployment) or by the HTTP CatalogClient (the
|
|
143
|
+
* external CronJob path).
|
|
144
|
+
*/
|
|
145
|
+
interface EntityFetcher {
|
|
146
|
+
getEntities(): Promise<Entity[]>;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
type ReconcileOptions = {
|
|
150
|
+
fetcher: EntityFetcher;
|
|
151
|
+
store: HistoryStore;
|
|
152
|
+
logger: LoggerService;
|
|
153
|
+
};
|
|
154
|
+
/**
|
|
155
|
+
* Snapshots the catalog via the EntityFetcher, diffs it against the
|
|
156
|
+
* reconciler's own previously recorded state, and records a single cycle
|
|
157
|
+
* attributed to provider='reconciler' / source='reconciler' summarizing
|
|
158
|
+
* the changes in served catalog truth.
|
|
159
|
+
*
|
|
160
|
+
* The baseline is intentionally scoped to source='reconciler'. Etags from
|
|
161
|
+
* other capture layers are computed over different content (provider rows
|
|
162
|
+
* over pre-processing envelopes, processing rows over pre-stitch output),
|
|
163
|
+
* so comparing the served-catalog etag against them would report phantom
|
|
164
|
+
* updates on every real change. Each source keeps its own baseline; the
|
|
165
|
+
* first reconcile run therefore records the entire catalog as inserts —
|
|
166
|
+
* an expected one-time cost, same as first enabling processing capture.
|
|
167
|
+
*
|
|
168
|
+
* Records a heartbeat cycle (no row changes) when the served catalog and
|
|
169
|
+
* the reconciler's baseline agree, so an operator can tell from the
|
|
170
|
+
* cycles table that the reconciler is running on schedule.
|
|
171
|
+
*/
|
|
172
|
+
declare function reconcile(opts: ReconcileOptions): Promise<void>;
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* EntityFetcher used by the scheduled in-process reconciler mode. The HTTP
|
|
176
|
+
* CatalogClient path remains available for the external CLI.
|
|
177
|
+
*/
|
|
178
|
+
declare class CatalogServiceEntityFetcher implements EntityFetcher {
|
|
179
|
+
private readonly catalog;
|
|
180
|
+
private readonly auth;
|
|
181
|
+
constructor(options: {
|
|
182
|
+
catalog: CatalogService;
|
|
183
|
+
auth: AuthService;
|
|
184
|
+
});
|
|
185
|
+
getEntities(): Promise<Entity[]>;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Backstage backend module for the catalog-history plugin. On init it
|
|
190
|
+
* resolves the history store from Backstage DI and prepares it for use. When
|
|
191
|
+
* processor-layer capture is enabled, it also registers a history processor
|
|
192
|
+
* with the catalog processing extension point.
|
|
193
|
+
*
|
|
194
|
+
* Wrapping individual EntityProviders for cycle recording is handled at
|
|
195
|
+
* backend wiring time via the exported `HistoryRecordingEntityProvider`.
|
|
196
|
+
*
|
|
197
|
+
* Reconciliation is opt-in and can be scheduled in-process via Backstage's
|
|
198
|
+
* scheduler service. The external CLI shim (`bin/reconcile-catalog-history.js`)
|
|
199
|
+
* also remains available for ad-hoc drift detection or external CronJobs.
|
|
200
|
+
*
|
|
201
|
+
* Config (all optional, sensible defaults):
|
|
202
|
+
*
|
|
203
|
+
* ```yaml
|
|
204
|
+
* catalog:
|
|
205
|
+
* history:
|
|
206
|
+
* enabled: true # set to false to skip schema bootstrap entirely
|
|
207
|
+
* database: # deprecated compatibility override honored by the default historyStoreServiceFactory
|
|
208
|
+
* client: pg
|
|
209
|
+
* connection: ${PG_HISTORY_URL}
|
|
210
|
+
* provider:
|
|
211
|
+
* enabled: true # gates provider-layer recording wrapper wiring
|
|
212
|
+
* processing:
|
|
213
|
+
* enabled: false # opt-in CatalogProcessor capture
|
|
214
|
+
* reconciler:
|
|
215
|
+
* enabled: false # opt-in scheduled in-process reconciliation
|
|
216
|
+
* schedule:
|
|
217
|
+
* frequency: { minutes: 30 }
|
|
218
|
+
* timeout: { minutes: 5 }
|
|
219
|
+
* initialDelay: { seconds: 30 }
|
|
220
|
+
* ```
|
|
221
|
+
*/
|
|
222
|
+
declare const catalogModuleHistory: _backstage_backend_plugin_api.BackendFeature;
|
|
223
|
+
|
|
224
|
+
declare const _default: _backstage_backend_plugin_api.BackendFeature;
|
|
225
|
+
|
|
226
|
+
export { CatalogServiceEntityFetcher, HistoryRecordingCatalogProcessor, HistoryRecordingEntityProvider, catalogModuleHistory, _default as default, entityToRow, reconcile };
|
|
227
|
+
export type { EntityFetcher, HistoryRecordingCatalogProcessorOptions, HistoryRecordingEntityProviderOptions, ReconcileOptions };
|