@backstage/plugin-techdocs-backend 1.10.14-next.1 → 1.10.14-next.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/alpha/package.json +1 -1
- package/dist/DocsBuilder/BuildMetadataStorage.cjs.js +30 -0
- package/dist/DocsBuilder/BuildMetadataStorage.cjs.js.map +1 -0
- package/dist/DocsBuilder/builder.cjs.js +172 -0
- package/dist/DocsBuilder/builder.cjs.js.map +1 -0
- package/dist/alpha.cjs.js +3 -120
- package/dist/alpha.cjs.js.map +1 -1
- package/dist/cache/TechDocsCache.cjs.js +70 -0
- package/dist/cache/TechDocsCache.cjs.js.map +1 -0
- package/dist/cache/cacheMiddleware.cjs.js +52 -0
- package/dist/cache/cacheMiddleware.cjs.js.map +1 -0
- package/dist/index.cjs.js +6 -866
- package/dist/index.cjs.js.map +1 -1
- package/dist/plugin.cjs.js +130 -0
- package/dist/plugin.cjs.js.map +1 -0
- package/dist/search/DefaultTechDocsCollator.cjs.js +144 -0
- package/dist/search/DefaultTechDocsCollator.cjs.js.map +1 -0
- package/dist/search/index.cjs.js +15 -0
- package/dist/search/index.cjs.js.map +1 -0
- package/dist/service/CachedEntityLoader.cjs.js +41 -0
- package/dist/service/CachedEntityLoader.cjs.js.map +1 -0
- package/dist/service/DefaultDocsBuildStrategy.cjs.js +19 -0
- package/dist/service/DefaultDocsBuildStrategy.cjs.js.map +1 -0
- package/dist/service/DocsSynchronizer.cjs.js +197 -0
- package/dist/service/DocsSynchronizer.cjs.js.map +1 -0
- package/dist/service/router.cjs.js +219 -0
- package/dist/service/router.cjs.js.map +1 -0
- package/package.json +10 -10
package/dist/index.cjs.js
CHANGED
|
@@ -1,875 +1,15 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
var
|
|
4
|
-
var
|
|
5
|
-
var catalogModel = require('@backstage/catalog-model');
|
|
6
|
-
var errors = require('@backstage/errors');
|
|
3
|
+
var router = require('./service/router.cjs.js');
|
|
4
|
+
var index = require('./search/index.cjs.js');
|
|
7
5
|
var pluginTechdocsNode = require('@backstage/plugin-techdocs-node');
|
|
8
|
-
var
|
|
9
|
-
var integration = require('@backstage/integration');
|
|
10
|
-
var fetch = require('node-fetch');
|
|
11
|
-
var pLimit = require('p-limit');
|
|
12
|
-
var stream = require('stream');
|
|
13
|
-
var winston = require('winston');
|
|
14
|
-
var fs = require('fs-extra');
|
|
15
|
-
var os = require('os');
|
|
16
|
-
var path = require('path');
|
|
17
|
-
var unescape = require('lodash/unescape');
|
|
18
|
-
var alpha = require('@backstage/plugin-catalog-common/alpha');
|
|
19
|
-
var pluginTechdocsCommon = require('@backstage/plugin-techdocs-common');
|
|
20
|
-
var pluginSearchBackendModuleTechdocs = require('@backstage/plugin-search-backend-module-techdocs');
|
|
6
|
+
var DefaultTechDocsCollator = require('./search/DefaultTechDocsCollator.cjs.js');
|
|
21
7
|
|
|
22
|
-
function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e : { default: e }; }
|
|
23
8
|
|
|
24
|
-
function _interopNamespaceCompat(e) {
|
|
25
|
-
if (e && typeof e === 'object' && 'default' in e) return e;
|
|
26
|
-
var n = Object.create(null);
|
|
27
|
-
if (e) {
|
|
28
|
-
Object.keys(e).forEach(function (k) {
|
|
29
|
-
if (k !== 'default') {
|
|
30
|
-
var d = Object.getOwnPropertyDescriptor(e, k);
|
|
31
|
-
Object.defineProperty(n, k, d.get ? d : {
|
|
32
|
-
enumerable: true,
|
|
33
|
-
get: function () { return e[k]; }
|
|
34
|
-
});
|
|
35
|
-
}
|
|
36
|
-
});
|
|
37
|
-
}
|
|
38
|
-
n.default = e;
|
|
39
|
-
return Object.freeze(n);
|
|
40
|
-
}
|
|
41
9
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
var winston__namespace = /*#__PURE__*/_interopNamespaceCompat(winston);
|
|
46
|
-
var fs__default = /*#__PURE__*/_interopDefaultCompat(fs);
|
|
47
|
-
var os__default = /*#__PURE__*/_interopDefaultCompat(os);
|
|
48
|
-
var path__default = /*#__PURE__*/_interopDefaultCompat(path);
|
|
49
|
-
var unescape__default = /*#__PURE__*/_interopDefaultCompat(unescape);
|
|
50
|
-
|
|
51
|
-
const lastUpdatedRecord = {};
|
|
52
|
-
class BuildMetadataStorage {
|
|
53
|
-
entityUid;
|
|
54
|
-
lastUpdatedRecord;
|
|
55
|
-
constructor(entityUid) {
|
|
56
|
-
this.entityUid = entityUid;
|
|
57
|
-
this.lastUpdatedRecord = lastUpdatedRecord;
|
|
58
|
-
}
|
|
59
|
-
setLastUpdated() {
|
|
60
|
-
this.lastUpdatedRecord[this.entityUid] = Date.now();
|
|
61
|
-
}
|
|
62
|
-
getLastUpdated() {
|
|
63
|
-
return this.lastUpdatedRecord[this.entityUid];
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
const shouldCheckForUpdate = (entityUid) => {
|
|
67
|
-
const lastUpdated = new BuildMetadataStorage(entityUid).getLastUpdated();
|
|
68
|
-
if (lastUpdated) {
|
|
69
|
-
if (Date.now() - lastUpdated < 60 * 1e3) {
|
|
70
|
-
return false;
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
return true;
|
|
74
|
-
};
|
|
75
|
-
|
|
76
|
-
class DocsBuilder {
|
|
77
|
-
preparer;
|
|
78
|
-
generator;
|
|
79
|
-
publisher;
|
|
80
|
-
entity;
|
|
81
|
-
logger;
|
|
82
|
-
config;
|
|
83
|
-
scmIntegrations;
|
|
84
|
-
logStream;
|
|
85
|
-
cache;
|
|
86
|
-
constructor({
|
|
87
|
-
preparers,
|
|
88
|
-
generators,
|
|
89
|
-
publisher,
|
|
90
|
-
entity,
|
|
91
|
-
logger,
|
|
92
|
-
config,
|
|
93
|
-
scmIntegrations,
|
|
94
|
-
logStream,
|
|
95
|
-
cache
|
|
96
|
-
}) {
|
|
97
|
-
this.preparer = preparers.get(entity);
|
|
98
|
-
this.generator = generators.get(entity);
|
|
99
|
-
this.publisher = publisher;
|
|
100
|
-
this.entity = entity;
|
|
101
|
-
this.logger = logger;
|
|
102
|
-
this.config = config;
|
|
103
|
-
this.scmIntegrations = scmIntegrations;
|
|
104
|
-
this.logStream = logStream;
|
|
105
|
-
this.cache = cache;
|
|
106
|
-
}
|
|
107
|
-
/**
|
|
108
|
-
* Build the docs and return whether they have been newly generated or have been cached
|
|
109
|
-
* @returns true, if the docs have been built. false, if the cached docs are still up-to-date.
|
|
110
|
-
*/
|
|
111
|
-
async build() {
|
|
112
|
-
if (!this.entity.metadata.uid) {
|
|
113
|
-
throw new Error(
|
|
114
|
-
"Trying to build documentation for entity not in software catalog"
|
|
115
|
-
);
|
|
116
|
-
}
|
|
117
|
-
this.logger.info(
|
|
118
|
-
`Step 1 of 3: Preparing docs for entity ${catalogModel.stringifyEntityRef(
|
|
119
|
-
this.entity
|
|
120
|
-
)}`
|
|
121
|
-
);
|
|
122
|
-
let storedEtag;
|
|
123
|
-
if (await this.publisher.hasDocsBeenGenerated(this.entity)) {
|
|
124
|
-
try {
|
|
125
|
-
storedEtag = (await this.publisher.fetchTechDocsMetadata({
|
|
126
|
-
namespace: this.entity.metadata.namespace ?? catalogModel.DEFAULT_NAMESPACE,
|
|
127
|
-
kind: this.entity.kind,
|
|
128
|
-
name: this.entity.metadata.name
|
|
129
|
-
})).etag;
|
|
130
|
-
} catch (err) {
|
|
131
|
-
this.logger.warn(
|
|
132
|
-
`Unable to read techdocs_metadata.json, proceeding with fresh build, error ${err}.`
|
|
133
|
-
);
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
let preparedDir;
|
|
137
|
-
let newEtag;
|
|
138
|
-
try {
|
|
139
|
-
const preparerResponse = await this.preparer.prepare(this.entity, {
|
|
140
|
-
etag: storedEtag,
|
|
141
|
-
logger: this.logger
|
|
142
|
-
});
|
|
143
|
-
preparedDir = preparerResponse.preparedDir;
|
|
144
|
-
newEtag = preparerResponse.etag;
|
|
145
|
-
} catch (err) {
|
|
146
|
-
if (errors.isError(err) && err.name === "NotModifiedError") {
|
|
147
|
-
new BuildMetadataStorage(this.entity.metadata.uid).setLastUpdated();
|
|
148
|
-
this.logger.debug(
|
|
149
|
-
`Docs for ${catalogModel.stringifyEntityRef(
|
|
150
|
-
this.entity
|
|
151
|
-
)} are unmodified. Using cache, skipping generate and prepare`
|
|
152
|
-
);
|
|
153
|
-
return false;
|
|
154
|
-
}
|
|
155
|
-
throw err;
|
|
156
|
-
}
|
|
157
|
-
this.logger.info(
|
|
158
|
-
`Prepare step completed for entity ${catalogModel.stringifyEntityRef(
|
|
159
|
-
this.entity
|
|
160
|
-
)}, stored at ${preparedDir}`
|
|
161
|
-
);
|
|
162
|
-
this.logger.info(
|
|
163
|
-
`Step 2 of 3: Generating docs for entity ${catalogModel.stringifyEntityRef(
|
|
164
|
-
this.entity
|
|
165
|
-
)}`
|
|
166
|
-
);
|
|
167
|
-
const workingDir = this.config.getOptionalString(
|
|
168
|
-
"backend.workingDirectory"
|
|
169
|
-
);
|
|
170
|
-
const tmpdirPath = workingDir || os__default.default.tmpdir();
|
|
171
|
-
const tmpdirResolvedPath = fs__default.default.realpathSync(tmpdirPath);
|
|
172
|
-
const outputDir = await fs__default.default.mkdtemp(
|
|
173
|
-
path__default.default.join(tmpdirResolvedPath, "techdocs-tmp-")
|
|
174
|
-
);
|
|
175
|
-
const parsedLocationAnnotation = pluginTechdocsNode.getLocationForEntity(
|
|
176
|
-
this.entity,
|
|
177
|
-
this.scmIntegrations
|
|
178
|
-
);
|
|
179
|
-
await this.generator.run({
|
|
180
|
-
inputDir: preparedDir,
|
|
181
|
-
outputDir,
|
|
182
|
-
parsedLocationAnnotation,
|
|
183
|
-
etag: newEtag,
|
|
184
|
-
logger: this.logger,
|
|
185
|
-
logStream: this.logStream,
|
|
186
|
-
siteOptions: {
|
|
187
|
-
name: this.entity.metadata.title ?? this.entity.metadata.name
|
|
188
|
-
}
|
|
189
|
-
});
|
|
190
|
-
if (this.preparer.shouldCleanPreparedDirectory()) {
|
|
191
|
-
this.logger.debug(
|
|
192
|
-
`Removing prepared directory ${preparedDir} since the site has been generated`
|
|
193
|
-
);
|
|
194
|
-
try {
|
|
195
|
-
fs__default.default.remove(preparedDir);
|
|
196
|
-
} catch (error) {
|
|
197
|
-
errors.assertError(error);
|
|
198
|
-
this.logger.debug(`Error removing prepared directory ${error.message}`);
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
this.logger.info(
|
|
202
|
-
`Step 3 of 3: Publishing docs for entity ${catalogModel.stringifyEntityRef(
|
|
203
|
-
this.entity
|
|
204
|
-
)}`
|
|
205
|
-
);
|
|
206
|
-
const published = await this.publisher.publish({
|
|
207
|
-
entity: this.entity,
|
|
208
|
-
directory: outputDir
|
|
209
|
-
});
|
|
210
|
-
if (this.cache && published && published?.objects?.length) {
|
|
211
|
-
this.logger.debug(
|
|
212
|
-
`Invalidating ${published.objects.length} cache objects`
|
|
213
|
-
);
|
|
214
|
-
await this.cache.invalidateMultiple(published.objects);
|
|
215
|
-
}
|
|
216
|
-
try {
|
|
217
|
-
fs__default.default.remove(outputDir);
|
|
218
|
-
this.logger.debug(
|
|
219
|
-
`Removing generated directory ${outputDir} since the site has been published`
|
|
220
|
-
);
|
|
221
|
-
} catch (error) {
|
|
222
|
-
errors.assertError(error);
|
|
223
|
-
this.logger.debug(`Error removing generated directory ${error.message}`);
|
|
224
|
-
}
|
|
225
|
-
new BuildMetadataStorage(this.entity.metadata.uid).setLastUpdated();
|
|
226
|
-
return true;
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
class DocsSynchronizer {
|
|
231
|
-
publisher;
|
|
232
|
-
logger;
|
|
233
|
-
buildLogTransport;
|
|
234
|
-
config;
|
|
235
|
-
scmIntegrations;
|
|
236
|
-
cache;
|
|
237
|
-
buildLimiter;
|
|
238
|
-
constructor({
|
|
239
|
-
publisher,
|
|
240
|
-
logger,
|
|
241
|
-
buildLogTransport,
|
|
242
|
-
config,
|
|
243
|
-
scmIntegrations,
|
|
244
|
-
cache
|
|
245
|
-
}) {
|
|
246
|
-
this.config = config;
|
|
247
|
-
this.logger = logger;
|
|
248
|
-
this.buildLogTransport = buildLogTransport;
|
|
249
|
-
this.publisher = publisher;
|
|
250
|
-
this.scmIntegrations = scmIntegrations;
|
|
251
|
-
this.cache = cache;
|
|
252
|
-
this.buildLimiter = pLimit__default.default(10);
|
|
253
|
-
}
|
|
254
|
-
async doSync({
|
|
255
|
-
responseHandler: { log, error, finish },
|
|
256
|
-
entity,
|
|
257
|
-
preparers,
|
|
258
|
-
generators
|
|
259
|
-
}) {
|
|
260
|
-
const taskLogger = winston__namespace.createLogger({
|
|
261
|
-
level: process.env.LOG_LEVEL || "info",
|
|
262
|
-
format: winston__namespace.format.combine(
|
|
263
|
-
winston__namespace.format.colorize(),
|
|
264
|
-
winston__namespace.format.timestamp(),
|
|
265
|
-
winston__namespace.format.simple()
|
|
266
|
-
),
|
|
267
|
-
defaultMeta: {}
|
|
268
|
-
});
|
|
269
|
-
const logStream = new stream.PassThrough();
|
|
270
|
-
logStream.on("data", async (data) => {
|
|
271
|
-
log(data.toString().trim());
|
|
272
|
-
});
|
|
273
|
-
taskLogger.add(new winston__namespace.transports.Stream({ stream: logStream }));
|
|
274
|
-
if (this.buildLogTransport) {
|
|
275
|
-
taskLogger.add(this.buildLogTransport);
|
|
276
|
-
}
|
|
277
|
-
if (!shouldCheckForUpdate(entity.metadata.uid)) {
|
|
278
|
-
finish({ updated: false });
|
|
279
|
-
return;
|
|
280
|
-
}
|
|
281
|
-
let foundDocs = false;
|
|
282
|
-
try {
|
|
283
|
-
const docsBuilder = new DocsBuilder({
|
|
284
|
-
preparers,
|
|
285
|
-
generators,
|
|
286
|
-
publisher: this.publisher,
|
|
287
|
-
logger: taskLogger,
|
|
288
|
-
entity,
|
|
289
|
-
config: this.config,
|
|
290
|
-
scmIntegrations: this.scmIntegrations,
|
|
291
|
-
logStream,
|
|
292
|
-
cache: this.cache
|
|
293
|
-
});
|
|
294
|
-
const interval = setInterval(() => {
|
|
295
|
-
taskLogger.info(
|
|
296
|
-
"The docs building process is taking a little bit longer to process this entity. Please bear with us."
|
|
297
|
-
);
|
|
298
|
-
}, 1e4);
|
|
299
|
-
const updated = await this.buildLimiter(() => docsBuilder.build());
|
|
300
|
-
clearInterval(interval);
|
|
301
|
-
if (!updated) {
|
|
302
|
-
finish({ updated: false });
|
|
303
|
-
return;
|
|
304
|
-
}
|
|
305
|
-
} catch (e) {
|
|
306
|
-
errors.assertError(e);
|
|
307
|
-
const msg = `Failed to build the docs page for entity ${catalogModel.stringifyEntityRef(
|
|
308
|
-
entity
|
|
309
|
-
)}: ${e.message}`;
|
|
310
|
-
taskLogger.error(msg);
|
|
311
|
-
this.logger.error(msg, e);
|
|
312
|
-
error(e);
|
|
313
|
-
return;
|
|
314
|
-
}
|
|
315
|
-
for (let attempt = 0; attempt < 5; attempt++) {
|
|
316
|
-
if (await this.publisher.hasDocsBeenGenerated(entity)) {
|
|
317
|
-
foundDocs = true;
|
|
318
|
-
break;
|
|
319
|
-
}
|
|
320
|
-
await new Promise((r) => setTimeout(r, 1e3));
|
|
321
|
-
}
|
|
322
|
-
if (!foundDocs) {
|
|
323
|
-
this.logger.error(
|
|
324
|
-
"Published files are taking longer to show up in storage. Something went wrong."
|
|
325
|
-
);
|
|
326
|
-
error(
|
|
327
|
-
new errors.NotFoundError(
|
|
328
|
-
"Sorry! It took too long for the generated docs to show up in storage. Are you sure the docs project is generating an `index.html` file? Otherwise, check back later."
|
|
329
|
-
)
|
|
330
|
-
);
|
|
331
|
-
return;
|
|
332
|
-
}
|
|
333
|
-
finish({ updated: true });
|
|
334
|
-
}
|
|
335
|
-
async doCacheSync({
|
|
336
|
-
responseHandler: { finish },
|
|
337
|
-
discovery,
|
|
338
|
-
token,
|
|
339
|
-
entity
|
|
340
|
-
}) {
|
|
341
|
-
if (!shouldCheckForUpdate(entity.metadata.uid) || !this.cache) {
|
|
342
|
-
finish({ updated: false });
|
|
343
|
-
return;
|
|
344
|
-
}
|
|
345
|
-
const baseUrl = await discovery.getBaseUrl("techdocs");
|
|
346
|
-
const namespace = entity.metadata?.namespace || catalogModel.DEFAULT_NAMESPACE;
|
|
347
|
-
const kind = entity.kind;
|
|
348
|
-
const name = entity.metadata.name;
|
|
349
|
-
const legacyPathCasing = this.config.getOptionalBoolean(
|
|
350
|
-
"techdocs.legacyUseCaseSensitiveTripletPaths"
|
|
351
|
-
) || false;
|
|
352
|
-
const tripletPath = `${namespace}/${kind}/${name}`;
|
|
353
|
-
const entityTripletPath = `${legacyPathCasing ? tripletPath : tripletPath.toLocaleLowerCase("en-US")}`;
|
|
354
|
-
try {
|
|
355
|
-
const [sourceMetadata, cachedMetadata] = await Promise.all([
|
|
356
|
-
this.publisher.fetchTechDocsMetadata({ namespace, kind, name }),
|
|
357
|
-
fetch__default.default(
|
|
358
|
-
`${baseUrl}/static/docs/${entityTripletPath}/techdocs_metadata.json`,
|
|
359
|
-
{
|
|
360
|
-
headers: token ? { Authorization: `Bearer ${token}` } : {}
|
|
361
|
-
}
|
|
362
|
-
).then(
|
|
363
|
-
(f) => f.json().catch(() => void 0)
|
|
364
|
-
)
|
|
365
|
-
]);
|
|
366
|
-
if (sourceMetadata.build_timestamp !== cachedMetadata.build_timestamp) {
|
|
367
|
-
const files = [
|
|
368
|
-
.../* @__PURE__ */ new Set([
|
|
369
|
-
...sourceMetadata.files || [],
|
|
370
|
-
...cachedMetadata.files || []
|
|
371
|
-
])
|
|
372
|
-
].map((f) => `${entityTripletPath}/${f}`);
|
|
373
|
-
await this.cache.invalidateMultiple(files);
|
|
374
|
-
finish({ updated: true });
|
|
375
|
-
} else {
|
|
376
|
-
finish({ updated: false });
|
|
377
|
-
}
|
|
378
|
-
} catch (e) {
|
|
379
|
-
errors.assertError(e);
|
|
380
|
-
this.logger.error(
|
|
381
|
-
`Error syncing cache for ${entityTripletPath}: ${e.message}`
|
|
382
|
-
);
|
|
383
|
-
finish({ updated: false });
|
|
384
|
-
} finally {
|
|
385
|
-
new BuildMetadataStorage(entity.metadata.uid).setLastUpdated();
|
|
386
|
-
}
|
|
387
|
-
}
|
|
388
|
-
}
|
|
389
|
-
|
|
390
|
-
const createCacheMiddleware = ({
|
|
391
|
-
cache
|
|
392
|
-
}) => {
|
|
393
|
-
const cacheMiddleware = router__default.default();
|
|
394
|
-
cacheMiddleware.use(async (req, res, next) => {
|
|
395
|
-
const socket = res.socket;
|
|
396
|
-
const isCacheable = req.path.startsWith("/static/docs/");
|
|
397
|
-
const isGetRequest = req.method === "GET";
|
|
398
|
-
if (!isCacheable || !socket) {
|
|
399
|
-
next();
|
|
400
|
-
return;
|
|
401
|
-
}
|
|
402
|
-
const reqPath = decodeURI(req.path.match(/\/static\/docs\/(.*)$/)[1]);
|
|
403
|
-
const realEnd = socket.end.bind(socket);
|
|
404
|
-
const realWrite = socket.write.bind(socket);
|
|
405
|
-
let writeToCache = true;
|
|
406
|
-
const chunks = [];
|
|
407
|
-
socket.write = (data, encoding, callback) => {
|
|
408
|
-
chunks.push(Buffer.from(data));
|
|
409
|
-
if (typeof encoding === "function") {
|
|
410
|
-
return realWrite(data, encoding);
|
|
411
|
-
}
|
|
412
|
-
return realWrite(data, encoding, callback);
|
|
413
|
-
};
|
|
414
|
-
socket.on("close", async (hadError) => {
|
|
415
|
-
const content = Buffer.concat(chunks);
|
|
416
|
-
const head = content.toString("utf8", 0, 12);
|
|
417
|
-
if (isGetRequest && writeToCache && !hadError && head.match(/HTTP\/\d\.\d 200/)) {
|
|
418
|
-
await cache.set(reqPath, content);
|
|
419
|
-
}
|
|
420
|
-
});
|
|
421
|
-
const cached = await cache.get(reqPath);
|
|
422
|
-
if (cached) {
|
|
423
|
-
writeToCache = false;
|
|
424
|
-
realEnd(cached);
|
|
425
|
-
return;
|
|
426
|
-
}
|
|
427
|
-
next();
|
|
428
|
-
});
|
|
429
|
-
return cacheMiddleware;
|
|
430
|
-
};
|
|
431
|
-
|
|
432
|
-
class CacheInvalidationError extends errors.CustomErrorBase {
|
|
433
|
-
}
|
|
434
|
-
class TechDocsCache {
|
|
435
|
-
cache;
|
|
436
|
-
logger;
|
|
437
|
-
readTimeout;
|
|
438
|
-
constructor({
|
|
439
|
-
cache,
|
|
440
|
-
logger,
|
|
441
|
-
readTimeout
|
|
442
|
-
}) {
|
|
443
|
-
this.cache = cache;
|
|
444
|
-
this.logger = logger;
|
|
445
|
-
this.readTimeout = readTimeout;
|
|
446
|
-
}
|
|
447
|
-
static fromConfig(config, { cache, logger }) {
|
|
448
|
-
const timeout = config.getOptionalNumber("techdocs.cache.readTimeout");
|
|
449
|
-
const readTimeout = timeout === void 0 ? 1e3 : timeout;
|
|
450
|
-
return new TechDocsCache({ cache, logger, readTimeout });
|
|
451
|
-
}
|
|
452
|
-
async get(path) {
|
|
453
|
-
try {
|
|
454
|
-
const response = await Promise.race([
|
|
455
|
-
this.cache.get(path),
|
|
456
|
-
new Promise((cancelAfter) => setTimeout(cancelAfter, this.readTimeout))
|
|
457
|
-
]);
|
|
458
|
-
if (response !== void 0) {
|
|
459
|
-
this.logger.debug(`Cache hit: ${path}`);
|
|
460
|
-
return Buffer.from(response, "base64");
|
|
461
|
-
}
|
|
462
|
-
this.logger.debug(`Cache miss: ${path}`);
|
|
463
|
-
return response;
|
|
464
|
-
} catch (e) {
|
|
465
|
-
errors.assertError(e);
|
|
466
|
-
this.logger.warn(`Error getting cache entry ${path}: ${e.message}`);
|
|
467
|
-
this.logger.debug(e.message, e);
|
|
468
|
-
return void 0;
|
|
469
|
-
}
|
|
470
|
-
}
|
|
471
|
-
async set(path, data) {
|
|
472
|
-
this.logger.debug(`Writing cache entry for ${path}`);
|
|
473
|
-
this.cache.set(path, data.toString("base64")).catch((e) => this.logger.error("write error", e));
|
|
474
|
-
}
|
|
475
|
-
async invalidate(path) {
|
|
476
|
-
return this.cache.delete(path);
|
|
477
|
-
}
|
|
478
|
-
async invalidateMultiple(paths) {
|
|
479
|
-
const settled = await Promise.allSettled(
|
|
480
|
-
paths.map((path) => this.cache.delete(path))
|
|
481
|
-
);
|
|
482
|
-
const rejected = settled.filter(
|
|
483
|
-
(s) => s.status === "rejected"
|
|
484
|
-
);
|
|
485
|
-
if (rejected.length) {
|
|
486
|
-
throw new CacheInvalidationError(
|
|
487
|
-
"TechDocs cache invalidation error",
|
|
488
|
-
rejected
|
|
489
|
-
);
|
|
490
|
-
}
|
|
491
|
-
return settled;
|
|
492
|
-
}
|
|
493
|
-
}
|
|
494
|
-
|
|
495
|
-
class CachedEntityLoader {
|
|
496
|
-
catalog;
|
|
497
|
-
cache;
|
|
498
|
-
readTimeout = 1e3;
|
|
499
|
-
constructor({ catalog, cache }) {
|
|
500
|
-
this.catalog = catalog;
|
|
501
|
-
this.cache = cache;
|
|
502
|
-
}
|
|
503
|
-
async load(entityRef, token) {
|
|
504
|
-
const cacheKey = this.getCacheKey(entityRef, token);
|
|
505
|
-
let result = await this.getFromCache(cacheKey);
|
|
506
|
-
if (result) {
|
|
507
|
-
return result;
|
|
508
|
-
}
|
|
509
|
-
result = await this.catalog.getEntityByRef(entityRef, { token });
|
|
510
|
-
if (result) {
|
|
511
|
-
this.cache.set(cacheKey, result, { ttl: 5e3 });
|
|
512
|
-
}
|
|
513
|
-
return result;
|
|
514
|
-
}
|
|
515
|
-
async getFromCache(key) {
|
|
516
|
-
return await Promise.race([
|
|
517
|
-
this.cache.get(key),
|
|
518
|
-
new Promise((cancelAfter) => setTimeout(cancelAfter, this.readTimeout))
|
|
519
|
-
]);
|
|
520
|
-
}
|
|
521
|
-
getCacheKey(entityName, token) {
|
|
522
|
-
const key = ["catalog", catalogModel.stringifyEntityRef(entityName)];
|
|
523
|
-
if (token) {
|
|
524
|
-
key.push(token);
|
|
525
|
-
}
|
|
526
|
-
return key.join(":");
|
|
527
|
-
}
|
|
528
|
-
}
|
|
529
|
-
|
|
530
|
-
class DefaultDocsBuildStrategy {
|
|
531
|
-
config;
|
|
532
|
-
constructor(config) {
|
|
533
|
-
this.config = config;
|
|
534
|
-
}
|
|
535
|
-
static fromConfig(config) {
|
|
536
|
-
return new DefaultDocsBuildStrategy(config);
|
|
537
|
-
}
|
|
538
|
-
async shouldBuild(_) {
|
|
539
|
-
return [void 0, "local"].includes(
|
|
540
|
-
this.config.getOptionalString("techdocs.builder")
|
|
541
|
-
);
|
|
542
|
-
}
|
|
543
|
-
}
|
|
544
|
-
|
|
545
|
-
function isOutOfTheBoxOption(opt) {
|
|
546
|
-
return opt.preparers !== void 0;
|
|
547
|
-
}
|
|
548
|
-
async function createRouter(options) {
|
|
549
|
-
const router = router__default.default();
|
|
550
|
-
const { publisher, config, logger, discovery } = options;
|
|
551
|
-
const { auth, httpAuth } = backendCommon.createLegacyAuthAdapters(options);
|
|
552
|
-
const catalogClient$1 = options.catalogClient ?? new catalogClient.CatalogClient({ discoveryApi: discovery });
|
|
553
|
-
const docsBuildStrategy = options.docsBuildStrategy ?? DefaultDocsBuildStrategy.fromConfig(config);
|
|
554
|
-
const buildLogTransport = options.buildLogTransport;
|
|
555
|
-
const entityLoader = new CachedEntityLoader({
|
|
556
|
-
catalog: catalogClient$1,
|
|
557
|
-
cache: options.cache.getClient()
|
|
558
|
-
});
|
|
559
|
-
let cache;
|
|
560
|
-
const defaultTtl = config.getOptionalNumber("techdocs.cache.ttl");
|
|
561
|
-
if (defaultTtl) {
|
|
562
|
-
const cacheClient = options.cache.getClient({ defaultTtl });
|
|
563
|
-
cache = TechDocsCache.fromConfig(config, { cache: cacheClient, logger });
|
|
564
|
-
}
|
|
565
|
-
const scmIntegrations = integration.ScmIntegrations.fromConfig(config);
|
|
566
|
-
const docsSynchronizer = new DocsSynchronizer({
|
|
567
|
-
publisher,
|
|
568
|
-
logger,
|
|
569
|
-
buildLogTransport,
|
|
570
|
-
config,
|
|
571
|
-
scmIntegrations,
|
|
572
|
-
cache
|
|
573
|
-
});
|
|
574
|
-
router.get("/metadata/techdocs/:namespace/:kind/:name", async (req, res) => {
|
|
575
|
-
const { kind, namespace, name } = req.params;
|
|
576
|
-
const entityName = { kind, namespace, name };
|
|
577
|
-
const credentials = await httpAuth.credentials(req);
|
|
578
|
-
const { token } = await auth.getPluginRequestToken({
|
|
579
|
-
onBehalfOf: credentials,
|
|
580
|
-
targetPluginId: "catalog"
|
|
581
|
-
});
|
|
582
|
-
const entity = await entityLoader.load(entityName, token);
|
|
583
|
-
if (!entity) {
|
|
584
|
-
throw new errors.NotFoundError(
|
|
585
|
-
`Unable to get metadata for '${catalogModel.stringifyEntityRef(entityName)}'`
|
|
586
|
-
);
|
|
587
|
-
}
|
|
588
|
-
try {
|
|
589
|
-
const techdocsMetadata = await publisher.fetchTechDocsMetadata(
|
|
590
|
-
entityName
|
|
591
|
-
);
|
|
592
|
-
res.json(techdocsMetadata);
|
|
593
|
-
} catch (err) {
|
|
594
|
-
logger.info(
|
|
595
|
-
`Unable to get metadata for '${catalogModel.stringifyEntityRef(
|
|
596
|
-
entityName
|
|
597
|
-
)}' with error ${err}`
|
|
598
|
-
);
|
|
599
|
-
throw new errors.NotFoundError(
|
|
600
|
-
`Unable to get metadata for '${catalogModel.stringifyEntityRef(entityName)}'`,
|
|
601
|
-
err
|
|
602
|
-
);
|
|
603
|
-
}
|
|
604
|
-
});
|
|
605
|
-
router.get("/metadata/entity/:namespace/:kind/:name", async (req, res) => {
|
|
606
|
-
const { kind, namespace, name } = req.params;
|
|
607
|
-
const entityName = { kind, namespace, name };
|
|
608
|
-
const credentials = await httpAuth.credentials(req);
|
|
609
|
-
const { token } = await auth.getPluginRequestToken({
|
|
610
|
-
onBehalfOf: credentials,
|
|
611
|
-
targetPluginId: "catalog"
|
|
612
|
-
});
|
|
613
|
-
const entity = await entityLoader.load(entityName, token);
|
|
614
|
-
if (!entity) {
|
|
615
|
-
throw new errors.NotFoundError(
|
|
616
|
-
`Unable to get metadata for '${catalogModel.stringifyEntityRef(entityName)}'`
|
|
617
|
-
);
|
|
618
|
-
}
|
|
619
|
-
try {
|
|
620
|
-
const locationMetadata = pluginTechdocsNode.getLocationForEntity(entity, scmIntegrations);
|
|
621
|
-
res.json({ ...entity, locationMetadata });
|
|
622
|
-
} catch (err) {
|
|
623
|
-
logger.info(
|
|
624
|
-
`Unable to get metadata for '${catalogModel.stringifyEntityRef(
|
|
625
|
-
entityName
|
|
626
|
-
)}' with error ${err}`
|
|
627
|
-
);
|
|
628
|
-
throw new errors.NotFoundError(
|
|
629
|
-
`Unable to get metadata for '${catalogModel.stringifyEntityRef(entityName)}'`,
|
|
630
|
-
err
|
|
631
|
-
);
|
|
632
|
-
}
|
|
633
|
-
});
|
|
634
|
-
router.get("/sync/:namespace/:kind/:name", async (req, res) => {
|
|
635
|
-
const { kind, namespace, name } = req.params;
|
|
636
|
-
const credentials = await httpAuth.credentials(req);
|
|
637
|
-
const { token } = await auth.getPluginRequestToken({
|
|
638
|
-
onBehalfOf: credentials,
|
|
639
|
-
targetPluginId: "catalog"
|
|
640
|
-
});
|
|
641
|
-
const entity = await entityLoader.load({ kind, namespace, name }, token);
|
|
642
|
-
if (!entity?.metadata?.uid) {
|
|
643
|
-
throw new errors.NotFoundError("Entity metadata UID missing");
|
|
644
|
-
}
|
|
645
|
-
const responseHandler = createEventStream(res);
|
|
646
|
-
const shouldBuild = await docsBuildStrategy.shouldBuild({ entity });
|
|
647
|
-
if (!shouldBuild) {
|
|
648
|
-
if (cache) {
|
|
649
|
-
const { token: techDocsToken } = await auth.getPluginRequestToken({
|
|
650
|
-
onBehalfOf: await auth.getOwnServiceCredentials(),
|
|
651
|
-
targetPluginId: "techdocs"
|
|
652
|
-
});
|
|
653
|
-
await docsSynchronizer.doCacheSync({
|
|
654
|
-
responseHandler,
|
|
655
|
-
discovery,
|
|
656
|
-
token: techDocsToken,
|
|
657
|
-
entity
|
|
658
|
-
});
|
|
659
|
-
return;
|
|
660
|
-
}
|
|
661
|
-
responseHandler.finish({ updated: false });
|
|
662
|
-
return;
|
|
663
|
-
}
|
|
664
|
-
if (isOutOfTheBoxOption(options)) {
|
|
665
|
-
const { preparers, generators } = options;
|
|
666
|
-
await docsSynchronizer.doSync({
|
|
667
|
-
responseHandler,
|
|
668
|
-
entity,
|
|
669
|
-
preparers,
|
|
670
|
-
generators
|
|
671
|
-
});
|
|
672
|
-
return;
|
|
673
|
-
}
|
|
674
|
-
responseHandler.error(
|
|
675
|
-
new Error(
|
|
676
|
-
"Invalid configuration. docsBuildStrategy.shouldBuild returned 'true', but no 'preparer' was provided to the router initialization."
|
|
677
|
-
)
|
|
678
|
-
);
|
|
679
|
-
});
|
|
680
|
-
if (config.getOptionalBoolean("permission.enabled")) {
|
|
681
|
-
router.use(
|
|
682
|
-
"/static/docs/:namespace/:kind/:name",
|
|
683
|
-
async (req, _res, next) => {
|
|
684
|
-
const { kind, namespace, name } = req.params;
|
|
685
|
-
const entityName = { kind, namespace, name };
|
|
686
|
-
const credentials = await httpAuth.credentials(req, {
|
|
687
|
-
allowLimitedAccess: true
|
|
688
|
-
});
|
|
689
|
-
const { token } = await auth.getPluginRequestToken({
|
|
690
|
-
onBehalfOf: credentials,
|
|
691
|
-
targetPluginId: "catalog"
|
|
692
|
-
});
|
|
693
|
-
const entity = await entityLoader.load(entityName, token);
|
|
694
|
-
if (!entity) {
|
|
695
|
-
throw new errors.NotFoundError(
|
|
696
|
-
`Entity not found for ${catalogModel.stringifyEntityRef(entityName)}`
|
|
697
|
-
);
|
|
698
|
-
}
|
|
699
|
-
next();
|
|
700
|
-
}
|
|
701
|
-
);
|
|
702
|
-
}
|
|
703
|
-
if (cache) {
|
|
704
|
-
router.use(createCacheMiddleware({ logger, cache }));
|
|
705
|
-
}
|
|
706
|
-
router.use("/static/docs", publisher.docsRouter());
|
|
707
|
-
return router;
|
|
708
|
-
}
|
|
709
|
-
function createEventStream(res) {
|
|
710
|
-
res.writeHead(200, {
|
|
711
|
-
Connection: "keep-alive",
|
|
712
|
-
"Cache-Control": "no-cache",
|
|
713
|
-
"Content-Type": "text/event-stream"
|
|
714
|
-
});
|
|
715
|
-
res.socket?.on("close", () => {
|
|
716
|
-
res.end();
|
|
717
|
-
});
|
|
718
|
-
const send = (type, data) => {
|
|
719
|
-
res.write(`event: ${type}
|
|
720
|
-
data: ${JSON.stringify(data)}
|
|
721
|
-
|
|
722
|
-
`);
|
|
723
|
-
if (res.flush) {
|
|
724
|
-
res.flush();
|
|
725
|
-
}
|
|
726
|
-
};
|
|
727
|
-
return {
|
|
728
|
-
log: (data) => {
|
|
729
|
-
send("log", data);
|
|
730
|
-
},
|
|
731
|
-
error: (e) => {
|
|
732
|
-
send("error", e.message);
|
|
733
|
-
res.end();
|
|
734
|
-
},
|
|
735
|
-
finish: (result) => {
|
|
736
|
-
send("finish", result);
|
|
737
|
-
res.end();
|
|
738
|
-
}
|
|
739
|
-
};
|
|
740
|
-
}
|
|
741
|
-
|
|
742
|
-
class DefaultTechDocsCollator {
|
|
743
|
-
constructor(legacyPathCasing, options) {
|
|
744
|
-
this.legacyPathCasing = legacyPathCasing;
|
|
745
|
-
this.options = options;
|
|
746
|
-
}
|
|
747
|
-
type = "techdocs";
|
|
748
|
-
visibilityPermission = alpha.catalogEntityReadPermission;
|
|
749
|
-
static fromConfig(config, options) {
|
|
750
|
-
const legacyPathCasing = config.getOptionalBoolean(
|
|
751
|
-
"techdocs.legacyUseCaseSensitiveTripletPaths"
|
|
752
|
-
) || false;
|
|
753
|
-
return new DefaultTechDocsCollator(legacyPathCasing, options);
|
|
754
|
-
}
|
|
755
|
-
async execute() {
|
|
756
|
-
const {
|
|
757
|
-
parallelismLimit,
|
|
758
|
-
discovery,
|
|
759
|
-
tokenManager,
|
|
760
|
-
catalogClient: catalogClient$1,
|
|
761
|
-
locationTemplate,
|
|
762
|
-
logger
|
|
763
|
-
} = this.options;
|
|
764
|
-
const limit = pLimit__default.default(parallelismLimit ?? 10);
|
|
765
|
-
const techDocsBaseUrl = await discovery.getBaseUrl("techdocs");
|
|
766
|
-
const { token } = await tokenManager.getToken();
|
|
767
|
-
const entities = await (catalogClient$1 ?? new catalogClient.CatalogClient({ discoveryApi: discovery })).getEntities(
|
|
768
|
-
{
|
|
769
|
-
filter: {
|
|
770
|
-
[`metadata.annotations.${pluginTechdocsCommon.TECHDOCS_ANNOTATION}`]: catalogClient.CATALOG_FILTER_EXISTS
|
|
771
|
-
},
|
|
772
|
-
fields: [
|
|
773
|
-
"kind",
|
|
774
|
-
"namespace",
|
|
775
|
-
"metadata.annotations",
|
|
776
|
-
"metadata.name",
|
|
777
|
-
"metadata.title",
|
|
778
|
-
"metadata.namespace",
|
|
779
|
-
"spec.type",
|
|
780
|
-
"spec.lifecycle",
|
|
781
|
-
"relations"
|
|
782
|
-
]
|
|
783
|
-
},
|
|
784
|
-
{ token }
|
|
785
|
-
);
|
|
786
|
-
const docPromises = entities.items.map(
|
|
787
|
-
(entity) => limit(async () => {
|
|
788
|
-
const entityInfo = DefaultTechDocsCollator.handleEntityInfoCasing(
|
|
789
|
-
this.legacyPathCasing ?? false,
|
|
790
|
-
{
|
|
791
|
-
kind: entity.kind,
|
|
792
|
-
namespace: entity.metadata.namespace || "default",
|
|
793
|
-
name: entity.metadata.name
|
|
794
|
-
}
|
|
795
|
-
);
|
|
796
|
-
try {
|
|
797
|
-
const { token: newToken } = await tokenManager.getToken();
|
|
798
|
-
const searchIndexResponse = await fetch__default.default(
|
|
799
|
-
DefaultTechDocsCollator.constructDocsIndexUrl(
|
|
800
|
-
techDocsBaseUrl,
|
|
801
|
-
entityInfo
|
|
802
|
-
),
|
|
803
|
-
{
|
|
804
|
-
headers: {
|
|
805
|
-
Authorization: `Bearer ${newToken}`
|
|
806
|
-
}
|
|
807
|
-
}
|
|
808
|
-
);
|
|
809
|
-
const searchIndex = await searchIndexResponse.json();
|
|
810
|
-
return searchIndex.docs.map((doc) => ({
|
|
811
|
-
title: unescape__default.default(doc.title),
|
|
812
|
-
text: unescape__default.default(doc.text || ""),
|
|
813
|
-
location: this.applyArgsToFormat(
|
|
814
|
-
locationTemplate || "/docs/:namespace/:kind/:name/:path",
|
|
815
|
-
{
|
|
816
|
-
...entityInfo,
|
|
817
|
-
path: doc.location
|
|
818
|
-
}
|
|
819
|
-
),
|
|
820
|
-
path: doc.location,
|
|
821
|
-
...entityInfo,
|
|
822
|
-
entityTitle: entity.metadata.title,
|
|
823
|
-
componentType: entity.spec?.type?.toString() || "other",
|
|
824
|
-
lifecycle: entity.spec?.lifecycle || "",
|
|
825
|
-
owner: getSimpleEntityOwnerString(entity),
|
|
826
|
-
authorization: {
|
|
827
|
-
resourceRef: catalogModel.stringifyEntityRef(entity)
|
|
828
|
-
}
|
|
829
|
-
}));
|
|
830
|
-
} catch (e) {
|
|
831
|
-
logger.debug(
|
|
832
|
-
`Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`,
|
|
833
|
-
e
|
|
834
|
-
);
|
|
835
|
-
return [];
|
|
836
|
-
}
|
|
837
|
-
})
|
|
838
|
-
);
|
|
839
|
-
return (await Promise.all(docPromises)).flat();
|
|
840
|
-
}
|
|
841
|
-
applyArgsToFormat(format, args) {
|
|
842
|
-
let formatted = format;
|
|
843
|
-
for (const [key, value] of Object.entries(args)) {
|
|
844
|
-
formatted = formatted.replace(`:${key}`, value);
|
|
845
|
-
}
|
|
846
|
-
return formatted;
|
|
847
|
-
}
|
|
848
|
-
static constructDocsIndexUrl(techDocsBaseUrl, entityInfo) {
|
|
849
|
-
return `${techDocsBaseUrl}/static/docs/${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}/search/search_index.json`;
|
|
850
|
-
}
|
|
851
|
-
static handleEntityInfoCasing(legacyPaths, entityInfo) {
|
|
852
|
-
return legacyPaths ? entityInfo : Object.entries(entityInfo).reduce((acc, [key, value]) => {
|
|
853
|
-
return { ...acc, [key]: value.toLocaleLowerCase("en-US") };
|
|
854
|
-
}, {});
|
|
855
|
-
}
|
|
856
|
-
}
|
|
857
|
-
function getSimpleEntityOwnerString(entity) {
|
|
858
|
-
if (entity.relations) {
|
|
859
|
-
const owner = entity.relations.find((r) => r.type === catalogModel.RELATION_OWNED_BY);
|
|
860
|
-
if (owner) {
|
|
861
|
-
const { name } = catalogModel.parseEntityRef(owner.targetRef);
|
|
862
|
-
return name;
|
|
863
|
-
}
|
|
864
|
-
}
|
|
865
|
-
return "";
|
|
866
|
-
}
|
|
867
|
-
|
|
868
|
-
const DefaultTechDocsCollatorFactory = pluginSearchBackendModuleTechdocs.DefaultTechDocsCollatorFactory;
|
|
869
|
-
|
|
870
|
-
exports.DefaultTechDocsCollator = DefaultTechDocsCollator;
|
|
871
|
-
exports.DefaultTechDocsCollatorFactory = DefaultTechDocsCollatorFactory;
|
|
872
|
-
exports.createRouter = createRouter;
|
|
10
|
+
exports.createRouter = router.createRouter;
|
|
11
|
+
exports.DefaultTechDocsCollatorFactory = index.DefaultTechDocsCollatorFactory;
|
|
12
|
+
exports.DefaultTechDocsCollator = DefaultTechDocsCollator.DefaultTechDocsCollator;
|
|
873
13
|
Object.keys(pluginTechdocsNode).forEach(function (k) {
|
|
874
14
|
if (k !== 'default' && !Object.prototype.hasOwnProperty.call(exports, k)) Object.defineProperty(exports, k, {
|
|
875
15
|
enumerable: true,
|