@qualcomm-ui/mdx-vite 2.5.1 → 2.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +1185 -746
- package/dist/cli.js.map +4 -4
- package/dist/open-web-ui-knowledge/api.d.ts +153 -0
- package/dist/open-web-ui-knowledge/api.d.ts.map +1 -0
- package/dist/open-web-ui-knowledge/common.d.ts +0 -35
- package/dist/open-web-ui-knowledge/common.d.ts.map +1 -1
- package/dist/open-web-ui-knowledge/download-knowledge.d.ts.map +1 -1
- package/dist/open-web-ui-knowledge/generate-knowledge.d.ts +5 -1
- package/dist/open-web-ui-knowledge/generate-knowledge.d.ts.map +1 -1
- package/dist/open-web-ui-knowledge/knowledge-cleaner.d.ts +10 -0
- package/dist/open-web-ui-knowledge/knowledge-cleaner.d.ts.map +1 -0
- package/dist/open-web-ui-knowledge/load-config-from-env.d.ts.map +1 -1
- package/dist/open-web-ui-knowledge/types.d.ts +3 -8
- package/dist/open-web-ui-knowledge/types.d.ts.map +1 -1
- package/dist/open-web-ui-knowledge/upload-knowledge.d.ts.map +1 -1
- package/dist/tsbuildinfo +1 -1
- package/package.json +2 -2
package/dist/cli.js
CHANGED
|
@@ -5168,138 +5168,359 @@ function addGeneratePageMapCommand() {
|
|
|
5168
5168
|
import { mkdir, writeFile as writeFile2 } from "node:fs/promises";
|
|
5169
5169
|
import { resolve as resolve2 } from "node:path";
|
|
5170
5170
|
|
|
5171
|
-
// src/open-web-ui-knowledge/
|
|
5172
|
-
|
|
5173
|
-
|
|
5174
|
-
const options = program.optsWithGlobals();
|
|
5175
|
-
console.debug(options);
|
|
5176
|
-
if (options.env) {
|
|
5177
|
-
config({ path: options.env });
|
|
5178
|
-
} else {
|
|
5179
|
-
config();
|
|
5180
|
-
}
|
|
5181
|
-
}
|
|
5182
|
-
function getConfigFromEnv() {
|
|
5183
|
-
const openWebUiUrl = process.env.WEB_UI_URL;
|
|
5184
|
-
const openWebUiKey = process.env.WEB_UI_KEY;
|
|
5185
|
-
const knowledgeId = process.env.KNOWLEDGE_ID;
|
|
5186
|
-
if (!openWebUiUrl || !openWebUiKey || !knowledgeId) {
|
|
5187
|
-
throw new Error("WEB_UI_URL, WEB_UI_KEY, and KNOWLEDGE_ID must be set");
|
|
5188
|
-
}
|
|
5189
|
-
return {
|
|
5190
|
-
knowledgeId,
|
|
5191
|
-
webUiKey: openWebUiKey,
|
|
5192
|
-
webUiUrl: openWebUiUrl
|
|
5193
|
-
};
|
|
5171
|
+
// src/open-web-ui-knowledge/api.ts
|
|
5172
|
+
function isErrorResponse(response) {
|
|
5173
|
+
return typeof response === "object" && response !== null && "detail" in response;
|
|
5194
5174
|
}
|
|
5195
|
-
var
|
|
5175
|
+
var FilesApi = class {
|
|
5196
5176
|
config;
|
|
5197
|
-
knowledgeCache = null;
|
|
5198
5177
|
constructor(config2) {
|
|
5199
5178
|
this.config = config2;
|
|
5200
5179
|
}
|
|
5201
5180
|
get headers() {
|
|
5202
5181
|
return {
|
|
5203
|
-
Authorization: `Bearer ${this.config.
|
|
5182
|
+
Authorization: `Bearer ${this.config.apiKey}`
|
|
5204
5183
|
};
|
|
5205
5184
|
}
|
|
5206
|
-
|
|
5207
|
-
|
|
5208
|
-
|
|
5185
|
+
get jsonHeaders() {
|
|
5186
|
+
return {
|
|
5187
|
+
...this.headers,
|
|
5188
|
+
"Content-Type": "application/json"
|
|
5189
|
+
};
|
|
5190
|
+
}
|
|
5191
|
+
async handleResponse(response) {
|
|
5192
|
+
const data = await response.json();
|
|
5193
|
+
if (isErrorResponse(data)) {
|
|
5194
|
+
throw new Error(data.detail);
|
|
5195
|
+
}
|
|
5196
|
+
return data;
|
|
5197
|
+
}
|
|
5198
|
+
async upload(file, filename, options) {
|
|
5199
|
+
const formData = new FormData();
|
|
5200
|
+
let blob;
|
|
5201
|
+
if (file instanceof Blob) {
|
|
5202
|
+
blob = file;
|
|
5203
|
+
} else if (file instanceof ArrayBuffer) {
|
|
5204
|
+
blob = new Blob([file]);
|
|
5205
|
+
} else {
|
|
5206
|
+
const copy = new Uint8Array(file).buffer;
|
|
5207
|
+
blob = new Blob([copy]);
|
|
5208
|
+
}
|
|
5209
|
+
formData.append("file", blob, filename);
|
|
5210
|
+
if (options?.metadata) {
|
|
5211
|
+
formData.append("metadata", JSON.stringify(options.metadata));
|
|
5209
5212
|
}
|
|
5210
|
-
const
|
|
5211
|
-
|
|
5213
|
+
const params = new URLSearchParams();
|
|
5214
|
+
if (options?.process !== void 0) {
|
|
5215
|
+
params.set("process", String(options.process));
|
|
5216
|
+
}
|
|
5217
|
+
if (options?.processInBackground !== void 0) {
|
|
5218
|
+
params.set("process_in_background", String(options.processInBackground));
|
|
5219
|
+
}
|
|
5220
|
+
const url = `${this.config.baseUrl}/api/v1/files/${params.toString() ? `?${params}` : ""}`;
|
|
5221
|
+
const response = await fetch(url, {
|
|
5222
|
+
body: formData,
|
|
5223
|
+
headers: this.headers,
|
|
5224
|
+
method: "POST"
|
|
5225
|
+
});
|
|
5226
|
+
return this.handleResponse(response);
|
|
5227
|
+
}
|
|
5228
|
+
async list(includeContent = true) {
|
|
5229
|
+
const params = new URLSearchParams({ content: String(includeContent) });
|
|
5230
|
+
const response = await fetch(
|
|
5231
|
+
`${this.config.baseUrl}/api/v1/files/?${params}`,
|
|
5212
5232
|
{
|
|
5213
|
-
headers:
|
|
5214
|
-
...this.headers,
|
|
5215
|
-
Accept: "application/json"
|
|
5216
|
-
}
|
|
5233
|
+
headers: this.headers
|
|
5217
5234
|
}
|
|
5218
|
-
)
|
|
5219
|
-
|
|
5220
|
-
|
|
5221
|
-
|
|
5222
|
-
|
|
5235
|
+
);
|
|
5236
|
+
return this.handleResponse(response);
|
|
5237
|
+
}
|
|
5238
|
+
async search(pattern, includeContent = true) {
|
|
5239
|
+
const params = new URLSearchParams({
|
|
5240
|
+
content: String(includeContent),
|
|
5241
|
+
filename: pattern
|
|
5242
|
+
});
|
|
5243
|
+
const response = await fetch(
|
|
5244
|
+
`${this.config.baseUrl}/api/v1/files/search?${params}`,
|
|
5245
|
+
{ headers: this.headers }
|
|
5246
|
+
);
|
|
5247
|
+
if (response.status === 404) {
|
|
5248
|
+
return [];
|
|
5249
|
+
}
|
|
5250
|
+
return this.handleResponse(response);
|
|
5251
|
+
}
|
|
5252
|
+
async deleteAll() {
|
|
5253
|
+
const response = await fetch(`${this.config.baseUrl}/api/v1/files/all`, {
|
|
5254
|
+
headers: this.headers,
|
|
5255
|
+
method: "DELETE"
|
|
5256
|
+
});
|
|
5257
|
+
return this.handleResponse(response);
|
|
5258
|
+
}
|
|
5259
|
+
async getById(id) {
|
|
5260
|
+
const response = await fetch(`${this.config.baseUrl}/api/v1/files/${id}`, {
|
|
5261
|
+
headers: this.headers
|
|
5262
|
+
});
|
|
5263
|
+
return this.handleResponse(response);
|
|
5264
|
+
}
|
|
5265
|
+
async getProcessStatus(id) {
|
|
5266
|
+
const response = await fetch(
|
|
5267
|
+
`${this.config.baseUrl}/api/v1/files/${id}/process/status`,
|
|
5268
|
+
{ headers: this.headers }
|
|
5269
|
+
);
|
|
5270
|
+
return this.handleResponse(response);
|
|
5271
|
+
}
|
|
5272
|
+
async waitForProcessing(id, options) {
|
|
5273
|
+
const maxAttempts = options?.maxAttempts ?? 120;
|
|
5274
|
+
const intervalMs = options?.intervalMs ?? 1e3;
|
|
5275
|
+
for (let i = 0; i < maxAttempts; i++) {
|
|
5276
|
+
const status = await this.getProcessStatus(id);
|
|
5277
|
+
if (status.status === "completed" || status.status === "failed") {
|
|
5278
|
+
return status;
|
|
5279
|
+
}
|
|
5280
|
+
await new Promise((resolve9) => setTimeout(resolve9, intervalMs));
|
|
5223
5281
|
}
|
|
5224
|
-
|
|
5282
|
+
throw new Error(`File processing timed out after ${maxAttempts} attempts`);
|
|
5283
|
+
}
|
|
5284
|
+
async getDataContent(id) {
|
|
5285
|
+
const response = await fetch(
|
|
5286
|
+
`${this.config.baseUrl}/api/v1/files/${id}/data/content`,
|
|
5287
|
+
{ headers: this.headers }
|
|
5288
|
+
);
|
|
5289
|
+
return this.handleResponse(response);
|
|
5225
5290
|
}
|
|
5226
|
-
async
|
|
5227
|
-
const
|
|
5228
|
-
`${this.config.
|
|
5291
|
+
async updateDataContent(id, content) {
|
|
5292
|
+
const response = await fetch(
|
|
5293
|
+
`${this.config.baseUrl}/api/v1/files/${id}/data/content/update`,
|
|
5229
5294
|
{
|
|
5230
|
-
|
|
5231
|
-
|
|
5232
|
-
|
|
5233
|
-
}
|
|
5295
|
+
body: JSON.stringify({ content }),
|
|
5296
|
+
headers: this.jsonHeaders,
|
|
5297
|
+
method: "POST"
|
|
5234
5298
|
}
|
|
5235
|
-
)
|
|
5236
|
-
return
|
|
5299
|
+
);
|
|
5300
|
+
return this.handleResponse(response);
|
|
5237
5301
|
}
|
|
5238
|
-
async
|
|
5302
|
+
async getContent(id, asAttachment = false) {
|
|
5303
|
+
const params = new URLSearchParams({ attachment: String(asAttachment) });
|
|
5239
5304
|
return fetch(
|
|
5240
|
-
`${this.config.
|
|
5305
|
+
`${this.config.baseUrl}/api/v1/files/${id}/content?${params}`,
|
|
5241
5306
|
{
|
|
5242
|
-
|
|
5243
|
-
headers: {
|
|
5244
|
-
...this.headers,
|
|
5245
|
-
"Content-Type": "application/json"
|
|
5246
|
-
},
|
|
5247
|
-
method: "POST"
|
|
5307
|
+
headers: this.headers
|
|
5248
5308
|
}
|
|
5249
|
-
)
|
|
5309
|
+
);
|
|
5250
5310
|
}
|
|
5251
|
-
async
|
|
5252
|
-
|
|
5253
|
-
|
|
5254
|
-
|
|
5255
|
-
|
|
5256
|
-
|
|
5311
|
+
async getContentAsText(id) {
|
|
5312
|
+
const response = await this.getContent(id);
|
|
5313
|
+
if (!response.ok) {
|
|
5314
|
+
const error = await response.json();
|
|
5315
|
+
throw new Error(error.detail || "Failed to get file content");
|
|
5316
|
+
}
|
|
5317
|
+
return response.text();
|
|
5318
|
+
}
|
|
5319
|
+
async delete(id) {
|
|
5320
|
+
const response = await fetch(`${this.config.baseUrl}/api/v1/files/${id}`, {
|
|
5321
|
+
headers: this.headers,
|
|
5257
5322
|
method: "DELETE"
|
|
5258
|
-
})
|
|
5323
|
+
});
|
|
5324
|
+
return this.handleResponse(response);
|
|
5259
5325
|
}
|
|
5260
|
-
|
|
5261
|
-
|
|
5262
|
-
|
|
5263
|
-
|
|
5264
|
-
|
|
5265
|
-
body: formData,
|
|
5266
|
-
headers: {
|
|
5267
|
-
...this.headers,
|
|
5268
|
-
Accept: "application/json"
|
|
5269
|
-
},
|
|
5270
|
-
method: "POST"
|
|
5271
|
-
}).then((res) => res.json());
|
|
5326
|
+
};
|
|
5327
|
+
var KnowledgeApi = class {
|
|
5328
|
+
config;
|
|
5329
|
+
constructor(config2) {
|
|
5330
|
+
this.config = config2;
|
|
5272
5331
|
}
|
|
5273
|
-
|
|
5274
|
-
return
|
|
5275
|
-
|
|
5332
|
+
get headers() {
|
|
5333
|
+
return {
|
|
5334
|
+
Authorization: `Bearer ${this.config.apiKey}`
|
|
5335
|
+
};
|
|
5336
|
+
}
|
|
5337
|
+
get jsonHeaders() {
|
|
5338
|
+
return {
|
|
5339
|
+
...this.headers,
|
|
5340
|
+
"Content-Type": "application/json"
|
|
5341
|
+
};
|
|
5342
|
+
}
|
|
5343
|
+
async handleResponse(response) {
|
|
5344
|
+
const data = await response.json();
|
|
5345
|
+
if (isErrorResponse(data)) {
|
|
5346
|
+
throw new Error(data.detail);
|
|
5347
|
+
}
|
|
5348
|
+
return data;
|
|
5349
|
+
}
|
|
5350
|
+
async list() {
|
|
5351
|
+
const response = await fetch(`${this.config.baseUrl}/api/v1/knowledge/`, {
|
|
5352
|
+
headers: this.headers
|
|
5353
|
+
});
|
|
5354
|
+
return this.handleResponse(response);
|
|
5355
|
+
}
|
|
5356
|
+
async listWritable() {
|
|
5357
|
+
const response = await fetch(
|
|
5358
|
+
`${this.config.baseUrl}/api/v1/knowledge/list`,
|
|
5359
|
+
{
|
|
5360
|
+
headers: this.headers
|
|
5361
|
+
}
|
|
5362
|
+
);
|
|
5363
|
+
return this.handleResponse(response);
|
|
5364
|
+
}
|
|
5365
|
+
async create(form) {
|
|
5366
|
+
const response = await fetch(
|
|
5367
|
+
`${this.config.baseUrl}/api/v1/knowledge/create`,
|
|
5368
|
+
{
|
|
5369
|
+
body: JSON.stringify(form),
|
|
5370
|
+
headers: this.jsonHeaders,
|
|
5371
|
+
method: "POST"
|
|
5372
|
+
}
|
|
5373
|
+
);
|
|
5374
|
+
return this.handleResponse(response);
|
|
5375
|
+
}
|
|
5376
|
+
async reindex() {
|
|
5377
|
+
const response = await fetch(
|
|
5378
|
+
`${this.config.baseUrl}/api/v1/knowledge/reindex`,
|
|
5379
|
+
{
|
|
5380
|
+
headers: this.jsonHeaders,
|
|
5381
|
+
method: "POST"
|
|
5382
|
+
}
|
|
5383
|
+
);
|
|
5384
|
+
return this.handleResponse(response);
|
|
5385
|
+
}
|
|
5386
|
+
async getById(id) {
|
|
5387
|
+
const response = await fetch(
|
|
5388
|
+
`${this.config.baseUrl}/api/v1/knowledge/${id}`,
|
|
5389
|
+
{
|
|
5390
|
+
headers: this.headers
|
|
5391
|
+
}
|
|
5392
|
+
);
|
|
5393
|
+
return this.handleResponse(response);
|
|
5394
|
+
}
|
|
5395
|
+
async update(id, form) {
|
|
5396
|
+
const response = await fetch(
|
|
5397
|
+
`${this.config.baseUrl}/api/v1/knowledge/${id}/update`,
|
|
5398
|
+
{
|
|
5399
|
+
body: JSON.stringify(form),
|
|
5400
|
+
headers: this.jsonHeaders,
|
|
5401
|
+
method: "POST"
|
|
5402
|
+
}
|
|
5403
|
+
);
|
|
5404
|
+
return this.handleResponse(response);
|
|
5405
|
+
}
|
|
5406
|
+
async addFile(knowledgeId, fileId) {
|
|
5407
|
+
const response = await fetch(
|
|
5408
|
+
`${this.config.baseUrl}/api/v1/knowledge/${knowledgeId}/file/add`,
|
|
5409
|
+
{
|
|
5410
|
+
body: JSON.stringify({ file_id: fileId }),
|
|
5411
|
+
headers: this.jsonHeaders,
|
|
5412
|
+
method: "POST"
|
|
5413
|
+
}
|
|
5414
|
+
);
|
|
5415
|
+
return this.handleResponse(response);
|
|
5416
|
+
}
|
|
5417
|
+
async updateFile(knowledgeId, fileId) {
|
|
5418
|
+
const response = await fetch(
|
|
5419
|
+
`${this.config.baseUrl}/api/v1/knowledge/${knowledgeId}/file/update`,
|
|
5420
|
+
{
|
|
5421
|
+
body: JSON.stringify({ file_id: fileId }),
|
|
5422
|
+
headers: this.jsonHeaders,
|
|
5423
|
+
method: "POST"
|
|
5424
|
+
}
|
|
5425
|
+
);
|
|
5426
|
+
return this.handleResponse(response);
|
|
5427
|
+
}
|
|
5428
|
+
async removeFile(knowledgeId, fileId, deleteFile = true) {
|
|
5429
|
+
const params = new URLSearchParams({ delete_file: String(deleteFile) });
|
|
5430
|
+
const response = await fetch(
|
|
5431
|
+
`${this.config.baseUrl}/api/v1/knowledge/${knowledgeId}/file/remove?${params}`,
|
|
5276
5432
|
{
|
|
5277
5433
|
body: JSON.stringify({ file_id: fileId }),
|
|
5278
|
-
headers:
|
|
5279
|
-
|
|
5280
|
-
|
|
5281
|
-
|
|
5434
|
+
headers: this.jsonHeaders,
|
|
5435
|
+
method: "POST"
|
|
5436
|
+
}
|
|
5437
|
+
);
|
|
5438
|
+
return this.handleResponse(response);
|
|
5439
|
+
}
|
|
5440
|
+
async delete(id) {
|
|
5441
|
+
const response = await fetch(
|
|
5442
|
+
`${this.config.baseUrl}/api/v1/knowledge/${id}/delete`,
|
|
5443
|
+
{
|
|
5444
|
+
headers: this.headers,
|
|
5445
|
+
method: "DELETE"
|
|
5446
|
+
}
|
|
5447
|
+
);
|
|
5448
|
+
return this.handleResponse(response);
|
|
5449
|
+
}
|
|
5450
|
+
async reset(id) {
|
|
5451
|
+
const response = await fetch(
|
|
5452
|
+
`${this.config.baseUrl}/api/v1/knowledge/${id}/reset`,
|
|
5453
|
+
{
|
|
5454
|
+
headers: this.jsonHeaders,
|
|
5455
|
+
method: "POST"
|
|
5456
|
+
}
|
|
5457
|
+
);
|
|
5458
|
+
return this.handleResponse(response);
|
|
5459
|
+
}
|
|
5460
|
+
async addFilesBatch(knowledgeId, fileIds) {
|
|
5461
|
+
const response = await fetch(
|
|
5462
|
+
`${this.config.baseUrl}/api/v1/knowledge/${knowledgeId}/files/batch/add`,
|
|
5463
|
+
{
|
|
5464
|
+
body: JSON.stringify(fileIds.map((file_id) => ({ file_id }))),
|
|
5465
|
+
headers: this.jsonHeaders,
|
|
5282
5466
|
method: "POST"
|
|
5283
5467
|
}
|
|
5284
|
-
)
|
|
5468
|
+
);
|
|
5469
|
+
return this.handleResponse(response);
|
|
5285
5470
|
}
|
|
5286
5471
|
};
|
|
5287
5472
|
|
|
5473
|
+
// src/open-web-ui-knowledge/common.ts
|
|
5474
|
+
import { config } from "dotenv";
|
|
5475
|
+
function loadEnv() {
|
|
5476
|
+
const options = program.optsWithGlobals();
|
|
5477
|
+
console.debug(options);
|
|
5478
|
+
if (options.env) {
|
|
5479
|
+
config({ path: options.env });
|
|
5480
|
+
} else {
|
|
5481
|
+
config();
|
|
5482
|
+
}
|
|
5483
|
+
}
|
|
5484
|
+
function getConfigFromEnv() {
|
|
5485
|
+
const openWebUiUrl = process.env.WEB_UI_URL;
|
|
5486
|
+
const openWebUiKey = process.env.WEB_UI_KEY;
|
|
5487
|
+
const knowledgeId = process.env.KNOWLEDGE_ID;
|
|
5488
|
+
if (!openWebUiUrl || !openWebUiKey || !knowledgeId) {
|
|
5489
|
+
throw new Error("WEB_UI_URL, WEB_UI_KEY, and KNOWLEDGE_ID must be set");
|
|
5490
|
+
}
|
|
5491
|
+
return {
|
|
5492
|
+
knowledgeId,
|
|
5493
|
+
webUiKey: openWebUiKey,
|
|
5494
|
+
webUiUrl: openWebUiUrl
|
|
5495
|
+
};
|
|
5496
|
+
}
|
|
5497
|
+
|
|
5288
5498
|
// src/open-web-ui-knowledge/download-knowledge.ts
|
|
5289
5499
|
function addDownloadKnowledgeCommand() {
|
|
5290
5500
|
program.command("download-knowledge").description("Download files from an Open Web UI knowledge base").requiredOption("-o, --output-dir <outputDir>", "Folder path").action(async (opts) => {
|
|
5291
5501
|
loadEnv();
|
|
5292
5502
|
await mkdir(opts.outputDir, { recursive: true }).catch();
|
|
5293
|
-
const
|
|
5294
|
-
const
|
|
5295
|
-
|
|
5296
|
-
|
|
5297
|
-
|
|
5298
|
-
|
|
5299
|
-
|
|
5300
|
-
|
|
5301
|
-
|
|
5302
|
-
|
|
5503
|
+
const config2 = getConfigFromEnv();
|
|
5504
|
+
const apiConfig = { apiKey: config2.webUiKey, baseUrl: config2.webUiUrl };
|
|
5505
|
+
const knowledgeApi = new KnowledgeApi(apiConfig);
|
|
5506
|
+
const filesApi = new FilesApi(apiConfig);
|
|
5507
|
+
const knowledge = await knowledgeApi.getById(config2.knowledgeId);
|
|
5508
|
+
for (const file of knowledge.files ?? []) {
|
|
5509
|
+
const fileName = file.meta?.name;
|
|
5510
|
+
if (!fileName) {
|
|
5511
|
+
continue;
|
|
5512
|
+
}
|
|
5513
|
+
try {
|
|
5514
|
+
const content = await filesApi.getDataContent(file.id);
|
|
5515
|
+
if (content?.content) {
|
|
5516
|
+
await writeFile2(
|
|
5517
|
+
resolve2(opts.outputDir, fileName),
|
|
5518
|
+
content.content,
|
|
5519
|
+
"utf-8"
|
|
5520
|
+
);
|
|
5521
|
+
}
|
|
5522
|
+
} catch {
|
|
5523
|
+
console.warn(`Failed to download ${fileName}`);
|
|
5303
5524
|
}
|
|
5304
5525
|
}
|
|
5305
5526
|
});
|
|
@@ -5545,13 +5766,9 @@ import { dedent } from "@qualcomm-ui/utils/dedent";
|
|
|
5545
5766
|
import { existsSync } from "node:fs";
|
|
5546
5767
|
import { join as join2, resolve as resolve4 } from "node:path";
|
|
5547
5768
|
function loadKnowledgeConfigFromEnv(options) {
|
|
5548
|
-
const knowledgeId = process.env.KNOWLEDGE_ID;
|
|
5549
5769
|
const exclude = options.exclude || (process.env.FILE_EXCLUDE_PATTERN ?? "").split(",");
|
|
5550
5770
|
const outputPath = options.outputPath || process.env.KNOWLEDGE_OUTPUT_PATH;
|
|
5551
5771
|
const prefix = process.env.PAGE_TITLE_PREFIX;
|
|
5552
|
-
if (!knowledgeId) {
|
|
5553
|
-
throw new Error("Missing required KNOWLEDGE_ID environment variable");
|
|
5554
|
-
}
|
|
5555
5772
|
if (!outputPath) {
|
|
5556
5773
|
throw new Error("Missing required outputPath");
|
|
5557
5774
|
}
|
|
@@ -5569,7 +5786,6 @@ function loadKnowledgeConfigFromEnv(options) {
|
|
|
5569
5786
|
baseUrl: options.baseUrl || process.env.DOCS_SITE_BASE_URL,
|
|
5570
5787
|
docPropsPath: resolvedConfig.typeDocProps,
|
|
5571
5788
|
exclude,
|
|
5572
|
-
knowledgeId,
|
|
5573
5789
|
outputPath,
|
|
5574
5790
|
pageTitlePrefix: prefix,
|
|
5575
5791
|
routeDir
|
|
@@ -5580,95 +5796,6 @@ function loadKnowledgeConfigFromEnv(options) {
|
|
|
5580
5796
|
async function exists(dirPath) {
|
|
5581
5797
|
return access(dirPath).then(() => true).catch(() => false);
|
|
5582
5798
|
}
|
|
5583
|
-
async function loadDocProps(routesFolder, docPropsPath, verbose) {
|
|
5584
|
-
const resolvedDocPropsPath = docPropsPath ? await exists(docPropsPath) ? docPropsPath : resolve5(process.cwd(), docPropsPath) : join3(dirname(routesFolder), "doc-props.json");
|
|
5585
|
-
if (!await exists(resolvedDocPropsPath)) {
|
|
5586
|
-
if (verbose) {
|
|
5587
|
-
console.log(`Doc props file not found at: ${resolvedDocPropsPath}`);
|
|
5588
|
-
}
|
|
5589
|
-
return null;
|
|
5590
|
-
}
|
|
5591
|
-
try {
|
|
5592
|
-
const content = await readFile(resolvedDocPropsPath, "utf-8");
|
|
5593
|
-
const docProps = JSON.parse(content);
|
|
5594
|
-
if (verbose) {
|
|
5595
|
-
console.log(`Loaded doc props from: ${resolvedDocPropsPath}`);
|
|
5596
|
-
console.log(`Found ${Object.keys(docProps.props).length} component types`);
|
|
5597
|
-
}
|
|
5598
|
-
return docProps;
|
|
5599
|
-
} catch (error) {
|
|
5600
|
-
if (verbose) {
|
|
5601
|
-
console.log(`Error loading doc props: ${error}`);
|
|
5602
|
-
}
|
|
5603
|
-
return null;
|
|
5604
|
-
}
|
|
5605
|
-
}
|
|
5606
|
-
function formatComment(comment) {
|
|
5607
|
-
if (!comment) {
|
|
5608
|
-
return "";
|
|
5609
|
-
}
|
|
5610
|
-
const parts = [];
|
|
5611
|
-
if (comment.summary && comment.summary.length > 0) {
|
|
5612
|
-
const summaryText = formatCommentParts(comment.summary);
|
|
5613
|
-
if (summaryText.trim()) {
|
|
5614
|
-
parts.push(summaryText.trim());
|
|
5615
|
-
}
|
|
5616
|
-
}
|
|
5617
|
-
if (comment.blockTags && comment.blockTags.length > 0) {
|
|
5618
|
-
for (const blockTag of comment.blockTags) {
|
|
5619
|
-
const tagContent = formatCommentParts(blockTag.content);
|
|
5620
|
-
if (tagContent.trim()) {
|
|
5621
|
-
const tagName = blockTag.tag.replace("@", "");
|
|
5622
|
-
if (tagName === "default" || tagName === "defaultValue") {
|
|
5623
|
-
continue;
|
|
5624
|
-
}
|
|
5625
|
-
if (tagName === "example") {
|
|
5626
|
-
parts.push(`**Example:**
|
|
5627
|
-
\`\`\`
|
|
5628
|
-
${tagContent.trim()}
|
|
5629
|
-
\`\`\``);
|
|
5630
|
-
} else {
|
|
5631
|
-
parts.push(`**${tagName}:** ${tagContent.trim()}`);
|
|
5632
|
-
}
|
|
5633
|
-
}
|
|
5634
|
-
}
|
|
5635
|
-
}
|
|
5636
|
-
return parts.join("\n\n");
|
|
5637
|
-
}
|
|
5638
|
-
function formatCommentParts(parts) {
|
|
5639
|
-
return parts.map((part) => {
|
|
5640
|
-
switch (part.kind) {
|
|
5641
|
-
case "text":
|
|
5642
|
-
return part.text;
|
|
5643
|
-
case "code":
|
|
5644
|
-
const codeText = part.text.replace(/```\w*\n?/g, "").replace(/\n?```/g, "").trim();
|
|
5645
|
-
if (codeText.includes("\n")) {
|
|
5646
|
-
return `\`\`\`
|
|
5647
|
-
${codeText}
|
|
5648
|
-
\`\`\``;
|
|
5649
|
-
} else {
|
|
5650
|
-
return codeText;
|
|
5651
|
-
}
|
|
5652
|
-
default:
|
|
5653
|
-
if ("tag" in part && part.tag === "@link" && typeof part.target === "string") {
|
|
5654
|
-
return `[${part.text}](${part.target})`;
|
|
5655
|
-
}
|
|
5656
|
-
return part.text;
|
|
5657
|
-
}
|
|
5658
|
-
}).join("").replace(/\n/g, " ").replace(/\s+/g, " ").trim();
|
|
5659
|
-
}
|
|
5660
|
-
function convertPropInfo(propInfo, isPartial, propType = void 0) {
|
|
5661
|
-
return {
|
|
5662
|
-
name: propInfo.name,
|
|
5663
|
-
type: extractBestType(propInfo),
|
|
5664
|
-
...propInfo.defaultValue && {
|
|
5665
|
-
defaultValue: cleanDefaultValue(propInfo.defaultValue)
|
|
5666
|
-
},
|
|
5667
|
-
description: formatComment(propInfo.comment || null),
|
|
5668
|
-
propType,
|
|
5669
|
-
required: extractRequired(propInfo, isPartial) || void 0
|
|
5670
|
-
};
|
|
5671
|
-
}
|
|
5672
5799
|
function extractBestType(propInfo) {
|
|
5673
5800
|
const type = propInfo.resolvedType?.prettyType || propInfo.type;
|
|
5674
5801
|
return cleanType(type.startsWith("| ") ? type.substring(2) : type);
|
|
@@ -5682,121 +5809,20 @@ function cleanType(type) {
|
|
|
5682
5809
|
function cleanDefaultValue(defaultValue) {
|
|
5683
5810
|
return defaultValue.replace(/^\n+/, "").replace(/\n+$/, "").trim();
|
|
5684
5811
|
}
|
|
5685
|
-
async function scanPages(routesFolder, verbose, excludePatterns = [], baseUrl) {
|
|
5686
|
-
const components = [];
|
|
5687
|
-
function shouldExclude(fileOrDir) {
|
|
5688
|
-
const dirName = basename(fileOrDir);
|
|
5689
|
-
return excludePatterns.some((pattern) => {
|
|
5690
|
-
if (pattern.includes("*")) {
|
|
5691
|
-
const regex = new RegExp(`^${pattern.replace(/\*/g, ".*")}$`);
|
|
5692
|
-
return regex.test(dirName);
|
|
5693
|
-
}
|
|
5694
|
-
return dirName === pattern;
|
|
5695
|
-
});
|
|
5696
|
-
}
|
|
5697
|
-
async function scanDirectory(dirPath) {
|
|
5698
|
-
if (shouldExclude(dirPath)) {
|
|
5699
|
-
if (verbose) {
|
|
5700
|
-
console.log(`Excluding directory: ${basename(dirPath)}`);
|
|
5701
|
-
}
|
|
5702
|
-
return;
|
|
5703
|
-
}
|
|
5704
|
-
const entries = await readdir(dirPath, { withFileTypes: true });
|
|
5705
|
-
const mdxFiles = entries.filter(
|
|
5706
|
-
(f) => f.name.endsWith(".mdx") && !shouldExclude(f.name)
|
|
5707
|
-
) ?? [];
|
|
5708
|
-
for (const mdxFile of mdxFiles) {
|
|
5709
|
-
const demosFolder = entries.find((f) => f.name === "demos");
|
|
5710
|
-
const demosFolderPath = demosFolder ? join3(dirPath, demosFolder.name) : void 0;
|
|
5711
|
-
const segments = getPathSegmentsFromFileName(
|
|
5712
|
-
join3(dirPath, mdxFile.name),
|
|
5713
|
-
routesFolder
|
|
5714
|
-
);
|
|
5715
|
-
const url = getPathnameFromPathSegments(segments);
|
|
5716
|
-
components.push({
|
|
5717
|
-
demosFolder: demosFolderPath,
|
|
5718
|
-
id: segments.join("-").trim(),
|
|
5719
|
-
mdxFile: join3(dirPath, mdxFile.name),
|
|
5720
|
-
name: segments.at(-1),
|
|
5721
|
-
path: dirPath,
|
|
5722
|
-
url: baseUrl ? new URL(url, baseUrl).toString() : void 0
|
|
5723
|
-
});
|
|
5724
|
-
if (verbose) {
|
|
5725
|
-
console.log(`Found component: ${basename(dirPath)}`);
|
|
5726
|
-
console.log(` Demos folder: ${demosFolderPath || "NOT FOUND"}`);
|
|
5727
|
-
}
|
|
5728
|
-
}
|
|
5729
|
-
for (const entry of entries) {
|
|
5730
|
-
const fullPath = join3(dirPath, entry.name);
|
|
5731
|
-
const stats = await stat(fullPath);
|
|
5732
|
-
if (stats.isDirectory()) {
|
|
5733
|
-
await scanDirectory(fullPath);
|
|
5734
|
-
}
|
|
5735
|
-
}
|
|
5736
|
-
}
|
|
5737
|
-
await scanDirectory(routesFolder);
|
|
5738
|
-
return components;
|
|
5739
|
-
}
|
|
5740
5812
|
function isPreviewLine(trimmedLine) {
|
|
5741
5813
|
return trimmedLine === "// preview" || /^\{\s*\/\*\s*preview\s*\*\/\s*\}$/.test(trimmedLine) || /^<!--\s*preview\s*-->$/.test(trimmedLine);
|
|
5742
5814
|
}
|
|
5743
|
-
function extractProps(props, isPartial) {
|
|
5744
|
-
const propsInfo = [];
|
|
5745
|
-
if (props.props?.length) {
|
|
5746
|
-
propsInfo.push(
|
|
5747
|
-
...props.props.map((prop) => convertPropInfo(prop, isPartial))
|
|
5748
|
-
);
|
|
5749
|
-
}
|
|
5750
|
-
if (props.input?.length) {
|
|
5751
|
-
propsInfo.push(
|
|
5752
|
-
...props.input.map((prop) => convertPropInfo(prop, isPartial, "input"))
|
|
5753
|
-
);
|
|
5754
|
-
}
|
|
5755
|
-
if (props.output?.length) {
|
|
5756
|
-
propsInfo.push(
|
|
5757
|
-
...props.output.map((prop) => convertPropInfo(prop, isPartial, "output"))
|
|
5758
|
-
);
|
|
5759
|
-
}
|
|
5760
|
-
return propsInfo;
|
|
5761
|
-
}
|
|
5762
5815
|
function removePreviewLines(code) {
|
|
5763
5816
|
return code.split("\n").filter((line) => !isPreviewLine(line.trim())).join("\n");
|
|
5764
5817
|
}
|
|
5765
|
-
function getIntroLines(
|
|
5818
|
+
function getIntroLines(projectName, description) {
|
|
5766
5819
|
const lines = [];
|
|
5767
5820
|
if (projectName) {
|
|
5768
5821
|
lines.push(`# ${projectName}`);
|
|
5769
|
-
lines.push("");
|
|
5770
5822
|
}
|
|
5771
5823
|
if (description) {
|
|
5772
|
-
lines.push(`> ${description}`);
|
|
5773
|
-
lines.push("");
|
|
5774
|
-
}
|
|
5775
|
-
lines.push("## Components and Integrations");
|
|
5776
|
-
lines.push("");
|
|
5777
|
-
for (const page of pages) {
|
|
5778
|
-
const url = page.url ?? `#${kebabCase(page.title)}`;
|
|
5779
|
-
lines.push(`- [${page.title}](${url})`);
|
|
5780
|
-
}
|
|
5781
|
-
return lines.join("\n");
|
|
5782
|
-
}
|
|
5783
|
-
async function generateLlmsTxt(pages, projectName, description) {
|
|
5784
|
-
const lines = [getIntroLines(pages, projectName, description)];
|
|
5785
|
-
lines.push("");
|
|
5786
|
-
for (const page of pages) {
|
|
5787
|
-
const content = page.content.split("\n").map((line) => {
|
|
5788
|
-
if (line.startsWith("#")) {
|
|
5789
|
-
return `#${line}`;
|
|
5790
|
-
}
|
|
5791
|
-
return line;
|
|
5792
|
-
});
|
|
5793
|
-
if (content.every((line) => !line.trim())) {
|
|
5794
|
-
continue;
|
|
5795
|
-
}
|
|
5796
|
-
lines.push(`## ${page.title}`);
|
|
5797
|
-
lines.push("");
|
|
5798
|
-
lines.push(content.join("\n"));
|
|
5799
5824
|
lines.push("");
|
|
5825
|
+
lines.push(`> ${description}`);
|
|
5800
5826
|
}
|
|
5801
5827
|
return lines.join("\n");
|
|
5802
5828
|
}
|
|
@@ -5827,44 +5853,11 @@ async function resolveModulePath(importPath, fromFile) {
|
|
|
5827
5853
|
}
|
|
5828
5854
|
return null;
|
|
5829
5855
|
}
|
|
5830
|
-
|
|
5831
|
-
|
|
5832
|
-
|
|
5833
|
-
return [];
|
|
5834
|
-
}
|
|
5835
|
-
visited.add(normalizedPath);
|
|
5836
|
-
const modules = [];
|
|
5837
|
-
try {
|
|
5838
|
-
const content = await readFile(normalizedPath, "utf-8");
|
|
5839
|
-
const relativeImports = extractRelativeImports(content);
|
|
5840
|
-
for (const importPath of relativeImports) {
|
|
5841
|
-
const resolvedPath = await resolveModulePath(importPath, normalizedPath);
|
|
5842
|
-
if (!resolvedPath) {
|
|
5843
|
-
if (verbose) {
|
|
5844
|
-
console.log(
|
|
5845
|
-
` Could not resolve import: ${importPath} from ${normalizedPath}`
|
|
5846
|
-
);
|
|
5847
|
-
}
|
|
5848
|
-
continue;
|
|
5849
|
-
}
|
|
5850
|
-
const importContent = await readFile(resolvedPath, "utf-8");
|
|
5851
|
-
modules.push({
|
|
5852
|
-
content: importContent,
|
|
5853
|
-
path: resolvedPath
|
|
5854
|
-
});
|
|
5855
|
-
const nestedModules = await collectRelativeImports(
|
|
5856
|
-
resolvedPath,
|
|
5857
|
-
visited,
|
|
5858
|
-
verbose
|
|
5859
|
-
);
|
|
5860
|
-
modules.push(...nestedModules);
|
|
5861
|
-
}
|
|
5862
|
-
} catch (error) {
|
|
5863
|
-
if (verbose) {
|
|
5864
|
-
console.log(` Error processing ${normalizedPath}: ${error}`);
|
|
5865
|
-
}
|
|
5866
|
-
}
|
|
5867
|
-
return modules;
|
|
5856
|
+
function extractMetadata(metadata) {
|
|
5857
|
+
return (metadata ?? []).map((current) => {
|
|
5858
|
+
const [key, value] = current.split("=");
|
|
5859
|
+
return [key, value];
|
|
5860
|
+
});
|
|
5868
5861
|
}
|
|
5869
5862
|
var replaceNpmInstallTabs = () => {
|
|
5870
5863
|
return (tree, _file, done) => {
|
|
@@ -5885,376 +5878,675 @@ var replaceNpmInstallTabs = () => {
|
|
|
5885
5878
|
done();
|
|
5886
5879
|
};
|
|
5887
5880
|
};
|
|
5888
|
-
function
|
|
5889
|
-
return ()
|
|
5890
|
-
|
|
5891
|
-
|
|
5892
|
-
|
|
5893
|
-
|
|
5894
|
-
|
|
5895
|
-
|
|
5896
|
-
|
|
5897
|
-
|
|
5898
|
-
|
|
5899
|
-
|
|
5900
|
-
|
|
5901
|
-
|
|
5881
|
+
function getPath(obj, path) {
|
|
5882
|
+
return path.split(".").reduce(
|
|
5883
|
+
(acc, key) => acc && typeof acc === "object" ? acc[key] : void 0,
|
|
5884
|
+
obj
|
|
5885
|
+
);
|
|
5886
|
+
}
|
|
5887
|
+
var KnowledgeGenerator = class {
|
|
5888
|
+
config;
|
|
5889
|
+
docProps = null;
|
|
5890
|
+
constructor(config2) {
|
|
5891
|
+
this.config = config2;
|
|
5892
|
+
}
|
|
5893
|
+
async run() {
|
|
5894
|
+
const extractedMetadata = extractMetadata(this.config.metadata);
|
|
5895
|
+
if (this.config.verbose) {
|
|
5896
|
+
console.log(`Scanning pages in: ${this.config.routeDir}`);
|
|
5897
|
+
if (this.config.exclude?.length) {
|
|
5898
|
+
console.log(`Excluding patterns: ${this.config.exclude.join(", ")}`);
|
|
5899
|
+
}
|
|
5900
|
+
}
|
|
5901
|
+
const [docProps, pages] = await Promise.all([
|
|
5902
|
+
this.loadDocProps(),
|
|
5903
|
+
this.scanPages()
|
|
5904
|
+
]);
|
|
5905
|
+
this.docProps = docProps;
|
|
5906
|
+
if (pages.length === 0) {
|
|
5907
|
+
console.log("No pages found.");
|
|
5908
|
+
return;
|
|
5909
|
+
}
|
|
5910
|
+
if (this.config.verbose) {
|
|
5911
|
+
console.log(`Found ${pages.length} page(s)`);
|
|
5912
|
+
}
|
|
5913
|
+
const processedPages = [];
|
|
5914
|
+
for (const page of pages) {
|
|
5915
|
+
try {
|
|
5916
|
+
const processed = await this.processComponent(page);
|
|
5917
|
+
processedPages.push(processed);
|
|
5918
|
+
} catch (error) {
|
|
5919
|
+
console.error(`Failed to process page: ${page.name}`);
|
|
5920
|
+
process.exit(1);
|
|
5921
|
+
}
|
|
5922
|
+
}
|
|
5923
|
+
if (this.config.clean) {
|
|
5924
|
+
await rm(this.config.outputPath, { force: true, recursive: true }).catch(
|
|
5925
|
+
() => {
|
|
5926
|
+
}
|
|
5927
|
+
);
|
|
5928
|
+
}
|
|
5929
|
+
if (this.config.outputMode === "aggregated") {
|
|
5930
|
+
await this.generateAggregatedOutput(processedPages, pages);
|
|
5931
|
+
} else {
|
|
5932
|
+
await mkdir2(this.config.outputPath, { recursive: true }).catch(() => {
|
|
5933
|
+
});
|
|
5934
|
+
await this.generatePerPageExports(
|
|
5935
|
+
pages,
|
|
5936
|
+
processedPages,
|
|
5937
|
+
extractedMetadata
|
|
5938
|
+
);
|
|
5939
|
+
}
|
|
5940
|
+
}
|
|
5941
|
+
async loadDocProps() {
|
|
5942
|
+
const resolvedDocPropsPath = this.config.docPropsPath ? await exists(this.config.docPropsPath) ? this.config.docPropsPath : resolve5(process.cwd(), this.config.docPropsPath) : join3(dirname(this.config.routeDir), "doc-props.json");
|
|
5943
|
+
if (!await exists(resolvedDocPropsPath)) {
|
|
5944
|
+
if (this.config.verbose) {
|
|
5945
|
+
console.log(`Doc props file not found at: ${resolvedDocPropsPath}`);
|
|
5946
|
+
}
|
|
5947
|
+
return null;
|
|
5948
|
+
}
|
|
5949
|
+
try {
|
|
5950
|
+
const content = await readFile(resolvedDocPropsPath, "utf-8");
|
|
5951
|
+
const docProps = JSON.parse(content);
|
|
5952
|
+
if (this.config.verbose) {
|
|
5953
|
+
console.log(`Loaded doc props from: ${resolvedDocPropsPath}`);
|
|
5954
|
+
console.log(
|
|
5955
|
+
`Found ${Object.keys(docProps.props).length} component types`
|
|
5956
|
+
);
|
|
5957
|
+
}
|
|
5958
|
+
return docProps;
|
|
5959
|
+
} catch (error) {
|
|
5960
|
+
if (this.config.verbose) {
|
|
5961
|
+
console.log(`Error loading doc props: ${error}`);
|
|
5962
|
+
}
|
|
5963
|
+
return null;
|
|
5964
|
+
}
|
|
5965
|
+
}
|
|
5966
|
+
async scanPages() {
|
|
5967
|
+
const components = [];
|
|
5968
|
+
const excludePatterns = this.config.exclude ?? [];
|
|
5969
|
+
const shouldExclude = (fileOrDir) => {
|
|
5970
|
+
const dirName = basename(fileOrDir);
|
|
5971
|
+
return excludePatterns.some((pattern) => {
|
|
5972
|
+
if (pattern.includes("*")) {
|
|
5973
|
+
const regex = new RegExp(`^${pattern.replace(/\*/g, ".*")}$`);
|
|
5974
|
+
return regex.test(dirName);
|
|
5975
|
+
}
|
|
5976
|
+
return dirName === pattern;
|
|
5977
|
+
});
|
|
5978
|
+
};
|
|
5979
|
+
const scanDirectory = async (dirPath) => {
|
|
5980
|
+
if (shouldExclude(dirPath)) {
|
|
5981
|
+
if (this.config.verbose) {
|
|
5982
|
+
console.log(`Excluding directory: ${basename(dirPath)}`);
|
|
5983
|
+
}
|
|
5984
|
+
return;
|
|
5985
|
+
}
|
|
5986
|
+
const entries = await readdir(dirPath, { withFileTypes: true });
|
|
5987
|
+
const mdxFiles = entries.filter(
|
|
5988
|
+
(f) => f.name.endsWith(".mdx") && !shouldExclude(f.name)
|
|
5989
|
+
) ?? [];
|
|
5990
|
+
for (const mdxFile of mdxFiles) {
|
|
5991
|
+
const demosFolder = entries.find((f) => f.name === "demos");
|
|
5992
|
+
const demosFolderPath = demosFolder ? join3(dirPath, demosFolder.name) : void 0;
|
|
5993
|
+
const segments = getPathSegmentsFromFileName(
|
|
5994
|
+
join3(dirPath, mdxFile.name),
|
|
5995
|
+
this.config.routeDir
|
|
5902
5996
|
);
|
|
5903
|
-
|
|
5904
|
-
|
|
5905
|
-
|
|
5997
|
+
const url = getPathnameFromPathSegments(segments);
|
|
5998
|
+
components.push({
|
|
5999
|
+
demosFolder: demosFolderPath,
|
|
6000
|
+
id: segments.join("-").trim(),
|
|
6001
|
+
mdxFile: join3(dirPath, mdxFile.name),
|
|
6002
|
+
name: segments.at(-1),
|
|
6003
|
+
path: dirPath,
|
|
6004
|
+
url: this.config.baseUrl ? new URL(url, this.config.baseUrl).toString() : void 0
|
|
6005
|
+
});
|
|
6006
|
+
if (this.config.verbose) {
|
|
6007
|
+
console.log(`Found component: ${basename(dirPath)}`);
|
|
6008
|
+
console.log(` Demos folder: ${demosFolderPath || "NOT FOUND"}`);
|
|
6009
|
+
}
|
|
6010
|
+
}
|
|
6011
|
+
for (const entry of entries) {
|
|
6012
|
+
const fullPath = join3(dirPath, entry.name);
|
|
6013
|
+
const stats = await stat(fullPath);
|
|
6014
|
+
if (stats.isDirectory()) {
|
|
6015
|
+
await scanDirectory(fullPath);
|
|
6016
|
+
}
|
|
6017
|
+
}
|
|
6018
|
+
};
|
|
6019
|
+
await scanDirectory(this.config.routeDir);
|
|
6020
|
+
return components;
|
|
6021
|
+
}
|
|
6022
|
+
async collectRelativeImports(filePath, visited = /* @__PURE__ */ new Set()) {
|
|
6023
|
+
const normalizedPath = resolve5(filePath);
|
|
6024
|
+
if (visited.has(normalizedPath)) {
|
|
6025
|
+
return [];
|
|
6026
|
+
}
|
|
6027
|
+
visited.add(normalizedPath);
|
|
6028
|
+
const modules = [];
|
|
6029
|
+
try {
|
|
6030
|
+
const content = await readFile(normalizedPath, "utf-8");
|
|
6031
|
+
const relativeImports = extractRelativeImports(content);
|
|
6032
|
+
for (const importPath of relativeImports) {
|
|
6033
|
+
const resolvedPath = await resolveModulePath(importPath, normalizedPath);
|
|
6034
|
+
if (!resolvedPath) {
|
|
6035
|
+
if (this.config.verbose) {
|
|
6036
|
+
console.log(
|
|
6037
|
+
` Could not resolve import: ${importPath} from ${normalizedPath}`
|
|
6038
|
+
);
|
|
5906
6039
|
}
|
|
5907
|
-
|
|
6040
|
+
continue;
|
|
5908
6041
|
}
|
|
5909
|
-
const
|
|
5910
|
-
|
|
5911
|
-
|
|
5912
|
-
|
|
6042
|
+
const importContent = await readFile(resolvedPath, "utf-8");
|
|
6043
|
+
modules.push({
|
|
6044
|
+
content: importContent,
|
|
6045
|
+
path: resolvedPath
|
|
6046
|
+
});
|
|
6047
|
+
const nestedModules = await this.collectRelativeImports(
|
|
6048
|
+
resolvedPath,
|
|
6049
|
+
visited
|
|
6050
|
+
);
|
|
6051
|
+
modules.push(...nestedModules);
|
|
6052
|
+
}
|
|
6053
|
+
} catch (error) {
|
|
6054
|
+
if (this.config.verbose) {
|
|
6055
|
+
console.log(` Error processing ${normalizedPath}: ${error}`);
|
|
6056
|
+
}
|
|
6057
|
+
}
|
|
6058
|
+
return modules;
|
|
6059
|
+
}
|
|
6060
|
+
extractProps(props, isPartial) {
|
|
6061
|
+
const propsInfo = [];
|
|
6062
|
+
if (props.props?.length) {
|
|
6063
|
+
propsInfo.push(
|
|
6064
|
+
...props.props.map((prop) => this.convertPropInfo(prop, isPartial))
|
|
6065
|
+
);
|
|
6066
|
+
}
|
|
6067
|
+
if (props.input?.length) {
|
|
6068
|
+
propsInfo.push(
|
|
6069
|
+
...props.input.map(
|
|
6070
|
+
(prop) => this.convertPropInfo(prop, isPartial, "input")
|
|
6071
|
+
)
|
|
6072
|
+
);
|
|
6073
|
+
}
|
|
6074
|
+
if (props.output?.length) {
|
|
6075
|
+
propsInfo.push(
|
|
6076
|
+
...props.output.map(
|
|
6077
|
+
(prop) => this.convertPropInfo(prop, isPartial, "output")
|
|
6078
|
+
)
|
|
6079
|
+
);
|
|
6080
|
+
}
|
|
6081
|
+
return propsInfo;
|
|
6082
|
+
}
|
|
6083
|
+
formatComment(comment) {
|
|
6084
|
+
if (!comment) {
|
|
6085
|
+
return "";
|
|
6086
|
+
}
|
|
6087
|
+
const parts = [];
|
|
6088
|
+
if (comment.summary && comment.summary.length > 0) {
|
|
6089
|
+
const summaryText = this.formatCommentParts(comment.summary);
|
|
6090
|
+
if (summaryText.trim()) {
|
|
6091
|
+
parts.push(summaryText.trim());
|
|
6092
|
+
}
|
|
6093
|
+
}
|
|
6094
|
+
if (comment.blockTags && comment.blockTags.length > 0) {
|
|
6095
|
+
for (const blockTag of comment.blockTags) {
|
|
6096
|
+
const tagContent = this.formatCommentParts(blockTag.content);
|
|
6097
|
+
if (tagContent.trim()) {
|
|
6098
|
+
const tagName = blockTag.tag.replace("@", "");
|
|
6099
|
+
if (tagName === "default" || tagName === "defaultValue") {
|
|
6100
|
+
continue;
|
|
6101
|
+
}
|
|
6102
|
+
if (tagName === "example") {
|
|
6103
|
+
parts.push(`**Example:**
|
|
6104
|
+
\`\`\`
|
|
6105
|
+
${tagContent.trim()}
|
|
6106
|
+
\`\`\``);
|
|
6107
|
+
} else {
|
|
6108
|
+
parts.push(`**${tagName}:** ${tagContent.trim()}`);
|
|
5913
6109
|
}
|
|
5914
|
-
return;
|
|
5915
6110
|
}
|
|
5916
|
-
|
|
5917
|
-
|
|
5918
|
-
|
|
5919
|
-
|
|
5920
|
-
|
|
6111
|
+
}
|
|
6112
|
+
}
|
|
6113
|
+
return parts.join("\n\n");
|
|
6114
|
+
}
|
|
6115
|
+
formatCommentParts(parts) {
|
|
6116
|
+
return parts.map((part) => {
|
|
6117
|
+
switch (part.kind) {
|
|
6118
|
+
case "text":
|
|
6119
|
+
return part.text;
|
|
6120
|
+
case "code":
|
|
6121
|
+
const codeText = part.text.replace(/```\w*\n?/g, "").replace(/\n?```/g, "").trim();
|
|
6122
|
+
if (codeText.includes("\n")) {
|
|
6123
|
+
return `\`\`\`
|
|
6124
|
+
${codeText}
|
|
6125
|
+
\`\`\``;
|
|
6126
|
+
} else {
|
|
6127
|
+
return codeText;
|
|
5921
6128
|
}
|
|
5922
|
-
|
|
5923
|
-
|
|
6129
|
+
default:
|
|
6130
|
+
if (this.config.outputMode === "per-page" && "tag" in part && part.tag === "@link" && typeof part.target === "string") {
|
|
6131
|
+
return `[${part.text}](${part.target})`;
|
|
5924
6132
|
}
|
|
6133
|
+
return part.text;
|
|
6134
|
+
}
|
|
6135
|
+
}).join("").replace(/\n/g, " ").replace(/\s+/g, " ").trim();
|
|
6136
|
+
}
|
|
6137
|
+
convertPropInfo(propInfo, isPartial, propType = void 0) {
|
|
6138
|
+
return {
|
|
6139
|
+
name: propInfo.name,
|
|
6140
|
+
type: extractBestType(propInfo),
|
|
6141
|
+
...propInfo.defaultValue && {
|
|
6142
|
+
defaultValue: cleanDefaultValue(propInfo.defaultValue)
|
|
6143
|
+
},
|
|
6144
|
+
description: this.formatComment(propInfo.comment || null),
|
|
6145
|
+
propType,
|
|
6146
|
+
required: extractRequired(propInfo, isPartial) || void 0
|
|
6147
|
+
};
|
|
6148
|
+
}
|
|
6149
|
+
/**
|
|
6150
|
+
* Creates a remark plugin that replaces TypeDocProps JSX elements with JSON
|
|
6151
|
+
* code blocks containing component prop documentation.
|
|
6152
|
+
*/
|
|
6153
|
+
async replaceThemeNodes() {
|
|
6154
|
+
let themes = null;
|
|
6155
|
+
try {
|
|
6156
|
+
themes = await import("@qualcomm-ui/tailwind-plugin/theme");
|
|
6157
|
+
} catch {
|
|
6158
|
+
return () => {
|
|
6159
|
+
};
|
|
6160
|
+
}
|
|
6161
|
+
const handlers = {
|
|
6162
|
+
ColorTable: (node) => {
|
|
6163
|
+
const path = this.getAttrExpression(node, "data");
|
|
6164
|
+
return path && getPath(themes, path);
|
|
6165
|
+
},
|
|
6166
|
+
FontTable: (node) => {
|
|
6167
|
+
const path = this.getAttrExpression(node, "data");
|
|
6168
|
+
return path && getPath(themes, path);
|
|
6169
|
+
},
|
|
6170
|
+
ThemePropertyTable: (node) => {
|
|
6171
|
+
const path = this.getAttrExpression(node, "data");
|
|
6172
|
+
const property = this.getAttrExpression(node, "cssProperty");
|
|
6173
|
+
const data = path && getPath(themes, path);
|
|
6174
|
+
return path && property ? { cssPropertyName: property, data } : void 0;
|
|
6175
|
+
}
|
|
6176
|
+
};
|
|
6177
|
+
return () => (tree, _file, done) => {
|
|
6178
|
+
visit7(tree, "mdxJsxFlowElement", (node) => {
|
|
6179
|
+
const handler = node.name && handlers[node.name];
|
|
6180
|
+
if (!handler) {
|
|
5925
6181
|
return;
|
|
5926
6182
|
}
|
|
5927
|
-
const
|
|
5928
|
-
if (
|
|
5929
|
-
console.
|
|
5930
|
-
|
|
5931
|
-
);
|
|
6183
|
+
const data = handler(node);
|
|
6184
|
+
if (!data) {
|
|
6185
|
+
console.warn(`No theme data for ${node.name}`);
|
|
6186
|
+
return;
|
|
5932
6187
|
}
|
|
5933
6188
|
Object.assign(node, {
|
|
5934
6189
|
lang: "json",
|
|
5935
6190
|
meta: null,
|
|
5936
6191
|
type: "code",
|
|
5937
|
-
value: JSON.stringify(
|
|
6192
|
+
value: JSON.stringify(data, null, 2)
|
|
5938
6193
|
});
|
|
5939
|
-
}
|
|
6194
|
+
});
|
|
6195
|
+
done();
|
|
6196
|
+
};
|
|
6197
|
+
}
|
|
6198
|
+
getAttrExpression(node, name) {
|
|
6199
|
+
const attr = node.attributes?.find(
|
|
6200
|
+
(a) => a.type === "mdxJsxAttribute" && a.name === name
|
|
5940
6201
|
);
|
|
5941
|
-
|
|
5942
|
-
|
|
5943
|
-
}
|
|
5944
|
-
|
|
5945
|
-
|
|
5946
|
-
|
|
5947
|
-
|
|
5948
|
-
|
|
5949
|
-
|
|
5950
|
-
|
|
5951
|
-
|
|
5952
|
-
|
|
5953
|
-
|
|
5954
|
-
|
|
5955
|
-
|
|
5956
|
-
|
|
5957
|
-
|
|
5958
|
-
|
|
5959
|
-
|
|
5960
|
-
|
|
5961
|
-
|
|
5962
|
-
|
|
5963
|
-
|
|
5964
|
-
const
|
|
5965
|
-
|
|
5966
|
-
|
|
5967
|
-
|
|
5968
|
-
|
|
6202
|
+
if (!attr?.value) {
|
|
6203
|
+
return null;
|
|
6204
|
+
}
|
|
6205
|
+
if (typeof attr.value === "string") {
|
|
6206
|
+
return attr.value;
|
|
6207
|
+
} else if (typeof attr.value === "object" && "value" in attr.value) {
|
|
6208
|
+
return attr.value.value;
|
|
6209
|
+
}
|
|
6210
|
+
return null;
|
|
6211
|
+
}
|
|
6212
|
+
/**
|
|
6213
|
+
* Creates a remark plugin that replaces TypeDocProps JSX elements with JSON
|
|
6214
|
+
* code blocks containing component prop documentation.
|
|
6215
|
+
*/
|
|
6216
|
+
replaceTypeDocProps() {
|
|
6217
|
+
return () => (tree, _file, done) => {
|
|
6218
|
+
visit7(
|
|
6219
|
+
tree,
|
|
6220
|
+
"mdxJsxFlowElement",
|
|
6221
|
+
(node, index, parent) => {
|
|
6222
|
+
if (node?.name !== "TypeDocProps") {
|
|
6223
|
+
return;
|
|
6224
|
+
}
|
|
6225
|
+
const nameAttr = node.attributes?.find(
|
|
6226
|
+
(attr) => attr.type === "mdxJsxAttribute" && attr.name === "name"
|
|
6227
|
+
);
|
|
6228
|
+
const isPartial = node.attributes?.some(
|
|
6229
|
+
(attr) => attr.type === "mdxJsxAttribute" && attr.name === "partial"
|
|
6230
|
+
);
|
|
6231
|
+
if (!this.docProps || !nameAttr) {
|
|
6232
|
+
if (parent && index !== void 0) {
|
|
6233
|
+
parent.children.splice(index, 1);
|
|
5969
6234
|
}
|
|
6235
|
+
return;
|
|
5970
6236
|
}
|
|
5971
|
-
|
|
5972
|
-
|
|
5973
|
-
|
|
5974
|
-
|
|
6237
|
+
const propsNames = extractNamesFromAttribute(nameAttr);
|
|
6238
|
+
if (propsNames.length === 0) {
|
|
6239
|
+
if (parent && index !== void 0) {
|
|
6240
|
+
parent.children.splice(index, 1);
|
|
6241
|
+
}
|
|
6242
|
+
return;
|
|
5975
6243
|
}
|
|
5976
|
-
|
|
6244
|
+
const propsName = propsNames[0];
|
|
6245
|
+
const componentProps = this.docProps.props[propsName];
|
|
6246
|
+
if (!componentProps) {
|
|
6247
|
+
if (this.config.verbose) {
|
|
6248
|
+
console.log(` TypeDocProps not found: ${propsName}`);
|
|
6249
|
+
}
|
|
6250
|
+
if (parent && index !== void 0) {
|
|
6251
|
+
parent.children.splice(index, 1);
|
|
6252
|
+
}
|
|
6253
|
+
return;
|
|
6254
|
+
}
|
|
6255
|
+
const propsDoc = this.extractProps(componentProps, Boolean(isPartial));
|
|
6256
|
+
if (this.config.verbose) {
|
|
6257
|
+
console.log(
|
|
6258
|
+
` Replaced TypeDocProps ${propsName} with API documentation`
|
|
6259
|
+
);
|
|
6260
|
+
}
|
|
6261
|
+
Object.assign(node, {
|
|
6262
|
+
lang: "json",
|
|
6263
|
+
meta: null,
|
|
6264
|
+
type: "code",
|
|
6265
|
+
value: JSON.stringify(propsDoc, null, 2)
|
|
6266
|
+
});
|
|
5977
6267
|
}
|
|
5978
|
-
|
|
5979
|
-
|
|
5980
|
-
|
|
5981
|
-
|
|
5982
|
-
|
|
5983
|
-
|
|
5984
|
-
|
|
5985
|
-
|
|
5986
|
-
|
|
5987
|
-
|
|
6268
|
+
);
|
|
6269
|
+
done();
|
|
6270
|
+
};
|
|
6271
|
+
}
|
|
6272
|
+
/**
|
|
6273
|
+
* Creates a remark plugin that replaces demo JSX elements (QdsDemo, CodeDemo,
|
|
6274
|
+
* Demo) with code blocks containing the demo source code from the demos folder.
|
|
6275
|
+
*/
|
|
6276
|
+
replaceDemos(demosFolder, demoFiles) {
|
|
6277
|
+
return () => async (tree) => {
|
|
6278
|
+
const promises = [];
|
|
6279
|
+
visit7(
|
|
6280
|
+
tree,
|
|
6281
|
+
"mdxJsxFlowElement",
|
|
6282
|
+
(node, index, parent) => {
|
|
6283
|
+
if (!node?.name || !["QdsDemo", "CodeDemo", "Demo"].includes(node.name)) {
|
|
6284
|
+
return;
|
|
6285
|
+
}
|
|
6286
|
+
const nameAttr = node.attributes?.find(
|
|
6287
|
+
(attr) => attr.type === "mdxJsxAttribute" && attr.name === "name"
|
|
6288
|
+
);
|
|
6289
|
+
const nodeAttr = node.attributes?.find(
|
|
6290
|
+
(attr) => attr.type === "mdxJsxAttribute" && attr.name === "node"
|
|
6291
|
+
);
|
|
6292
|
+
let demoName;
|
|
6293
|
+
if (nameAttr && typeof nameAttr.value === "string") {
|
|
6294
|
+
demoName = nameAttr.value;
|
|
6295
|
+
} else if (nodeAttr?.value && typeof nodeAttr.value !== "string") {
|
|
6296
|
+
const estree = nodeAttr.value.data?.estree;
|
|
6297
|
+
if (estree?.body?.[0]?.type === "ExpressionStatement") {
|
|
6298
|
+
const expression = estree.body[0].expression;
|
|
6299
|
+
if (expression.type === "MemberExpression" && expression.object.type === "Identifier" && expression.object.name === "Demo" && expression.property.type === "Identifier") {
|
|
6300
|
+
demoName = expression.property.name;
|
|
5988
6301
|
}
|
|
5989
|
-
return;
|
|
5990
6302
|
}
|
|
5991
|
-
|
|
5992
|
-
|
|
5993
|
-
if (
|
|
5994
|
-
|
|
5995
|
-
|
|
5996
|
-
|
|
5997
|
-
|
|
5998
|
-
|
|
5999
|
-
|
|
6000
|
-
|
|
6303
|
+
}
|
|
6304
|
+
if (!demoName) {
|
|
6305
|
+
if (parent && index !== void 0) {
|
|
6306
|
+
parent.children.splice(index, 1);
|
|
6307
|
+
}
|
|
6308
|
+
return;
|
|
6309
|
+
}
|
|
6310
|
+
promises.push(
|
|
6311
|
+
(async () => {
|
|
6312
|
+
const kebabName = kebabCase(demoName);
|
|
6313
|
+
let filePath = `${kebabName}.tsx`;
|
|
6314
|
+
if (!demosFolder) {
|
|
6315
|
+
if (this.config.verbose) {
|
|
6316
|
+
console.log(` No demos folder for ${demoName}`);
|
|
6317
|
+
}
|
|
6001
6318
|
if (parent && index !== void 0) {
|
|
6002
6319
|
parent.children.splice(index, 1);
|
|
6003
6320
|
}
|
|
6004
6321
|
return;
|
|
6005
6322
|
}
|
|
6006
|
-
|
|
6007
|
-
|
|
6008
|
-
|
|
6009
|
-
|
|
6010
|
-
|
|
6011
|
-
|
|
6012
|
-
|
|
6013
|
-
|
|
6014
|
-
|
|
6015
|
-
|
|
6016
|
-
|
|
6017
|
-
|
|
6018
|
-
|
|
6019
|
-
|
|
6020
|
-
|
|
6021
|
-
if (verbose) {
|
|
6022
|
-
console.log(` Error reading demo ${demoName}: ${error}`);
|
|
6323
|
+
let demoFilePath = join3(demosFolder, filePath);
|
|
6324
|
+
let isAngularDemo = false;
|
|
6325
|
+
if (!await exists(demoFilePath)) {
|
|
6326
|
+
demoFilePath = join3(demosFolder, `${kebabName}.ts`);
|
|
6327
|
+
if (await exists(demoFilePath)) {
|
|
6328
|
+
isAngularDemo = true;
|
|
6329
|
+
filePath = `${kebabCase(demoName).replace("-component", ".component")}.ts`;
|
|
6330
|
+
demoFilePath = join3(demosFolder, filePath);
|
|
6331
|
+
} else {
|
|
6332
|
+
console.log(` Demo not found ${demoName}`);
|
|
6333
|
+
if (parent && index !== void 0) {
|
|
6334
|
+
parent.children.splice(index, 1);
|
|
6335
|
+
}
|
|
6336
|
+
return;
|
|
6337
|
+
}
|
|
6023
6338
|
}
|
|
6024
|
-
|
|
6025
|
-
|
|
6339
|
+
try {
|
|
6340
|
+
const demoCode = await readFile(demoFilePath, "utf-8");
|
|
6341
|
+
const cleanedCode = removePreviewLines(demoCode);
|
|
6342
|
+
if (this.config.verbose) {
|
|
6343
|
+
console.log(` Replaced demo ${demoName} with source code`);
|
|
6344
|
+
}
|
|
6345
|
+
demoFiles.push(demoFilePath);
|
|
6346
|
+
Object.assign(node, {
|
|
6347
|
+
lang: isAngularDemo ? "angular-ts" : "tsx",
|
|
6348
|
+
meta: null,
|
|
6349
|
+
type: "code",
|
|
6350
|
+
value: cleanedCode
|
|
6351
|
+
});
|
|
6352
|
+
} catch (error) {
|
|
6353
|
+
if (this.config.verbose) {
|
|
6354
|
+
console.log(` Error reading demo ${demoName}: ${error}`);
|
|
6355
|
+
}
|
|
6356
|
+
if (parent && index !== void 0) {
|
|
6357
|
+
parent.children.splice(index, 1);
|
|
6358
|
+
}
|
|
6026
6359
|
}
|
|
6027
|
-
}
|
|
6028
|
-
|
|
6029
|
-
|
|
6030
|
-
|
|
6031
|
-
|
|
6032
|
-
|
|
6033
|
-
}
|
|
6034
|
-
|
|
6035
|
-
|
|
6036
|
-
|
|
6037
|
-
|
|
6038
|
-
|
|
6039
|
-
|
|
6040
|
-
|
|
6041
|
-
|
|
6360
|
+
})()
|
|
6361
|
+
);
|
|
6362
|
+
}
|
|
6363
|
+
);
|
|
6364
|
+
await Promise.all(promises);
|
|
6365
|
+
};
|
|
6366
|
+
}
|
|
6367
|
+
/**
|
|
6368
|
+
* Processes MDX content by transforming JSX elements (TypeDocProps, demos)
|
|
6369
|
+
* into markdown, resolving relative links, and cleaning up formatting.
|
|
6370
|
+
*/
|
|
6371
|
+
async processMdxContent(mdxContent, pageUrl, demosFolder) {
|
|
6372
|
+
const demoFiles = [];
|
|
6373
|
+
let processedContent = mdxContent;
|
|
6374
|
+
const lines = processedContent.split("\n");
|
|
6375
|
+
const titleLine = lines.findIndex((line) => line.startsWith("# "));
|
|
6376
|
+
processedContent = titleLine >= 0 ? lines.slice(titleLine + 1).join("\n") : processedContent;
|
|
6042
6377
|
processedContent = processedContent.replace(
|
|
6043
6378
|
/\[([^\]]+)\]\(\.\/#([^)]+)\)/g,
|
|
6044
|
-
(_, text, anchor) => `[${text}](${pageUrl}#${anchor})`
|
|
6379
|
+
(_, text, anchor) => pageUrl && this.config.outputMode === "per-page" ? `[${text}](${pageUrl}#${anchor})` : text
|
|
6045
6380
|
);
|
|
6381
|
+
const processor = unified4().use(remarkParse4).use(remarkMdx3).use(this.replaceTypeDocProps()).use(await this.replaceThemeNodes()).use(this.replaceDemos(demosFolder, demoFiles)).use(remarkStringify3);
|
|
6382
|
+
const processed = await processor.process(processedContent);
|
|
6383
|
+
processedContent = String(processed);
|
|
6384
|
+
processedContent = processedContent.replace(/\n\s*\n\s*\n/g, "\n\n");
|
|
6385
|
+
return { content: processedContent, demoFiles };
|
|
6046
6386
|
}
|
|
6047
|
-
|
|
6048
|
-
|
|
6049
|
-
|
|
6050
|
-
|
|
6051
|
-
|
|
6052
|
-
}
|
|
6053
|
-
|
|
6054
|
-
|
|
6055
|
-
|
|
6056
|
-
|
|
6057
|
-
|
|
6058
|
-
|
|
6059
|
-
|
|
6060
|
-
|
|
6061
|
-
|
|
6062
|
-
|
|
6063
|
-
|
|
6064
|
-
|
|
6065
|
-
|
|
6066
|
-
|
|
6067
|
-
|
|
6068
|
-
|
|
6069
|
-
|
|
6070
|
-
|
|
6071
|
-
|
|
6072
|
-
|
|
6073
|
-
|
|
6074
|
-
|
|
6075
|
-
|
|
6076
|
-
|
|
6077
|
-
|
|
6078
|
-
|
|
6079
|
-
|
|
6080
|
-
|
|
6081
|
-
|
|
6082
|
-
|
|
6083
|
-
|
|
6084
|
-
}
|
|
6085
|
-
} catch (error) {
|
|
6086
|
-
console.error(`Error processing component ${component.name}:`, error);
|
|
6087
|
-
throw error;
|
|
6387
|
+
async processComponent(component) {
|
|
6388
|
+
try {
|
|
6389
|
+
const mdxContent = await readFile(component.mdxFile, "utf-8");
|
|
6390
|
+
if (this.config.verbose) {
|
|
6391
|
+
console.log(`Processing page: ${component.name}`);
|
|
6392
|
+
}
|
|
6393
|
+
const processor = unified4().use(remarkParse4).use(remarkMdx3).use(replaceNpmInstallTabs).use(remarkFrontmatter3, ["yaml"]).use(remarkParseFrontmatter2);
|
|
6394
|
+
if (this.config.outputMode === "per-page") {
|
|
6395
|
+
processor.use(remarkSelfLinkHeadings(component.url));
|
|
6396
|
+
}
|
|
6397
|
+
processor.use(remarkStringify3);
|
|
6398
|
+
const parsed = await processor.process(mdxContent);
|
|
6399
|
+
const frontmatter = parsed.data?.frontmatter || {};
|
|
6400
|
+
const { content: processedContent, demoFiles } = await this.processMdxContent(
|
|
6401
|
+
String(parsed),
|
|
6402
|
+
component.url,
|
|
6403
|
+
component.demosFolder
|
|
6404
|
+
);
|
|
6405
|
+
const removeJsxProcessor = unified4().use(remarkParse4).use(remarkMdx3).use(remarkRemoveJsx).use(remarkStringify3);
|
|
6406
|
+
const removedJsx = String(
|
|
6407
|
+
await removeJsxProcessor.process(processedContent)
|
|
6408
|
+
);
|
|
6409
|
+
const contentWithoutFrontmatter = removedJsx.replace(
|
|
6410
|
+
/^---[\s\S]*?---\n/,
|
|
6411
|
+
""
|
|
6412
|
+
);
|
|
6413
|
+
const title = frontmatter.title || component.name;
|
|
6414
|
+
return {
|
|
6415
|
+
content: contentWithoutFrontmatter.trim(),
|
|
6416
|
+
demoFiles,
|
|
6417
|
+
frontmatter,
|
|
6418
|
+
title,
|
|
6419
|
+
url: component.url
|
|
6420
|
+
};
|
|
6421
|
+
} catch (error) {
|
|
6422
|
+
console.error(`Error processing component ${component.name}:`, error);
|
|
6423
|
+
throw error;
|
|
6424
|
+
}
|
|
6088
6425
|
}
|
|
6089
|
-
|
|
6090
|
-
|
|
6091
|
-
|
|
6092
|
-
|
|
6093
|
-
|
|
6094
|
-
|
|
6095
|
-
|
|
6096
|
-
|
|
6097
|
-
|
|
6098
|
-
}) {
|
|
6099
|
-
await mkdir2(dirname(outputPath), { recursive: true }).catch();
|
|
6100
|
-
const count = processedPages.length;
|
|
6101
|
-
let totalSize = 0;
|
|
6102
|
-
await Promise.all(
|
|
6103
|
-
processedPages.map(async (processedPage, index) => {
|
|
6104
|
-
const page = pages[index];
|
|
6105
|
-
const lines = [];
|
|
6106
|
-
if (metadata.length || page.url) {
|
|
6107
|
-
lines.push("---");
|
|
6108
|
-
if (page.url) {
|
|
6109
|
-
lines.push(`url: ${page.url}`);
|
|
6110
|
-
}
|
|
6111
|
-
if (metadata.length) {
|
|
6112
|
-
for (const [key, value] of metadata) {
|
|
6113
|
-
lines.push(`${key}: ${value}`);
|
|
6114
|
-
}
|
|
6426
|
+
async generateLlmsTxt(pages) {
|
|
6427
|
+
const lines = [
|
|
6428
|
+
getIntroLines(this.config.name, this.config.description)
|
|
6429
|
+
];
|
|
6430
|
+
lines.push("");
|
|
6431
|
+
for (const page of pages) {
|
|
6432
|
+
const content = page.content.split("\n").map((line) => {
|
|
6433
|
+
if (line.startsWith("#")) {
|
|
6434
|
+
return `#${line}`;
|
|
6115
6435
|
}
|
|
6116
|
-
|
|
6117
|
-
|
|
6436
|
+
return line;
|
|
6437
|
+
});
|
|
6438
|
+
if (content.every((line) => !line.trim())) {
|
|
6439
|
+
continue;
|
|
6118
6440
|
}
|
|
6119
|
-
lines.push(
|
|
6441
|
+
lines.push(`## ${page.title}`);
|
|
6120
6442
|
lines.push("");
|
|
6121
|
-
|
|
6122
|
-
page.name = processedPage.frontmatter.title;
|
|
6123
|
-
}
|
|
6124
|
-
let content = processedPage.content;
|
|
6125
|
-
if (pageTitlePrefix) {
|
|
6126
|
-
content = content.replace(
|
|
6127
|
-
`# ${page.name}`,
|
|
6128
|
-
`# ${pageTitlePrefix} ${page.name}`
|
|
6129
|
-
);
|
|
6130
|
-
page.name = `${pageTitlePrefix} ${page.name}`;
|
|
6131
|
-
}
|
|
6132
|
-
lines.push(content);
|
|
6443
|
+
lines.push(content.join("\n"));
|
|
6133
6444
|
lines.push("");
|
|
6134
|
-
|
|
6135
|
-
|
|
6136
|
-
|
|
6137
|
-
|
|
6138
|
-
|
|
6445
|
+
}
|
|
6446
|
+
return lines.join("\n");
|
|
6447
|
+
}
|
|
6448
|
+
async generateAggregatedOutput(processedPages, pages) {
|
|
6449
|
+
const llmsTxtContent = await this.generateLlmsTxt(processedPages);
|
|
6450
|
+
await mkdir2(dirname(this.config.outputPath), { recursive: true }).catch(
|
|
6451
|
+
() => {
|
|
6452
|
+
}
|
|
6453
|
+
);
|
|
6454
|
+
await writeFile3(this.config.outputPath, llmsTxtContent, "utf-8");
|
|
6455
|
+
const outputStats = await stat(this.config.outputPath);
|
|
6456
|
+
const outputSizeKb = (outputStats.size / 1024).toFixed(1);
|
|
6457
|
+
console.log(
|
|
6458
|
+
`Generated ${this.config.outputPath} with ${pages.length} component(s) at: ${this.config.outputPath}`
|
|
6459
|
+
);
|
|
6460
|
+
console.log(`File size: ${outputSizeKb} KB`);
|
|
6461
|
+
}
|
|
6462
|
+
async generatePerPageExports(pages, processedPages, metadata) {
|
|
6463
|
+
await mkdir2(dirname(this.config.outputPath), { recursive: true }).catch(
|
|
6464
|
+
() => {
|
|
6465
|
+
}
|
|
6466
|
+
);
|
|
6467
|
+
const count = processedPages.length;
|
|
6468
|
+
let totalSize = 0;
|
|
6469
|
+
await Promise.all(
|
|
6470
|
+
processedPages.map(async (processedPage, index) => {
|
|
6471
|
+
const page = pages[index];
|
|
6472
|
+
const lines = [];
|
|
6473
|
+
if (metadata.length || page.url) {
|
|
6474
|
+
lines.push("---");
|
|
6475
|
+
if (page.url) {
|
|
6476
|
+
lines.push(`url: ${page.url}`);
|
|
6477
|
+
}
|
|
6478
|
+
if (metadata.length) {
|
|
6479
|
+
for (const [key, value] of metadata) {
|
|
6480
|
+
lines.push(`${key}: ${value}`);
|
|
6481
|
+
}
|
|
6482
|
+
}
|
|
6483
|
+
lines.push("---");
|
|
6484
|
+
lines.push("");
|
|
6139
6485
|
}
|
|
6140
|
-
|
|
6141
|
-
|
|
6142
|
-
|
|
6143
|
-
|
|
6144
|
-
|
|
6145
|
-
|
|
6486
|
+
lines.push(`# ${processedPage.title}`);
|
|
6487
|
+
lines.push("");
|
|
6488
|
+
if (processedPage.frontmatter?.title) {
|
|
6489
|
+
page.name = processedPage.frontmatter.title;
|
|
6490
|
+
}
|
|
6491
|
+
let content = processedPage.content;
|
|
6492
|
+
if (this.config.pageTitlePrefix) {
|
|
6493
|
+
content = content.replace(
|
|
6494
|
+
`# ${page.name}`,
|
|
6495
|
+
`# ${this.config.pageTitlePrefix} ${page.name}`
|
|
6146
6496
|
);
|
|
6147
|
-
|
|
6497
|
+
page.name = `${this.config.pageTitlePrefix} ${page.name}`;
|
|
6148
6498
|
}
|
|
6149
|
-
|
|
6150
|
-
|
|
6151
|
-
)
|
|
6152
|
-
|
|
6153
|
-
|
|
6154
|
-
|
|
6499
|
+
lines.push(content);
|
|
6500
|
+
lines.push("");
|
|
6501
|
+
if (this.config.includeImports && processedPage.demoFiles.length > 0) {
|
|
6502
|
+
if (this.config.verbose) {
|
|
6503
|
+
console.log(
|
|
6504
|
+
`Collecting imports for ${page.name} from ${processedPage.demoFiles.length} demo files`
|
|
6505
|
+
);
|
|
6506
|
+
}
|
|
6507
|
+
const allImports = [];
|
|
6508
|
+
for (const demoFile of processedPage.demoFiles) {
|
|
6509
|
+
const imports = await this.collectRelativeImports(
|
|
6510
|
+
demoFile,
|
|
6511
|
+
/* @__PURE__ */ new Set()
|
|
6512
|
+
);
|
|
6513
|
+
allImports.push(...imports);
|
|
6514
|
+
}
|
|
6515
|
+
const uniqueImports = Array.from(
|
|
6516
|
+
new Map(allImports.map((m) => [m.path, m])).values()
|
|
6155
6517
|
);
|
|
6156
|
-
|
|
6157
|
-
|
|
6158
|
-
|
|
6159
|
-
|
|
6160
|
-
|
|
6161
|
-
|
|
6162
|
-
lines.push(
|
|
6163
|
-
lines.push("");
|
|
6164
|
-
lines.push(`\`\`\`${ext}`);
|
|
6165
|
-
lines.push(importedModule.content);
|
|
6166
|
-
lines.push("```");
|
|
6518
|
+
if (this.config.verbose) {
|
|
6519
|
+
console.log(
|
|
6520
|
+
` Collected ${uniqueImports.length} unique import modules`
|
|
6521
|
+
);
|
|
6522
|
+
}
|
|
6523
|
+
if (uniqueImports.length > 0) {
|
|
6524
|
+
lines.push("## Related Source Files");
|
|
6167
6525
|
lines.push("");
|
|
6526
|
+
for (const importedModule of uniqueImports) {
|
|
6527
|
+
const ext = extname(importedModule.path).slice(1);
|
|
6528
|
+
lines.push(`### ${basename(importedModule.path)}`);
|
|
6529
|
+
lines.push("");
|
|
6530
|
+
lines.push(`\`\`\`${ext}`);
|
|
6531
|
+
lines.push(importedModule.content);
|
|
6532
|
+
lines.push("```");
|
|
6533
|
+
lines.push("");
|
|
6534
|
+
}
|
|
6168
6535
|
}
|
|
6169
6536
|
}
|
|
6170
|
-
|
|
6171
|
-
|
|
6172
|
-
|
|
6173
|
-
|
|
6174
|
-
|
|
6175
|
-
})
|
|
6176
|
-
);
|
|
6177
|
-
console.log(`Generated ${count} component(s) in ${outputPath}`);
|
|
6178
|
-
console.log(`Folder size: ${totalSize.toFixed(1)} KB`);
|
|
6179
|
-
}
|
|
6180
|
-
function extractMetadata(metadata) {
|
|
6181
|
-
return (metadata ?? []).map((current) => {
|
|
6182
|
-
const [key, value] = current.split("=");
|
|
6183
|
-
return [key, value];
|
|
6184
|
-
});
|
|
6185
|
-
}
|
|
6186
|
-
async function generate({
|
|
6187
|
-
baseUrl,
|
|
6188
|
-
clean,
|
|
6189
|
-
description,
|
|
6190
|
-
docPropsPath,
|
|
6191
|
-
exclude,
|
|
6192
|
-
includeImports,
|
|
6193
|
-
metadata,
|
|
6194
|
-
name,
|
|
6195
|
-
outputMode,
|
|
6196
|
-
outputPath,
|
|
6197
|
-
pageTitlePrefix,
|
|
6198
|
-
routeDir,
|
|
6199
|
-
verbose
|
|
6200
|
-
}) {
|
|
6201
|
-
const extractedMetadata = extractMetadata(metadata);
|
|
6202
|
-
if (verbose) {
|
|
6203
|
-
console.log(`Scanning pages in: ${routeDir}`);
|
|
6204
|
-
if (exclude?.length) {
|
|
6205
|
-
console.log(`Excluding patterns: ${exclude.join(", ")}`);
|
|
6206
|
-
}
|
|
6207
|
-
}
|
|
6208
|
-
const [docProps, pages] = await Promise.all([
|
|
6209
|
-
loadDocProps(routeDir, docPropsPath, verbose),
|
|
6210
|
-
scanPages(routeDir, verbose, exclude, baseUrl)
|
|
6211
|
-
]);
|
|
6212
|
-
if (pages.length === 0) {
|
|
6213
|
-
console.log("No pages found.");
|
|
6214
|
-
return;
|
|
6215
|
-
}
|
|
6216
|
-
if (verbose) {
|
|
6217
|
-
console.log(`Found ${pages.length} page(s)`);
|
|
6218
|
-
}
|
|
6219
|
-
const processedPages = [];
|
|
6220
|
-
for (const page of pages) {
|
|
6221
|
-
try {
|
|
6222
|
-
const processed = await processComponent(page, docProps, verbose);
|
|
6223
|
-
processedPages.push(processed);
|
|
6224
|
-
} catch (error) {
|
|
6225
|
-
console.error(`Failed to process page: ${page.name}`);
|
|
6226
|
-
process.exit(1);
|
|
6227
|
-
}
|
|
6228
|
-
}
|
|
6229
|
-
if (clean) {
|
|
6230
|
-
await rm(outputPath, { force: true, recursive: true }).catch();
|
|
6231
|
-
}
|
|
6232
|
-
if (outputMode === "aggregated") {
|
|
6233
|
-
const llmsTxtContent = await generateLlmsTxt(
|
|
6234
|
-
processedPages,
|
|
6235
|
-
name,
|
|
6236
|
-
description
|
|
6237
|
-
);
|
|
6238
|
-
await mkdir2(dirname(outputPath), { recursive: true }).catch();
|
|
6239
|
-
await writeFile3(outputPath, llmsTxtContent, "utf-8");
|
|
6240
|
-
const outputStats = await stat(outputPath);
|
|
6241
|
-
const outputSizeKb = (outputStats.size / 1024).toFixed(1);
|
|
6242
|
-
console.log(
|
|
6243
|
-
`Generated ${outputPath} with ${pages.length} component(s) at: ${outputPath}`
|
|
6537
|
+
const outfile = `${resolve5(this.config.outputPath)}/${kebabCase(page.id || page.name)}.md`;
|
|
6538
|
+
await writeFile3(outfile, lines.join("\n"), "utf-8");
|
|
6539
|
+
const stats = await stat(outfile);
|
|
6540
|
+
totalSize += stats.size / 1024;
|
|
6541
|
+
})
|
|
6244
6542
|
);
|
|
6245
|
-
console.log(`
|
|
6246
|
-
|
|
6247
|
-
await mkdir2(outputPath, { recursive: true }).catch();
|
|
6248
|
-
await generatePerPageExports({
|
|
6249
|
-
includeImports,
|
|
6250
|
-
metadata: extractedMetadata,
|
|
6251
|
-
outputPath,
|
|
6252
|
-
pages,
|
|
6253
|
-
pageTitlePrefix,
|
|
6254
|
-
processedPages,
|
|
6255
|
-
verbose
|
|
6256
|
-
});
|
|
6543
|
+
console.log(`Generated ${count} component(s) in ${this.config.outputPath}`);
|
|
6544
|
+
console.log(`Folder size: ${totalSize.toFixed(1)} KB`);
|
|
6257
6545
|
}
|
|
6546
|
+
};
|
|
6547
|
+
async function generate(config2) {
|
|
6548
|
+
const generator = new KnowledgeGenerator(config2);
|
|
6549
|
+
await generator.run();
|
|
6258
6550
|
}
|
|
6259
6551
|
function addGenerateKnowledgeCommand() {
|
|
6260
6552
|
program.description("Generate llms.txt from QUI Docs documentation").command("generate-llms-txt").option("-n, --name <name>", "Project name for llms.txt header").requiredOption("-m, --output-mode <outputMode>").option("-o, --outputPath <outputPath>", "Output file or directory.").option(
|
|
@@ -6276,32 +6568,132 @@ function addGenerateKnowledgeCommand() {
|
|
|
6276
6568
|
|
|
6277
6569
|
// src/open-web-ui-knowledge/upload-knowledge.ts
|
|
6278
6570
|
import { createHash as createHash2 } from "node:crypto";
|
|
6279
|
-
import {
|
|
6280
|
-
|
|
6281
|
-
mkdir as mkdir3,
|
|
6282
|
-
readdir as readdir2,
|
|
6283
|
-
readFile as readFile2,
|
|
6284
|
-
stat as stat2,
|
|
6285
|
-
writeFile as writeFile4
|
|
6286
|
-
} from "node:fs/promises";
|
|
6571
|
+
import { writeFileSync } from "node:fs";
|
|
6572
|
+
import { access as access2, readdir as readdir2, readFile as readFile2, stat as stat2 } from "node:fs/promises";
|
|
6287
6573
|
import { resolve as resolve6 } from "node:path";
|
|
6288
6574
|
import { setTimeout as setTimeout2 } from "node:timers/promises";
|
|
6289
6575
|
import ora from "ora";
|
|
6576
|
+
|
|
6577
|
+
// src/open-web-ui-knowledge/knowledge-cleaner.ts
|
|
6578
|
+
var KnowledgeCleaner = class {
|
|
6579
|
+
filesApi;
|
|
6580
|
+
knowledgeApi;
|
|
6581
|
+
constructor(config2) {
|
|
6582
|
+
const apiConfig = {
|
|
6583
|
+
apiKey: config2.webUiKey,
|
|
6584
|
+
baseUrl: config2.webUiUrl
|
|
6585
|
+
};
|
|
6586
|
+
this.filesApi = new FilesApi(apiConfig);
|
|
6587
|
+
this.knowledgeApi = new KnowledgeApi(apiConfig);
|
|
6588
|
+
}
|
|
6589
|
+
async cleanUpOrphanedFiles() {
|
|
6590
|
+
const files = await this.filesApi.list();
|
|
6591
|
+
const knowledgeBases = await this.knowledgeApi.list();
|
|
6592
|
+
const knowledgeIds = knowledgeBases.map((k) => k.id);
|
|
6593
|
+
for (const file of files) {
|
|
6594
|
+
const collectionName = file.meta?.collection_name;
|
|
6595
|
+
if (collectionName && !knowledgeIds.includes(collectionName)) {
|
|
6596
|
+
await this.filesApi.delete(file.id);
|
|
6597
|
+
} else if (file.data?.status === "failed") {
|
|
6598
|
+
await this.filesApi.delete(file.id);
|
|
6599
|
+
}
|
|
6600
|
+
}
|
|
6601
|
+
}
|
|
6602
|
+
};
|
|
6603
|
+
|
|
6604
|
+
// src/open-web-ui-knowledge/upload-knowledge.ts
|
|
6605
|
+
function toKnowledgeFile(file) {
|
|
6606
|
+
return {
|
|
6607
|
+
id: file.id,
|
|
6608
|
+
meta: { name: file.meta?.name }
|
|
6609
|
+
};
|
|
6610
|
+
}
|
|
6290
6611
|
function calculateFileHash(fileData) {
|
|
6291
6612
|
const normalized = fileData.normalize("NFC").replace(/\r\n/g, "\n").replace(/\r/g, "\n").replace(/\n+$/, "");
|
|
6292
6613
|
return createHash2("sha256").update(normalized).digest("hex");
|
|
6293
6614
|
}
|
|
6294
6615
|
var Uploader = class {
|
|
6295
6616
|
config;
|
|
6296
|
-
|
|
6617
|
+
knowledgeApi;
|
|
6618
|
+
filesApi;
|
|
6619
|
+
fileHashCache = /* @__PURE__ */ new Map();
|
|
6620
|
+
knowledgeFilesCache = null;
|
|
6621
|
+
cleaner;
|
|
6297
6622
|
constructor(config2) {
|
|
6298
6623
|
this.config = config2;
|
|
6299
|
-
|
|
6300
|
-
|
|
6301
|
-
|
|
6302
|
-
return {
|
|
6303
|
-
Authorization: `Bearer ${this.config.webUiKey}`
|
|
6624
|
+
const apiConfig = {
|
|
6625
|
+
apiKey: config2.webUiKey,
|
|
6626
|
+
baseUrl: config2.webUiUrl
|
|
6304
6627
|
};
|
|
6628
|
+
this.knowledgeApi = new KnowledgeApi(apiConfig);
|
|
6629
|
+
this.filesApi = new FilesApi(apiConfig);
|
|
6630
|
+
this.cleaner = new KnowledgeCleaner(config2);
|
|
6631
|
+
}
|
|
6632
|
+
async buildHashCache(files) {
|
|
6633
|
+
const results = await Promise.allSettled(
|
|
6634
|
+
files.map(async (f) => {
|
|
6635
|
+
try {
|
|
6636
|
+
const content = await this.filesApi.getDataContent(f.id);
|
|
6637
|
+
if (content?.content) {
|
|
6638
|
+
this.fileHashCache.set(f.id, calculateFileHash(content.content));
|
|
6639
|
+
}
|
|
6640
|
+
} catch {
|
|
6641
|
+
}
|
|
6642
|
+
})
|
|
6643
|
+
);
|
|
6644
|
+
const failures = results.filter((r) => r.status === "rejected");
|
|
6645
|
+
if (failures.length > 0) {
|
|
6646
|
+
console.warn(`Failed to cache ${failures.length} file hashes`);
|
|
6647
|
+
}
|
|
6648
|
+
}
|
|
6649
|
+
async waitForFileDeletion(fileId, fileName, maxAttempts = 15) {
|
|
6650
|
+
const spinner = ora(`File changed, deleting ${fileName}`).start();
|
|
6651
|
+
for (let i = 0; i < maxAttempts; i++) {
|
|
6652
|
+
this.knowledgeFilesCache = null;
|
|
6653
|
+
const knowledge = await this.knowledgeApi.getById(this.config.knowledgeId);
|
|
6654
|
+
const stillExists = (knowledge.files ?? []).some((f) => f.id === fileId);
|
|
6655
|
+
if (!stillExists) {
|
|
6656
|
+
this.fileHashCache.delete(fileId);
|
|
6657
|
+
spinner.succeed(`File deleted: ${fileId}`);
|
|
6658
|
+
return true;
|
|
6659
|
+
}
|
|
6660
|
+
await setTimeout2(100 * (i + 1));
|
|
6661
|
+
}
|
|
6662
|
+
spinner.stop();
|
|
6663
|
+
console.debug(`File ${fileId} may not have been fully deleted`);
|
|
6664
|
+
return false;
|
|
6665
|
+
}
|
|
6666
|
+
async uploadWithRetry(name, contents, maxRetries = 10) {
|
|
6667
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
6668
|
+
const result = await this.uploadFile(name, contents);
|
|
6669
|
+
if (result.success) {
|
|
6670
|
+
return result;
|
|
6671
|
+
}
|
|
6672
|
+
if (result.error?.detail?.includes("Duplicate content detected")) {
|
|
6673
|
+
console.warn(
|
|
6674
|
+
`Duplicate content: ${name} is already in knowledge base, skipping`
|
|
6675
|
+
);
|
|
6676
|
+
if (result.error.fileId) {
|
|
6677
|
+
try {
|
|
6678
|
+
console.debug(`Removing duplicate file: ${result.error.fileId}`);
|
|
6679
|
+
await this.filesApi.delete(result.error.fileId);
|
|
6680
|
+
await this.waitForFileDeletion(result.error.fileId, name);
|
|
6681
|
+
} catch (e) {
|
|
6682
|
+
console.debug("Failed to remove duplicate file", e);
|
|
6683
|
+
}
|
|
6684
|
+
}
|
|
6685
|
+
return { skipped: true, success: true };
|
|
6686
|
+
}
|
|
6687
|
+
if (attempt < maxRetries - 1) {
|
|
6688
|
+
const delay = 100 * Math.pow(2, attempt);
|
|
6689
|
+
console.debug(
|
|
6690
|
+
`Retrying ${name} in ${delay}ms (attempt ${attempt + 2}/${maxRetries})`
|
|
6691
|
+
);
|
|
6692
|
+
await setTimeout2(delay);
|
|
6693
|
+
}
|
|
6694
|
+
}
|
|
6695
|
+
console.debug(`Failed to upload ${name}`);
|
|
6696
|
+
return { success: false };
|
|
6305
6697
|
}
|
|
6306
6698
|
async uploadDirectory() {
|
|
6307
6699
|
const fileNames = await readdir2(this.config.knowledgeFilePath);
|
|
@@ -6314,32 +6706,28 @@ var Uploader = class {
|
|
|
6314
6706
|
name
|
|
6315
6707
|
}))
|
|
6316
6708
|
);
|
|
6317
|
-
const
|
|
6709
|
+
const knowledge = await this.knowledgeApi.getById(this.config.knowledgeId);
|
|
6710
|
+
this.knowledgeFilesCache = (knowledge.files ?? []).map(toKnowledgeFile);
|
|
6711
|
+
await this.buildHashCache(this.knowledgeFilesCache);
|
|
6712
|
+
let skippedCount = 0;
|
|
6318
6713
|
let successCount = 0;
|
|
6319
6714
|
let failureCount = 0;
|
|
6320
|
-
await this.api.listKnowledgeFiles();
|
|
6321
6715
|
for (const file of files) {
|
|
6322
|
-
|
|
6323
|
-
while (!result.success && result.count && result.count < 5) {
|
|
6324
|
-
console.debug("Failed to upload, retrying with count: ", result.count);
|
|
6325
|
-
await setTimeout2(100);
|
|
6326
|
-
result = await this.uploadFile(file.name, file.contents, result.count);
|
|
6327
|
-
}
|
|
6716
|
+
const result = await this.uploadWithRetry(file.name, file.contents);
|
|
6328
6717
|
if (result.skipped) {
|
|
6329
|
-
|
|
6330
|
-
}
|
|
6331
|
-
if (result.success) {
|
|
6718
|
+
skippedCount++;
|
|
6719
|
+
} else if (result.success) {
|
|
6332
6720
|
successCount++;
|
|
6333
6721
|
} else {
|
|
6334
6722
|
failureCount++;
|
|
6335
6723
|
}
|
|
6336
6724
|
}
|
|
6337
|
-
if (
|
|
6725
|
+
if (skippedCount > 0) {
|
|
6338
6726
|
console.debug(
|
|
6339
|
-
`Skipped uploading ${
|
|
6727
|
+
`Skipped uploading ${skippedCount} files because their contents did not change`
|
|
6340
6728
|
);
|
|
6341
6729
|
}
|
|
6342
|
-
const uploadCount = successCount
|
|
6730
|
+
const uploadCount = Math.abs(successCount);
|
|
6343
6731
|
if (uploadCount) {
|
|
6344
6732
|
console.debug(`Successfully uploaded ${uploadCount} files`);
|
|
6345
6733
|
}
|
|
@@ -6347,67 +6735,69 @@ var Uploader = class {
|
|
|
6347
6735
|
console.debug(`Failed to upload ${failureCount} files`);
|
|
6348
6736
|
}
|
|
6349
6737
|
}
|
|
6350
|
-
async uploadFile(name, contents
|
|
6351
|
-
const
|
|
6352
|
-
const knowledgeFile = (
|
|
6353
|
-
|
|
6354
|
-
)
|
|
6738
|
+
async uploadFile(name, contents) {
|
|
6739
|
+
const knowledgeFiles = this.knowledgeFilesCache ?? [];
|
|
6740
|
+
const knowledgeFile = knowledgeFiles.find((f) => f.meta.name === name);
|
|
6741
|
+
const contentHash = calculateFileHash(contents);
|
|
6742
|
+
if (knowledgeFile && !this.config.force) {
|
|
6743
|
+
const existingHash = this.fileHashCache.get(knowledgeFile.id);
|
|
6744
|
+
if (existingHash === contentHash) {
|
|
6745
|
+
return { skipped: true, success: true };
|
|
6746
|
+
}
|
|
6747
|
+
}
|
|
6355
6748
|
if (knowledgeFile) {
|
|
6356
|
-
|
|
6357
|
-
|
|
6358
|
-
|
|
6359
|
-
|
|
6360
|
-
return { skipped: true, success: true };
|
|
6361
|
-
}
|
|
6362
|
-
await mkdir3(resolve6(process.cwd(), `./temp/diff`), {
|
|
6363
|
-
recursive: true
|
|
6364
|
-
}).catch();
|
|
6365
|
-
await writeFile4(
|
|
6366
|
-
resolve6(process.cwd(), `./temp/diff/${name}-current.md`),
|
|
6367
|
-
contents,
|
|
6368
|
-
"utf-8"
|
|
6369
|
-
);
|
|
6370
|
-
await writeFile4(
|
|
6371
|
-
resolve6(process.cwd(), `./temp/diff/${name}-owui.md`),
|
|
6372
|
-
data,
|
|
6749
|
+
try {
|
|
6750
|
+
const fileId = knowledgeFile.id;
|
|
6751
|
+
const fileString = await readFile2(
|
|
6752
|
+
resolve6(this.config.knowledgeFilePath, name),
|
|
6373
6753
|
"utf-8"
|
|
6374
6754
|
);
|
|
6375
|
-
const
|
|
6376
|
-
|
|
6377
|
-
|
|
6378
|
-
|
|
6379
|
-
|
|
6380
|
-
|
|
6381
|
-
|
|
6382
|
-
return { skipped: true, success: true };
|
|
6383
|
-
}
|
|
6384
|
-
}
|
|
6755
|
+
const spinner2 = ora(`Updating ${name}`).start();
|
|
6756
|
+
await this.filesApi.updateDataContent(fileId, fileString);
|
|
6757
|
+
spinner2.succeed(`Updated ${name}`);
|
|
6758
|
+
return { success: true };
|
|
6759
|
+
} catch (e) {
|
|
6760
|
+
console.warn(`Failed to update existing file ${name}:`, e);
|
|
6761
|
+
return { success: false };
|
|
6385
6762
|
}
|
|
6386
6763
|
}
|
|
6764
|
+
const spinner = ora(`Uploading ${name}`).start();
|
|
6387
6765
|
const fileBuffer = await readFile2(
|
|
6388
6766
|
resolve6(this.config.knowledgeFilePath, name)
|
|
6389
6767
|
);
|
|
6390
|
-
|
|
6391
|
-
|
|
6392
|
-
|
|
6393
|
-
|
|
6394
|
-
|
|
6395
|
-
|
|
6396
|
-
|
|
6397
|
-
|
|
6398
|
-
|
|
6399
|
-
|
|
6400
|
-
|
|
6401
|
-
|
|
6402
|
-
|
|
6403
|
-
|
|
6404
|
-
|
|
6405
|
-
|
|
6406
|
-
|
|
6407
|
-
|
|
6408
|
-
|
|
6409
|
-
|
|
6410
|
-
|
|
6768
|
+
let uploadedFileId = void 0;
|
|
6769
|
+
try {
|
|
6770
|
+
const uploadResponse = await this.filesApi.upload(fileBuffer, name, {
|
|
6771
|
+
processInBackground: false
|
|
6772
|
+
});
|
|
6773
|
+
uploadedFileId = uploadResponse.id;
|
|
6774
|
+
if (!uploadResponse.id || !uploadResponse.filename) {
|
|
6775
|
+
spinner.fail(`Error uploading ${name}`);
|
|
6776
|
+
return {
|
|
6777
|
+
error: { fileId: uploadResponse.id, uploadResponse },
|
|
6778
|
+
success: false
|
|
6779
|
+
};
|
|
6780
|
+
}
|
|
6781
|
+
spinner.text = `Associating ${name} with knowledge base`;
|
|
6782
|
+
const addResponse = await this.knowledgeApi.addFile(
|
|
6783
|
+
this.config.knowledgeId,
|
|
6784
|
+
uploadResponse.id
|
|
6785
|
+
);
|
|
6786
|
+
if (addResponse.name) {
|
|
6787
|
+
spinner.succeed(`${name} associated with knowledge base`);
|
|
6788
|
+
this.fileHashCache.set(uploadResponse.id, contentHash);
|
|
6789
|
+
return { success: true };
|
|
6790
|
+
} else {
|
|
6791
|
+
spinner.stop();
|
|
6792
|
+
return {
|
|
6793
|
+
error: { addResponse, fileId: uploadResponse.id },
|
|
6794
|
+
success: false
|
|
6795
|
+
};
|
|
6796
|
+
}
|
|
6797
|
+
} catch (e) {
|
|
6798
|
+
spinner.fail(`Error uploading ${name}`);
|
|
6799
|
+
const detail = e instanceof Error ? e.message : String(e);
|
|
6800
|
+
return { error: { detail, fileId: uploadedFileId }, success: false };
|
|
6411
6801
|
}
|
|
6412
6802
|
}
|
|
6413
6803
|
async uploadKnowledge() {
|
|
@@ -6419,35 +6809,84 @@ var Uploader = class {
|
|
|
6419
6809
|
if (stats.isDirectory()) {
|
|
6420
6810
|
return this.uploadDirectory();
|
|
6421
6811
|
} else {
|
|
6812
|
+
console.error("Not a directory, can't upload.");
|
|
6422
6813
|
}
|
|
6423
6814
|
}
|
|
6424
6815
|
};
|
|
6425
6816
|
function addUploadKnowledgeCommand() {
|
|
6426
|
-
|
|
6427
|
-
"--force",
|
|
6428
|
-
"force upload files, even if their contents have not changed"
|
|
6429
|
-
).action(async (options) => {
|
|
6430
|
-
loadEnv();
|
|
6817
|
+
function getUploader(knowledgePath, forceUpload) {
|
|
6431
6818
|
const sharedConfig = getConfigFromEnv();
|
|
6432
|
-
const knowledgeFilePath =
|
|
6819
|
+
const knowledgeFilePath = knowledgePath || process.env.KNOWLEDGE_OUTPUT_PATH;
|
|
6433
6820
|
if (!knowledgeFilePath) {
|
|
6434
6821
|
throw new Error(
|
|
6435
6822
|
"KNOWLEDGE_FILE_PATH must be set or provided as the --path option"
|
|
6436
6823
|
);
|
|
6437
6824
|
}
|
|
6438
|
-
|
|
6825
|
+
return new Uploader({
|
|
6439
6826
|
...sharedConfig,
|
|
6440
|
-
force:
|
|
6827
|
+
force: forceUpload,
|
|
6441
6828
|
knowledgeFilePath
|
|
6442
6829
|
});
|
|
6443
|
-
|
|
6830
|
+
}
|
|
6831
|
+
program.name("upload-knowledge").description("Upload files to OpenWebUI knowledge base").command("upload-knowledge").option("-p, --path <path>", "Path to file or folder relative to script").option(
|
|
6832
|
+
"--force",
|
|
6833
|
+
"force upload files, even if their contents have not changed"
|
|
6834
|
+
).action(async (options) => {
|
|
6835
|
+
loadEnv();
|
|
6836
|
+
return getUploader(options.path, options.force).uploadKnowledge();
|
|
6837
|
+
});
|
|
6838
|
+
program.command("get-knowledge-files").description("Get files from OpenWebUI knowledge base").option("-p, --path <path>", "Path to file or folder relative to script").action(async (options) => {
|
|
6839
|
+
loadEnv();
|
|
6840
|
+
const uploader = getUploader(options.path);
|
|
6841
|
+
const files = await uploader.filesApi.search("*");
|
|
6842
|
+
console.debug(`found ${files.length} files`);
|
|
6843
|
+
writeFileSync(
|
|
6844
|
+
resolve6(uploader.config.knowledgeFilePath, "files.json"),
|
|
6845
|
+
JSON.stringify(files, null, 2),
|
|
6846
|
+
"utf-8"
|
|
6847
|
+
);
|
|
6848
|
+
});
|
|
6849
|
+
program.command("clear-knowledge").description("Remove all files from the knowledge base collection").action(async () => {
|
|
6850
|
+
loadEnv();
|
|
6851
|
+
const sharedConfig = getConfigFromEnv();
|
|
6852
|
+
const apiConfig = {
|
|
6853
|
+
apiKey: sharedConfig.webUiKey,
|
|
6854
|
+
baseUrl: sharedConfig.webUiUrl
|
|
6855
|
+
};
|
|
6856
|
+
const filesApi = new FilesApi(apiConfig);
|
|
6857
|
+
const knowledgeApi = new KnowledgeApi(apiConfig);
|
|
6858
|
+
const cleaner = new KnowledgeCleaner(sharedConfig);
|
|
6859
|
+
await cleaner.cleanUpOrphanedFiles();
|
|
6860
|
+
const knowledge = await knowledgeApi.getById(sharedConfig.knowledgeId);
|
|
6861
|
+
const knowledgeFiles = knowledge.files ?? [];
|
|
6862
|
+
if (!knowledge) {
|
|
6863
|
+
console.log("Knowledge base not found");
|
|
6864
|
+
return;
|
|
6865
|
+
}
|
|
6866
|
+
const files = await filesApi.list(false).then(
|
|
6867
|
+
(files2) => files2.filter((file) => file.meta?.collection_name === knowledge.id)
|
|
6868
|
+
);
|
|
6869
|
+
if (files.length === 0) {
|
|
6870
|
+
console.log("No files in knowledge base");
|
|
6871
|
+
return;
|
|
6872
|
+
}
|
|
6873
|
+
console.log(`Removing ${files.length} files from knowledge base...`);
|
|
6874
|
+
for (const file of files) {
|
|
6875
|
+
if (knowledgeFiles.some((f) => f.id === file.id)) {
|
|
6876
|
+
await knowledgeApi.removeFile(knowledge.id, file.id, true);
|
|
6877
|
+
} else {
|
|
6878
|
+
await filesApi.delete(file.id);
|
|
6879
|
+
}
|
|
6880
|
+
console.log(`Removed ${file.id}`);
|
|
6881
|
+
}
|
|
6882
|
+
console.log(`Removed ${files.length} files`);
|
|
6444
6883
|
});
|
|
6445
6884
|
}
|
|
6446
6885
|
|
|
6447
6886
|
// src/react-demo-plugin/generate-lazy-demo-map.ts
|
|
6448
6887
|
import { glob as glob3 } from "glob";
|
|
6449
6888
|
import { uniqBy } from "lodash-es";
|
|
6450
|
-
import { writeFile as
|
|
6889
|
+
import { writeFile as writeFile4 } from "node:fs/promises";
|
|
6451
6890
|
import { resolve as resolve8 } from "node:path";
|
|
6452
6891
|
import { dedent as dedent2 } from "@qualcomm-ui/utils/dedent";
|
|
6453
6892
|
|
|
@@ -6516,7 +6955,7 @@ async function generateLazyDemoMap(options) {
|
|
|
6516
6955
|
const demoPages = await scanForDemoPages({ routesDir });
|
|
6517
6956
|
console.log(`Found ${demoPages.length} pages with demos`);
|
|
6518
6957
|
const content = generateLazyDemoLoader(demoPages);
|
|
6519
|
-
await
|
|
6958
|
+
await writeFile4(outputPath, content, "utf-8");
|
|
6520
6959
|
console.log(`
|
|
6521
6960
|
Generated lazy demo loader at: ${outputPath}`);
|
|
6522
6961
|
}
|