@astrofoundry/grimoire 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +148 -0
- package/dist/apikey.d.ts +5 -0
- package/dist/apikey.d.ts.map +1 -0
- package/dist/apikey.js +85 -0
- package/dist/apikey.js.map +1 -0
- package/dist/chunker.d.ts +7 -0
- package/dist/chunker.d.ts.map +1 -0
- package/dist/chunker.js +153 -0
- package/dist/chunker.js.map +1 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +496 -0
- package/dist/cli.js.map +1 -0
- package/dist/config.d.ts +18 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +76 -0
- package/dist/config.js.map +1 -0
- package/dist/consumer-config.d.ts +11 -0
- package/dist/consumer-config.d.ts.map +1 -0
- package/dist/consumer-config.js +58 -0
- package/dist/consumer-config.js.map +1 -0
- package/dist/consumer.d.ts +8 -0
- package/dist/consumer.d.ts.map +1 -0
- package/dist/consumer.js +71 -0
- package/dist/consumer.js.map +1 -0
- package/dist/converter.d.ts +12 -0
- package/dist/converter.d.ts.map +1 -0
- package/dist/converter.js +95 -0
- package/dist/converter.js.map +1 -0
- package/dist/embedder.d.ts +3 -0
- package/dist/embedder.d.ts.map +1 -0
- package/dist/embedder.js +38 -0
- package/dist/embedder.js.map +1 -0
- package/dist/format.d.ts +5 -0
- package/dist/format.d.ts.map +1 -0
- package/dist/format.js +6 -0
- package/dist/format.js.map +1 -0
- package/dist/reranker.d.ts +6 -0
- package/dist/reranker.d.ts.map +1 -0
- package/dist/reranker.js +21 -0
- package/dist/reranker.js.map +1 -0
- package/dist/scraper.d.ts +9 -0
- package/dist/scraper.d.ts.map +1 -0
- package/dist/scraper.js +77 -0
- package/dist/scraper.js.map +1 -0
- package/dist/search.d.ts +8 -0
- package/dist/search.d.ts.map +1 -0
- package/dist/search.js +43 -0
- package/dist/search.js.map +1 -0
- package/dist/store.d.ts +11 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +102 -0
- package/dist/store.js.map +1 -0
- package/dist/types.d.ts +25 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +47 -0
package/dist/cli.js
ADDED
|
@@ -0,0 +1,496 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { parseArgs } from "node:util";
|
|
3
|
+
import { readFile, readdir, rm } from "node:fs/promises";
|
|
4
|
+
import { readFileSync, existsSync } from "node:fs";
|
|
5
|
+
import { join, resolve } from "node:path";
|
|
6
|
+
import { createInterface } from "node:readline";
|
|
7
|
+
import { stringify } from "yaml";
|
|
8
|
+
import { bold, cyan, yellow } from "./format.js";
|
|
9
|
+
import { detectConsumerMode, resolveConsumerConfig, cmdInit } from "./consumer-config.js";
|
|
10
|
+
import { cmdConsumerSearch, cmdConsumerList, cmdConsumerStats } from "./consumer.js";
|
|
11
|
+
import { loadConfig } from "./config.js";
|
|
12
|
+
import { scrapeSource, createBrowser } from "./scraper.js";
|
|
13
|
+
import { convertSource } from "./converter.js";
|
|
14
|
+
import { chunkMarkdown } from "./chunker.js";
|
|
15
|
+
import { embedTexts } from "./embedder.js";
|
|
16
|
+
import { storeChunks, purgeSource, updateSourceMeta, getAllSourcesMeta } from "./store.js";
|
|
17
|
+
import { search } from "./search.js";
|
|
18
|
+
import { cmdApiKey } from "./apikey.js";
|
|
19
|
+
const PROJECT_ROOT = resolve(import.meta.dirname, "..");
|
|
20
|
+
const CONFIG_PATH = join(PROJECT_ROOT, "config", "sources.yaml");
|
|
21
|
+
const DATA_DIR = join(PROJECT_ROOT, "data");
|
|
22
|
+
const envPath = join(PROJECT_ROOT, ".env");
|
|
23
|
+
if (existsSync(envPath)) {
|
|
24
|
+
for (const line of readFileSync(envPath, "utf-8").split("\n")) {
|
|
25
|
+
const trimmed = line.trim();
|
|
26
|
+
if (!trimmed || trimmed.startsWith("#"))
|
|
27
|
+
continue;
|
|
28
|
+
const eqIndex = trimmed.indexOf("=");
|
|
29
|
+
if (eqIndex === -1)
|
|
30
|
+
continue;
|
|
31
|
+
const key = trimmed.slice(0, eqIndex);
|
|
32
|
+
const value = trimmed.slice(eqIndex + 1);
|
|
33
|
+
if (!process.env[key]) {
|
|
34
|
+
process.env[key] = value;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
function prompt(rl, question) {
|
|
39
|
+
return new Promise((resolve) => rl.question(question, resolve));
|
|
40
|
+
}
|
|
41
|
+
async function cmdAdd() {
|
|
42
|
+
const args = parseArgs({
|
|
43
|
+
args: process.argv.slice(3),
|
|
44
|
+
options: {
|
|
45
|
+
url: { type: "string" },
|
|
46
|
+
},
|
|
47
|
+
allowPositionals: true,
|
|
48
|
+
});
|
|
49
|
+
const name = args.positionals[0];
|
|
50
|
+
const url = args.values.url;
|
|
51
|
+
if (!name || !url) {
|
|
52
|
+
console.error("Usage: grimoire add <name> --url <start_url>");
|
|
53
|
+
process.exit(1);
|
|
54
|
+
}
|
|
55
|
+
console.log("Scanning page...\n");
|
|
56
|
+
const browser = await createBrowser();
|
|
57
|
+
const context = await browser.newContext();
|
|
58
|
+
const page = await context.newPage();
|
|
59
|
+
try {
|
|
60
|
+
await page.goto(url, { waitUntil: "domcontentloaded" });
|
|
61
|
+
const navCandidates = await page.evaluate(() => {
|
|
62
|
+
const selectors = ["nav", "[role='navigation']"];
|
|
63
|
+
const results = [];
|
|
64
|
+
const seen = new Set();
|
|
65
|
+
for (const sel of selectors) {
|
|
66
|
+
for (const el of document.querySelectorAll(sel)) {
|
|
67
|
+
if (seen.has(el))
|
|
68
|
+
continue;
|
|
69
|
+
seen.add(el);
|
|
70
|
+
const links = el.querySelectorAll("a[href]");
|
|
71
|
+
const label = el.getAttribute("aria-label") ||
|
|
72
|
+
el.getAttribute("class") ||
|
|
73
|
+
el.tagName.toLowerCase();
|
|
74
|
+
results.push({
|
|
75
|
+
selector: sel,
|
|
76
|
+
label,
|
|
77
|
+
linkCount: links.length,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return results.sort((a, b) => b.linkCount - a.linkCount);
|
|
82
|
+
});
|
|
83
|
+
if (navCandidates.length === 0) {
|
|
84
|
+
console.error("No navigation elements found on this page.");
|
|
85
|
+
process.exit(1);
|
|
86
|
+
}
|
|
87
|
+
console.log("Navigation candidates:");
|
|
88
|
+
for (let i = 0; i < navCandidates.length; i++) {
|
|
89
|
+
const c = navCandidates[i];
|
|
90
|
+
console.log(` [${i + 1}] ${c.selector} (${c.label}) — ${c.linkCount} links`);
|
|
91
|
+
}
|
|
92
|
+
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
93
|
+
const navChoice = await prompt(rl, "\nSelect navigation: ");
|
|
94
|
+
const navIndex = parseInt(navChoice, 10) - 1;
|
|
95
|
+
if (isNaN(navIndex) || navIndex < 0 || navIndex >= navCandidates.length) {
|
|
96
|
+
console.error("Invalid selection.");
|
|
97
|
+
rl.close();
|
|
98
|
+
process.exit(1);
|
|
99
|
+
}
|
|
100
|
+
const selectedNav = navCandidates[navIndex];
|
|
101
|
+
const parsedUrl = new URL(url);
|
|
102
|
+
const defaultPattern = parsedUrl.pathname.replace(/\/$/, "");
|
|
103
|
+
const allLinks = await page.$$eval(`${selectedNav.selector} a[href]`, (links, pattern) => {
|
|
104
|
+
return [...new Set(links
|
|
105
|
+
.map((a) => a.href)
|
|
106
|
+
.filter((h) => h.startsWith("http") && !h.includes("?hl=") && !h.endsWith("#") && h.includes(pattern)))];
|
|
107
|
+
}, defaultPattern);
|
|
108
|
+
console.log(`\nFound ${allLinks.length} links matching ${defaultPattern}`);
|
|
109
|
+
const patternInput = await prompt(rl, `Include pattern [default: ${defaultPattern}]: `);
|
|
110
|
+
const includePattern = patternInput.trim() || defaultPattern;
|
|
111
|
+
const excludeInput = await prompt(rl, "Exclude patterns (comma-separated, optional): ");
|
|
112
|
+
const excludePatterns = excludeInput.trim()
|
|
113
|
+
? excludeInput.split(",").map((p) => p.trim())
|
|
114
|
+
: undefined;
|
|
115
|
+
rl.close();
|
|
116
|
+
const contentSelector = await page.evaluate(() => {
|
|
117
|
+
if (document.querySelector("article"))
|
|
118
|
+
return "article";
|
|
119
|
+
if (document.querySelector("main"))
|
|
120
|
+
return "main";
|
|
121
|
+
return "body";
|
|
122
|
+
});
|
|
123
|
+
const source = {
|
|
124
|
+
name: name.replace(/-/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()),
|
|
125
|
+
start_url: url,
|
|
126
|
+
nav_selector: selectedNav.selector,
|
|
127
|
+
content_selector: contentSelector,
|
|
128
|
+
include_patterns: [includePattern],
|
|
129
|
+
...(excludePatterns ? { exclude_patterns: excludePatterns } : {}),
|
|
130
|
+
rate_limit_ms: 1000,
|
|
131
|
+
};
|
|
132
|
+
let existingContent = "";
|
|
133
|
+
try {
|
|
134
|
+
existingContent = await readFile(CONFIG_PATH, "utf-8");
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
existingContent = "sources:\n";
|
|
138
|
+
}
|
|
139
|
+
const newEntry = stringify({ [name]: source }, { indent: 2 });
|
|
140
|
+
const indented = newEntry
|
|
141
|
+
.split("\n")
|
|
142
|
+
.map((line) => (line.trim() ? ` ${line}` : ""))
|
|
143
|
+
.join("\n");
|
|
144
|
+
await import("node:fs/promises").then((fs) => fs.writeFile(CONFIG_PATH, existingContent.trimEnd() + "\n" + indented, "utf-8"));
|
|
145
|
+
console.log(`\nSource "${name}" added to config/sources.yaml`);
|
|
146
|
+
console.log(`Run "grimoire refresh ${name}" to start scraping.`);
|
|
147
|
+
}
|
|
148
|
+
finally {
|
|
149
|
+
await browser.close();
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
async function loadEmbeddingsCache(cachePath) {
|
|
153
|
+
try {
|
|
154
|
+
const data = await readFile(cachePath, "utf-8");
|
|
155
|
+
return JSON.parse(data);
|
|
156
|
+
}
|
|
157
|
+
catch {
|
|
158
|
+
return null;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
async function cmdRefresh() {
|
|
162
|
+
const args = parseArgs({
|
|
163
|
+
args: process.argv.slice(3),
|
|
164
|
+
options: {
|
|
165
|
+
full: { type: "boolean", default: false },
|
|
166
|
+
all: { type: "boolean", default: false },
|
|
167
|
+
concurrency: { type: "string" },
|
|
168
|
+
limit: { type: "string" },
|
|
169
|
+
"from-raw": { type: "boolean", default: false },
|
|
170
|
+
"from-store": { type: "boolean", default: false },
|
|
171
|
+
},
|
|
172
|
+
allowPositionals: true,
|
|
173
|
+
});
|
|
174
|
+
const config = await loadConfig(CONFIG_PATH);
|
|
175
|
+
const sourcesToRefresh = args.values.all
|
|
176
|
+
? Object.keys(config.sources)
|
|
177
|
+
: [args.positionals[0]];
|
|
178
|
+
if (!args.values.all && !sourcesToRefresh[0]) {
|
|
179
|
+
console.error("Usage: grimoire refresh <source> [--full] [--from-raw] [--from-store] [--limit <n>] [--concurrency <n>]");
|
|
180
|
+
process.exit(1);
|
|
181
|
+
}
|
|
182
|
+
const concurrencyOverride = args.values.concurrency ? parseInt(args.values.concurrency, 10) : undefined;
|
|
183
|
+
const urlLimit = args.values.limit ? parseInt(args.values.limit, 10) : undefined;
|
|
184
|
+
for (const sourceName of sourcesToRefresh) {
|
|
185
|
+
const source = config.sources[sourceName];
|
|
186
|
+
if (!source) {
|
|
187
|
+
console.error(`Source "${sourceName}" not found in config.`);
|
|
188
|
+
process.exit(1);
|
|
189
|
+
}
|
|
190
|
+
if (concurrencyOverride) {
|
|
191
|
+
source.concurrency = concurrencyOverride;
|
|
192
|
+
}
|
|
193
|
+
const rawDir = join(DATA_DIR, "raw", sourceName);
|
|
194
|
+
const mdDir = join(DATA_DIR, "markdown", sourceName);
|
|
195
|
+
const embeddingsCachePath = join(rawDir, "embeddings.json");
|
|
196
|
+
console.log(`\nRefreshing "${sourceName}"...`);
|
|
197
|
+
if (args.values.full) {
|
|
198
|
+
console.log(" Purging existing chunks...");
|
|
199
|
+
const deleted = await purgeSource(sourceName);
|
|
200
|
+
console.log(` Deleted ${deleted} chunks.`);
|
|
201
|
+
await rm(rawDir, { recursive: true, force: true });
|
|
202
|
+
await rm(mdDir, { recursive: true, force: true });
|
|
203
|
+
}
|
|
204
|
+
let urls;
|
|
205
|
+
if (args.values["from-store"]) {
|
|
206
|
+
console.log(" Loading cached embeddings...");
|
|
207
|
+
const cached = await loadEmbeddingsCache(embeddingsCachePath);
|
|
208
|
+
if (!cached) {
|
|
209
|
+
console.error(" No cached embeddings found. Run without --from-store first.");
|
|
210
|
+
process.exit(1);
|
|
211
|
+
}
|
|
212
|
+
const mdFiles = await readdir(mdDir);
|
|
213
|
+
const allPages = [];
|
|
214
|
+
for (const f of mdFiles.filter((f) => f.endsWith(".md"))) {
|
|
215
|
+
const content = await readFile(join(mdDir, f), "utf-8");
|
|
216
|
+
const urlMatch = content.match(/^url: "(.+)"$/m);
|
|
217
|
+
const titleMatch = content.match(/^title: "(.+)"$/m);
|
|
218
|
+
allPages.push({
|
|
219
|
+
markdown: content,
|
|
220
|
+
url: urlMatch?.[1] ?? "",
|
|
221
|
+
title: titleMatch?.[1] ?? "Untitled",
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
console.log(" Chunking...");
|
|
225
|
+
const allChunks = allPages.flatMap((p) => chunkMarkdown(p.markdown, sourceName, p.url, p.title));
|
|
226
|
+
console.log(` Created ${allChunks.length} chunks.`);
|
|
227
|
+
if (cached.length !== allChunks.length) {
|
|
228
|
+
console.error(` Embeddings cache (${cached.length}) doesn't match chunk count (${allChunks.length}). Re-embed with --from-raw.`);
|
|
229
|
+
process.exit(1);
|
|
230
|
+
}
|
|
231
|
+
console.log(" Purging old chunks...");
|
|
232
|
+
await purgeSource(sourceName);
|
|
233
|
+
console.log(" Storing in Firestore...");
|
|
234
|
+
await storeChunks(allChunks, cached, (cur, total) => {
|
|
235
|
+
console.log(` [${cur}/${total}] stored`);
|
|
236
|
+
});
|
|
237
|
+
await updateSourceMeta(sourceName, allChunks.length, allPages.length);
|
|
238
|
+
console.log(` Done. ${allChunks.length} chunks stored for "${sourceName}".`);
|
|
239
|
+
continue;
|
|
240
|
+
}
|
|
241
|
+
if (args.values["from-raw"]) {
|
|
242
|
+
console.log(" Reading URLs from cached HTML...");
|
|
243
|
+
const rawFiles = await readdir(rawDir);
|
|
244
|
+
const htmlFiles = rawFiles.filter((f) => f.endsWith(".html"));
|
|
245
|
+
urls = [];
|
|
246
|
+
for (const f of htmlFiles) {
|
|
247
|
+
const htmlPath = join(rawDir, f);
|
|
248
|
+
const html = await readFile(htmlPath, "utf-8");
|
|
249
|
+
const match = html.match(/<link[^>]+rel="canonical"[^>]+href="([^"]+)"/);
|
|
250
|
+
if (match) {
|
|
251
|
+
urls.push(match[1]);
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
const ogMatch = html.match(/<meta[^>]+property="og:url"[^>]+content="([^"]+)"/);
|
|
255
|
+
if (ogMatch) {
|
|
256
|
+
urls.push(ogMatch[1]);
|
|
257
|
+
continue;
|
|
258
|
+
}
|
|
259
|
+
const slug = f.replace(/\.html$/, "");
|
|
260
|
+
urls.push(`https://unknown/${slug.replace(/-/g, "/")}`);
|
|
261
|
+
}
|
|
262
|
+
console.log(` Found ${urls.length} cached pages.`);
|
|
263
|
+
}
|
|
264
|
+
else {
|
|
265
|
+
console.log(" Scraping URLs...");
|
|
266
|
+
urls = await scrapeSource(source, sourceName, DATA_DIR, (cur, total, url) => {
|
|
267
|
+
console.log(` [${cur}/${total}] ${url}`);
|
|
268
|
+
});
|
|
269
|
+
console.log(` Found ${urls.length} pages.`);
|
|
270
|
+
}
|
|
271
|
+
if (urlLimit && urls.length > urlLimit) {
|
|
272
|
+
urls = urls.slice(0, urlLimit);
|
|
273
|
+
console.log(` Limited to ${urlLimit} pages.`);
|
|
274
|
+
}
|
|
275
|
+
console.log(" Converting to markdown...");
|
|
276
|
+
const pages = await convertSource(sourceName, urls, source.content_selector, source.remove_selectors, source.remove_text_patterns, DATA_DIR, source.concurrency, (cur, total) => {
|
|
277
|
+
if (cur % 10 === 0 || cur === total)
|
|
278
|
+
console.log(` [${cur}/${total}] converted`);
|
|
279
|
+
});
|
|
280
|
+
console.log(" Chunking...");
|
|
281
|
+
const allChunks = pages.flatMap((p) => chunkMarkdown(p.markdown, sourceName, p.url, p.title));
|
|
282
|
+
console.log(` Created ${allChunks.length} chunks.`);
|
|
283
|
+
console.log(" Embedding chunks...");
|
|
284
|
+
const texts = allChunks.map((c) => c.content);
|
|
285
|
+
const embeddings = [];
|
|
286
|
+
const EMBED_BATCH = 100;
|
|
287
|
+
for (let i = 0; i < texts.length; i += EMBED_BATCH) {
|
|
288
|
+
const batch = texts.slice(i, i + EMBED_BATCH);
|
|
289
|
+
const result = await embedTexts(batch);
|
|
290
|
+
embeddings.push(...result);
|
|
291
|
+
console.log(` [${Math.min(i + EMBED_BATCH, texts.length)}/${texts.length}] embedded`);
|
|
292
|
+
}
|
|
293
|
+
console.log(" Caching embeddings to disk...");
|
|
294
|
+
const { writeFile: writeFileAsync, mkdir: mkdirAsync } = await import("node:fs/promises");
|
|
295
|
+
await mkdirAsync(rawDir, { recursive: true });
|
|
296
|
+
await writeFileAsync(embeddingsCachePath, JSON.stringify(embeddings), "utf-8");
|
|
297
|
+
console.log(" Purging old chunks...");
|
|
298
|
+
await purgeSource(sourceName);
|
|
299
|
+
console.log(" Storing in Firestore...");
|
|
300
|
+
await storeChunks(allChunks, embeddings, (cur, total) => {
|
|
301
|
+
console.log(` [${cur}/${total}] stored`);
|
|
302
|
+
});
|
|
303
|
+
await updateSourceMeta(sourceName, allChunks.length, urls.length);
|
|
304
|
+
console.log(` Done. ${allChunks.length} chunks stored for "${sourceName}".`);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
async function cmdSearch() {
|
|
308
|
+
const args = parseArgs({
|
|
309
|
+
args: process.argv.slice(3),
|
|
310
|
+
options: {
|
|
311
|
+
source: { type: "string" },
|
|
312
|
+
},
|
|
313
|
+
allowPositionals: true,
|
|
314
|
+
});
|
|
315
|
+
const query = args.positionals.join(" ");
|
|
316
|
+
if (!query) {
|
|
317
|
+
console.error("Usage: grimoire search \"<query>\" [--source <name>]");
|
|
318
|
+
process.exit(1);
|
|
319
|
+
}
|
|
320
|
+
const results = await search(query, { source: args.values.source });
|
|
321
|
+
if (results.length === 0) {
|
|
322
|
+
console.log("No results found.");
|
|
323
|
+
return;
|
|
324
|
+
}
|
|
325
|
+
for (let i = 0; i < results.length; i++) {
|
|
326
|
+
const r = results[i];
|
|
327
|
+
const score = r.relevance_score < 0.001 ? r.relevance_score.toExponential(2) : r.relevance_score.toFixed(4);
|
|
328
|
+
console.log(`\n${bold(`[${i + 1}] ${r.title}`)} (${score})`);
|
|
329
|
+
console.log(` ${cyan(r.url)}`);
|
|
330
|
+
console.log(` ${yellow(r.heading_path.join(" > "))}`);
|
|
331
|
+
console.log(` ${r.content.slice(0, 200).replace(/\n/g, " ")}...`);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
async function cmdList() {
|
|
335
|
+
const config = await loadConfig(CONFIG_PATH);
|
|
336
|
+
const metas = await getAllSourcesMeta();
|
|
337
|
+
const metaMap = new Map(metas.map((m) => [m.source, m]));
|
|
338
|
+
console.log("\nSources:\n");
|
|
339
|
+
for (const [key, source] of Object.entries(config.sources)) {
|
|
340
|
+
const meta = metaMap.get(key);
|
|
341
|
+
const status = meta
|
|
342
|
+
? `${meta.chunk_count} chunks, ${meta.url_count} URLs, last refreshed ${meta.last_refreshed}`
|
|
343
|
+
: "not yet refreshed";
|
|
344
|
+
console.log(` ${bold(key)} — ${source.name}`);
|
|
345
|
+
console.log(` ${status}`);
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
async function cmdStats() {
|
|
349
|
+
const metas = await getAllSourcesMeta();
|
|
350
|
+
if (metas.length === 0) {
|
|
351
|
+
console.log("No sources have been refreshed yet.");
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
let totalChunks = 0;
|
|
355
|
+
let totalUrls = 0;
|
|
356
|
+
console.log("\nSource Statistics:\n");
|
|
357
|
+
for (const meta of metas) {
|
|
358
|
+
console.log(` ${bold(meta.source)}`);
|
|
359
|
+
console.log(` Chunks: ${meta.chunk_count}`);
|
|
360
|
+
console.log(` URLs: ${meta.url_count}`);
|
|
361
|
+
console.log(` Last refreshed: ${meta.last_refreshed}`);
|
|
362
|
+
totalChunks += meta.chunk_count;
|
|
363
|
+
totalUrls += meta.url_count;
|
|
364
|
+
}
|
|
365
|
+
console.log(`\n Total: ${totalChunks} chunks across ${totalUrls} URLs from ${metas.length} sources`);
|
|
366
|
+
}
|
|
367
|
+
async function cmdExport() {
|
|
368
|
+
const args = parseArgs({
|
|
369
|
+
args: process.argv.slice(3),
|
|
370
|
+
options: {
|
|
371
|
+
format: { type: "string", default: "json" },
|
|
372
|
+
},
|
|
373
|
+
allowPositionals: true,
|
|
374
|
+
});
|
|
375
|
+
const sourceName = args.positionals[0];
|
|
376
|
+
if (!sourceName) {
|
|
377
|
+
console.error("Usage: grimoire export <source> [--format json]");
|
|
378
|
+
process.exit(1);
|
|
379
|
+
}
|
|
380
|
+
const mdDir = join(DATA_DIR, "markdown", sourceName);
|
|
381
|
+
let files;
|
|
382
|
+
try {
|
|
383
|
+
files = await readdir(mdDir);
|
|
384
|
+
}
|
|
385
|
+
catch {
|
|
386
|
+
console.error(`No markdown data found for source "${sourceName}".`);
|
|
387
|
+
process.exit(1);
|
|
388
|
+
}
|
|
389
|
+
const pages = [];
|
|
390
|
+
for (const file of files.filter((f) => f.endsWith(".md"))) {
|
|
391
|
+
const content = await readFile(join(mdDir, file), "utf-8");
|
|
392
|
+
pages.push({ file, content });
|
|
393
|
+
}
|
|
394
|
+
console.log(JSON.stringify(pages, null, 2));
|
|
395
|
+
}
|
|
396
|
+
const ADMIN_COMMANDS = {
|
|
397
|
+
add: cmdAdd,
|
|
398
|
+
refresh: cmdRefresh,
|
|
399
|
+
search: cmdSearch,
|
|
400
|
+
list: cmdList,
|
|
401
|
+
stats: cmdStats,
|
|
402
|
+
export: cmdExport,
|
|
403
|
+
apikey: cmdApiKey,
|
|
404
|
+
};
|
|
405
|
+
const ADMIN_ONLY_COMMANDS = ["add", "refresh", "export", "apikey"];
|
|
406
|
+
function showHelp(isConsumer) {
|
|
407
|
+
if (isConsumer) {
|
|
408
|
+
console.log(`
|
|
409
|
+
grimoire — Documentation Search
|
|
410
|
+
|
|
411
|
+
Commands:
|
|
412
|
+
search "<query>" [--source <n>] Search documentation
|
|
413
|
+
list List available sources
|
|
414
|
+
stats Show statistics
|
|
415
|
+
init Configure API connection
|
|
416
|
+
|
|
417
|
+
Environment:
|
|
418
|
+
GRIMOIRE_API_URL API endpoint URL
|
|
419
|
+
GRIMOIRE_API_KEY API key
|
|
420
|
+
`);
|
|
421
|
+
}
|
|
422
|
+
else {
|
|
423
|
+
console.log(`
|
|
424
|
+
grimoire — Documentation RAG System (admin)
|
|
425
|
+
|
|
426
|
+
Commands:
|
|
427
|
+
add <name> --url <url> Add a new documentation source
|
|
428
|
+
refresh <source> [--full] Refresh a source (partial by default)
|
|
429
|
+
refresh --all Refresh all sources
|
|
430
|
+
search "<query>" [--source <n>] Search documentation
|
|
431
|
+
list List all sources
|
|
432
|
+
stats Show chunk/source statistics
|
|
433
|
+
export <source> Export source as JSON
|
|
434
|
+
apikey create <name> Generate an API key
|
|
435
|
+
apikey list List API keys
|
|
436
|
+
apikey revoke <name> Revoke an API key
|
|
437
|
+
`);
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
async function main() {
|
|
441
|
+
const command = process.argv[2];
|
|
442
|
+
const isConsumer = await detectConsumerMode();
|
|
443
|
+
if (!command || command === "--help" || command === "-h") {
|
|
444
|
+
showHelp(isConsumer);
|
|
445
|
+
process.exit(0);
|
|
446
|
+
}
|
|
447
|
+
if (isConsumer) {
|
|
448
|
+
if (ADMIN_ONLY_COMMANDS.includes(command)) {
|
|
449
|
+
console.error(`The '${command}' command is only available in admin mode.`);
|
|
450
|
+
process.exit(1);
|
|
451
|
+
}
|
|
452
|
+
const config = await resolveConsumerConfig();
|
|
453
|
+
if (command === "init") {
|
|
454
|
+
await cmdInit();
|
|
455
|
+
}
|
|
456
|
+
else if (command === "search") {
|
|
457
|
+
const args = parseArgs({
|
|
458
|
+
args: process.argv.slice(3),
|
|
459
|
+
options: { source: { type: "string" } },
|
|
460
|
+
allowPositionals: true,
|
|
461
|
+
});
|
|
462
|
+
const query = args.positionals[0];
|
|
463
|
+
if (!query) {
|
|
464
|
+
console.error("Usage: grimoire search \"<query>\" [--source <name>]");
|
|
465
|
+
process.exit(1);
|
|
466
|
+
}
|
|
467
|
+
await cmdConsumerSearch(config, query, { source: args.values.source });
|
|
468
|
+
}
|
|
469
|
+
else if (command === "list") {
|
|
470
|
+
await cmdConsumerList(config);
|
|
471
|
+
}
|
|
472
|
+
else if (command === "stats") {
|
|
473
|
+
await cmdConsumerStats(config);
|
|
474
|
+
}
|
|
475
|
+
else {
|
|
476
|
+
console.error(`Unknown command: ${command}. Run "grimoire --help" for usage.`);
|
|
477
|
+
process.exit(1);
|
|
478
|
+
}
|
|
479
|
+
return;
|
|
480
|
+
}
|
|
481
|
+
if (command === "init") {
|
|
482
|
+
await cmdInit();
|
|
483
|
+
return;
|
|
484
|
+
}
|
|
485
|
+
const handler = ADMIN_COMMANDS[command];
|
|
486
|
+
if (!handler) {
|
|
487
|
+
console.error(`Unknown command: ${command}. Run "grimoire --help" for usage.`);
|
|
488
|
+
process.exit(1);
|
|
489
|
+
}
|
|
490
|
+
await handler();
|
|
491
|
+
}
|
|
492
|
+
main().catch((err) => {
|
|
493
|
+
console.error(`Error: ${err.message}`);
|
|
494
|
+
process.exit(1);
|
|
495
|
+
});
|
|
496
|
+
//# sourceMappingURL=cli.js.map
|
package/dist/cli.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC1F,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACrF,OAAO,EAAE,UAAU,EAAqB,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC3F,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AACxD,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC;AACjE,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;AAE5C,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;AAC3C,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;IACxB,KAAK,MAAM,IAAI,IAAI,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAClD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,OAAO,KAAK,CAAC,CAAC;YAAE,SAAS;QAC7B,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;QACzC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QAC3B,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,MAAM,CAAC,EAAsC,EAAE,QAAgB;IACtE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,KAAK,UAAU,MAAM;IACnB,MAAM,IAAI,GAAG,SAAS,CAAC;QACrB,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAC3B,OAAO,EAAE;YACP,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;SACxB;QACD,gBAAgB,EAAE,IAAI;KACvB,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;IACjC,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC;IAE5B,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IAElC,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;IAC3C,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IAErC,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;QAExD,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;YAC7C,MAAM,SAAS,GAAG,CAAC,KAAK,EAAE,qBAAqB,CAAC,CAAC;YACjD,MAAM,OAAO,GAA6D,EAAE,CAAC;YAC7E,MAAM,IAAI,GAAG,IAAI,GAAG,EAAW,CAAC;YAEhC,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;oBAChD,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;wBAAE,SAAS;oBAC3B,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACb,MAAM,KAAK,GAAG,EAAE,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;oBAC7C,MAAM,KAAK,GACT,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC;wBAC7B,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC;wBACxB,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;oBAC3B,OAAO,CAAC,IAAI,CAAC;wBACX,QAAQ,EAAE,GAAG;wBACb,KAAK;wBACL,SAAS,EAAE,KAAK,CAAC,MAAM;qBACxB,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;QAEH,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;YAC5D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9C,MAAM,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,SAAS,QAAQ,CAAC,CAAC;QAChF,CAAC;QAED,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAE7E,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,EAAE,EAAE,uBAAuB,CAAC,CAAC;QAC5D,MAAM,QAAQ,GAAG,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,IAAI,QAAQ,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YACxE,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;YACpC,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,MAAM,WAAW,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;QAE5C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,cAAc,GAAG,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAE7D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAChC,GAAG,WAAW,CAAC,QAAQ,UAAU,EACjC,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YACjB,OAAO,CAAC,GAAG,IAAI,GAAG,CACf,KAA6B;qBAC3B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;qBAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CACzG,CAAC,CAAC;QACL,CAAC,EACD,cAAc,CACf,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,mBAAmB,cAAc,EAAE,CAAC,CAAC;QAE3E,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,EAAE,EAAE,6BAA6B,cAAc,KAAK,CAAC,CAAC;QACxF,MAAM,cAAc,GAAG,YAAY,CAAC,IAAI,EAAE,IAAI,cAAc,CAAC;QAE7D,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,EAAE,EAAE,gDAAgD,CAAC,CAAC;QACxF,MAAM,eAAe,GAAG,YAAY,CAAC,IAAI,EAAE;YACzC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC9C,CAAC,CAAC,SAAS,CAAC;QAEd,EAAE,CAAC,KAAK,EAAE,CAAC;QAEX,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;YAC/C,IAAI,QAAQ,CAAC,aAAa,CAAC,SAAS,CAAC;gBAAE,OAAO,SAAS,CAAC;YACxD,IAAI,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC;gBAAE,OAAO,MAAM,CAAC;YAClD,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAiB;YAC3B,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACtE,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,WAAW,CAAC,QAAQ;YAClC,gBAAgB,EAAE,eAAe;YACjC,gBAAgB,EAAE,CAAC,cAAc,CAAC;YAClC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,aAAa,EAAE,IAAI;SACpB,CAAC;QAEF,IAAI,eAAe,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,eAAe,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACzD,CAAC;QAAC,MAAM,CAAC;YACP,eAAe,GAAG,YAAY,CAAC;QACjC,CAAC;QAED,MAAM,QAAQ,GAAG,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9D,MAAM,QAAQ,GAAG,QAAQ;aACtB,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;aAC/C,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAC3C,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,eAAe,CAAC,OAAO,EAAE,GAAG,IAAI,GAAG,QAAQ,EAAE,OAAO,CAAC,CAChF,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,gCAAgC,CAAC,CAAC;QAC/D,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,sBAAsB,CAAC,CAAC;IACnE,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,SAAiB;IAClD,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAChD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAe,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,MAAM,IAAI,GAAG,SAAS,CAAC;QACrB,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAC3B,OAAO,EAAE;YACP,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YACzC,GAAG,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YACxC,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC/B,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACzB,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YAC/C,YAAY,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;SAClD;QACD,gBAAgB,EAAE,IAAI;KACvB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,WAAW,CAAC,CAAC;IAC7C,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG;QACtC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;QAC7B,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IAE1B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,OAAO,CAAC,KAAK,CAAC,yGAAyG,CAAC,CAAC;QACzH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACxG,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEjF,KAAK,MAAM,UAAU,IAAI,gBAAgB,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,CAAC,KAAK,CAAC,WAAW,UAAU,wBAAwB,CAAC,CAAC;YAC7D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,IAAI,mBAAmB,EAAE,CAAC;YACxB,MAAM,CAAC,WAAW,GAAG,mBAAmB,CAAC;QAC3C,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;QACrD,MAAM,mBAAmB,GAAG,IAAI,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;QAE5D,OAAO,CAAC,GAAG,CAAC,iBAAiB,UAAU,MAAM,CAAC,CAAC;QAE/C,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACrB,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,CAAC;YAC9C,OAAO,CAAC,GAAG,CAAC,aAAa,OAAO,UAAU,CAAC,CAAC;YAC5C,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACnD,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACpD,CAAC;QAED,IAAI,IAAc,CAAC;QAEnB,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;YAC9B,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;YAC9C,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,mBAAmB,CAAC,CAAC;YAC9D,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAC;gBAC/E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,QAAQ,GAAG,EAAE,CAAC;YACpB,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACzD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBACxD,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;gBACjD,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACrD,QAAQ,CAAC,IAAI,CAAC;oBACZ,QAAQ,EAAE,OAAO;oBACjB,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE;oBACxB,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU;iBACrC,CAAC,CAAC;YACL,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;YAC7B,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YACjG,OAAO,CAAC,GAAG,CAAC,aAAa,SAAS,CAAC,MAAM,UAAU,CAAC,CAAC;YAErD,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;gBACvC,OAAO,CAAC,KAAK,CAAC,uBAAuB,MAAM,CAAC,MAAM,gCAAgC,SAAS,CAAC,MAAM,8BAA8B,CAAC,CAAC;gBAClI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;YACvC,MAAM,WAAW,CAAC,UAAU,CAAC,CAAC;YAE9B,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;YACzC,MAAM,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;gBAClD,OAAO,CAAC,GAAG,CAAC,MAAM,GAAG,IAAI,KAAK,UAAU,CAAC,CAAC;YAC5C,CAAC,CAAC,CAAC;YAEH,MAAM,gBAAgB,CAAC,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;YACtE,OAAO,CAAC,GAAG,CAAC,WAAW,SAAS,CAAC,MAAM,uBAAuB,UAAU,IAAI,CAAC,CAAC;YAC9E,SAAS;QACX,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;YAClD,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;YACvC,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;YAC9D,IAAI,GAAG,EAAE,CAAC;YACV,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;gBAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBACjC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;gBACzE,IAAI,KAAK,EAAE,CAAC;oBAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;oBAAC,SAAS;gBAAC,CAAC;gBAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,mDAAmD,CAAC,CAAC;gBAChF,IAAI,OAAO,EAAE,CAAC;oBAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;oBAAC,SAAS;gBAAC,CAAC;gBACjD,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;gBACtC,IAAI,CAAC,IAAI,CAAC,mBAAmB,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YAC1D,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,MAAM,gBAAgB,CAAC,CAAC;QACtD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;YAClC,IAAI,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;gBAC1E,OAAO,CAAC,GAAG,CAAC,MAAM,GAAG,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC,CAAC;YAC5C,CAAC,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,MAAM,SAAS,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YACvC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,gBAAgB,QAAQ,SAAS,CAAC,CAAC;QACjD,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,MAAM,aAAa,CAC/B,UAAU,EACV,IAAI,EACJ,MAAM,CAAC,gBAAgB,EACvB,MAAM,CAAC,gBAAgB,EACvB,MAAM,CAAC,oBAAoB,EAC3B,QAAQ,EACR,MAAM,CAAC,WAAW,EAClB,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YACb,IAAI,GAAG,GAAG,EAAE,KAAK,CAAC,IAAI,GAAG,KAAK,KAAK;gBAAE,OAAO,CAAC,GAAG,CAAC,MAAM,GAAG,IAAI,KAAK,aAAa,CAAC,CAAC;QACpF,CAAC,CACF,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9F,OAAO,CAAC,GAAG,CAAC,aAAa,SAAS,CAAC,MAAM,UAAU,CAAC,CAAC;QAErD,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC9C,MAAM,UAAU,GAAe,EAAE,CAAC;QAClC,MAAM,WAAW,GAAG,GAAG,CAAC;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;YACnD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;YAC9C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;YACvC,UAAU,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,YAAY,CAAC,CAAC;QACzF,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;QAC/C,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;QAC1F,MAAM,UAAU,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,MAAM,cAAc,CAAC,mBAAmB,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC;QAE/E,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;QACvC,MAAM,WAAW,CAAC,UAAU,CAAC,CAAC;QAE9B,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;QACzC,MAAM,WAAW,CAAC,SAAS,EAAE,UAAU,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YACtD,OAAO,CAAC,GAAG,CAAC,MAAM,GAAG,IAAI,KAAK,UAAU,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,MAAM,gBAAgB,CAAC,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAClE,OAAO,CAAC,GAAG,CAAC,WAAW,SAAS,CAAC,MAAM,uBAAuB,UAAU,IAAI,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS;IACtB,MAAM,IAAI,GAAG,SAAS,CAAC;QACrB,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAC3B,OAAO,EAAE;YACP,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;SAC3B;QACD,gBAAgB,EAAE,IAAI;KACvB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAC;QACtE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAEpE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,OAAO;IACT,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,KAAK,GAAG,CAAC,CAAC,eAAe,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC5G,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,KAAK,KAAK,GAAG,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,OAAO,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;QACzD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,OAAO;IACpB,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,WAAW,CAAC,CAAC;IAC7C,MAAM,KAAK,GAAG,MAAM,iBAAiB,EAAE,CAAC;IACxC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAEzD,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAC5B,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,IAAI;YACjB,CAAC,CAAC,GAAG,IAAI,CAAC,WAAW,YAAY,IAAI,CAAC,SAAS,yBAAyB,IAAI,CAAC,cAAc,EAAE;YAC7F,CAAC,CAAC,mBAAmB,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/C,OAAO,CAAC,GAAG,CAAC,OAAO,MAAM,EAAE,CAAC,CAAC;IAC/B,CAAC;AACH,CAAC;AAED,KAAK,UAAU,QAAQ;IACrB,MAAM,KAAK,GAAG,MAAM,iBAAiB,EAAE,CAAC;IAExC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;QACnD,OAAO;IACT,CAAC;IAED,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QAC/C,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;QAC1D,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC;QAChC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;IAC9B,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,cAAc,WAAW,kBAAkB,SAAS,cAAc,KAAK,CAAC,MAAM,UAAU,CAAC,CAAC;AACxG,CAAC;AAED,KAAK,UAAU,SAAS;IACtB,MAAM,IAAI,GAAG,SAAS,CAAC;QACrB,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAC3B,OAAO,EAAE;YACP,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE;SAC5C;QACD,gBAAgB,EAAE,IAAI;KACvB,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;IACvC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,iDAAiD,CAAC,CAAC;QACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;IACrD,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,KAAK,CAAC,sCAAsC,UAAU,IAAI,CAAC,CAAC;QACpE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,KAAK,GAAG,EAAE,CAAC;IACjB,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAC1D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,cAAc,GAAwC;IAC1D,GAAG,EAAE,MAAM;IACX,OAAO,EAAE,UAAU;IACnB,MAAM,EAAE,SAAS;IACjB,IAAI,EAAE,OAAO;IACb,KAAK,EAAE,QAAQ;IACf,MAAM,EAAE,SAAS;IACjB,MAAM,EAAE,SAAS;CAClB,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;AAEnE,SAAS,QAAQ,CAAC,UAAmB;IACnC,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,CAAC,GAAG,CAAC;;;;;;;;;;;;CAYf,CAAC,CAAC;IACD,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC;;;;;;;;;;;;;;CAcf,CAAC,CAAC;IACD,CAAC;AACH,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChC,MAAM,UAAU,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAE9C,IAAI,CAAC,OAAO,IAAI,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACzD,QAAQ,CAAC,UAAU,CAAC,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,UAAU,EAAE,CAAC;QACf,IAAI,mBAAmB,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1C,OAAO,CAAC,KAAK,CAAC,QAAQ,OAAO,4CAA4C,CAAC,CAAC;YAC3E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,qBAAqB,EAAE,CAAC;QAE7C,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;YACvB,MAAM,OAAO,EAAE,CAAC;QAClB,CAAC;aAAM,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,SAAS,CAAC;gBACrB,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC3B,OAAO,EAAE,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;gBACvC,gBAAgB,EAAE,IAAI;aACvB,CAAC,CAAC;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;YAClC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAC;gBACtE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YACD,MAAM,iBAAiB,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACzE,CAAC;aAAM,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;YAC9B,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC;aAAM,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;YAC/B,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,oBAAoB,OAAO,oCAAoC,CAAC,CAAC;YAC/E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;QACvB,MAAM,OAAO,EAAE,CAAC;QAChB,OAAO;IACT,CAAC;IAED,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACxC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,oBAAoB,OAAO,oCAAoC,CAAC,CAAC;QAC/E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,OAAO,EAAE,CAAC;AAClB,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAU,EAAE,EAAE;IAC1B,OAAO,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
package/dist/config.d.ts
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export interface SourceConfig {
|
|
2
|
+
name: string;
|
|
3
|
+
start_url: string;
|
|
4
|
+
nav_selector: string;
|
|
5
|
+
content_selector: string;
|
|
6
|
+
remove_selectors?: string[];
|
|
7
|
+
remove_text_patterns?: string[];
|
|
8
|
+
include_patterns?: string[];
|
|
9
|
+
exclude_patterns?: string[];
|
|
10
|
+
rate_limit_ms?: number;
|
|
11
|
+
concurrency?: number;
|
|
12
|
+
}
|
|
13
|
+
export interface GrimoireConfig {
|
|
14
|
+
sources: Record<string, SourceConfig>;
|
|
15
|
+
}
|
|
16
|
+
export declare function validateConfig(data: unknown): GrimoireConfig;
|
|
17
|
+
export declare function loadConfig(path: string): Promise<GrimoireConfig>;
|
|
18
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;IAChC,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACvC;AASD,wBAAgB,cAAc,CAAC,IAAI,EAAE,OAAO,GAAG,cAAc,CA6E5D;AAED,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAItE"}
|
package/dist/config.js
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { parse } from "yaml";
|
|
3
|
+
const REQUIRED_SOURCE_FIELDS = [
|
|
4
|
+
"name",
|
|
5
|
+
"start_url",
|
|
6
|
+
"nav_selector",
|
|
7
|
+
"content_selector",
|
|
8
|
+
];
|
|
9
|
+
export function validateConfig(data) {
|
|
10
|
+
if (typeof data !== "object" || data === null || !("sources" in data)) {
|
|
11
|
+
throw new Error("Config must have a 'sources' key");
|
|
12
|
+
}
|
|
13
|
+
const { sources } = data;
|
|
14
|
+
if (typeof sources !== "object" || sources === null) {
|
|
15
|
+
throw new Error("'sources' must be an object");
|
|
16
|
+
}
|
|
17
|
+
const entries = Object.entries(sources);
|
|
18
|
+
if (entries.length === 0) {
|
|
19
|
+
throw new Error("'sources' must contain at least one source");
|
|
20
|
+
}
|
|
21
|
+
const validated = {};
|
|
22
|
+
for (const [key, value] of entries) {
|
|
23
|
+
if (typeof value !== "object" || value === null) {
|
|
24
|
+
throw new Error(`Source '${key}' must be an object`);
|
|
25
|
+
}
|
|
26
|
+
const source = value;
|
|
27
|
+
for (const field of REQUIRED_SOURCE_FIELDS) {
|
|
28
|
+
if (typeof source[field] !== "string" || source[field] === "") {
|
|
29
|
+
throw new Error(`Source '${key}' is missing required field '${field}'`);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
try {
|
|
33
|
+
new URL(source.start_url);
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
throw new Error(`Source '${key}' has invalid start_url: ${source.start_url}`);
|
|
37
|
+
}
|
|
38
|
+
if (source.remove_selectors !== undefined && !Array.isArray(source.remove_selectors)) {
|
|
39
|
+
throw new Error(`Source '${key}': remove_selectors must be an array`);
|
|
40
|
+
}
|
|
41
|
+
if (source.remove_text_patterns !== undefined && !Array.isArray(source.remove_text_patterns)) {
|
|
42
|
+
throw new Error(`Source '${key}': remove_text_patterns must be an array`);
|
|
43
|
+
}
|
|
44
|
+
if (source.include_patterns !== undefined && !Array.isArray(source.include_patterns)) {
|
|
45
|
+
throw new Error(`Source '${key}': include_patterns must be an array`);
|
|
46
|
+
}
|
|
47
|
+
if (source.exclude_patterns !== undefined && !Array.isArray(source.exclude_patterns)) {
|
|
48
|
+
throw new Error(`Source '${key}': exclude_patterns must be an array`);
|
|
49
|
+
}
|
|
50
|
+
if (source.rate_limit_ms !== undefined && typeof source.rate_limit_ms !== "number") {
|
|
51
|
+
throw new Error(`Source '${key}': rate_limit_ms must be a number`);
|
|
52
|
+
}
|
|
53
|
+
if (source.concurrency !== undefined && typeof source.concurrency !== "number") {
|
|
54
|
+
throw new Error(`Source '${key}': concurrency must be a number`);
|
|
55
|
+
}
|
|
56
|
+
validated[key] = {
|
|
57
|
+
name: source.name,
|
|
58
|
+
start_url: source.start_url,
|
|
59
|
+
nav_selector: source.nav_selector,
|
|
60
|
+
content_selector: source.content_selector,
|
|
61
|
+
remove_selectors: source.remove_selectors,
|
|
62
|
+
remove_text_patterns: source.remove_text_patterns,
|
|
63
|
+
include_patterns: source.include_patterns,
|
|
64
|
+
exclude_patterns: source.exclude_patterns,
|
|
65
|
+
rate_limit_ms: source.rate_limit_ms,
|
|
66
|
+
concurrency: source.concurrency,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
return { sources: validated };
|
|
70
|
+
}
|
|
71
|
+
export async function loadConfig(path) {
|
|
72
|
+
const content = await readFile(path, "utf-8");
|
|
73
|
+
const data = parse(content, { merge: true });
|
|
74
|
+
return validateConfig(data);
|
|
75
|
+
}
|
|
76
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAmB7B,MAAM,sBAAsB,GAAG;IAC7B,MAAM;IACN,WAAW;IACX,cAAc;IACd,kBAAkB;CACV,CAAC;AAEX,MAAM,UAAU,cAAc,CAAC,IAAa;IAC1C,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,EAAE,CAAC;QACtE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,EAAE,OAAO,EAAE,GAAG,IAA4B,CAAC;IAEjD,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IACjD,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,OAAkC,CAAC,CAAC;IAEnE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,SAAS,GAAiC,EAAE,CAAC;IAEnD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,WAAW,GAAG,qBAAqB,CAAC,CAAC;QACvD,CAAC;QAED,MAAM,MAAM,GAAG,KAAgC,CAAC;QAEhD,KAAK,MAAM,KAAK,IAAI,sBAAsB,EAAE,CAAC;YAC3C,IAAI,OAAO,MAAM,CAAC,KAAK,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;gBAC9D,MAAM,IAAI,KAAK,CAAC,WAAW,GAAG,gCAAgC,KAAK,GAAG,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;QAED,IAAI,CAAC;YACH,IAAI,GAAG,CAAC,MAAM,CAAC,SAAmB,CAAC,CAAC;QACtC,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,WAAW,GAAG,4BAA4B,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;QAChF,CAAC;QAED,IAAI,MAAM,CAAC,gBAAgB,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACrF,MAAM,IAAI,KAAK,CAAC,WAAW,GAAG,sCAAsC,CAAC,CAAC;QACxE,CAAC;QAED,IAAI,MAAM,CAAC,oBAAoB,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,oBAAoB,CAAC,EAAE,CAAC;YAC7F,MAAM,IAAI,KAAK,CAAC,WAAW,GAAG,0CAA0C,CAAC,CAAC;QAC5E,CAAC;QAED,IAAI,MAAM,CAAC,gBAAgB,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACrF,MAAM,IAAI,KAAK,CAAC,WAAW,GAAG,sCAAsC,CAAC,CAAC;QACxE,CAAC;QAED,IAAI,MAAM,CAAC,gBAAgB,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACrF,MAAM,IAAI,KAAK,CAAC,WAAW,GAAG,sCAAsC,CAAC,CAAC;QACxE,CAAC;QAED,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,IAAI,OAAO,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;YACnF,MAAM,IAAI,KAAK,CAAC,WAAW,GAAG,mCAAmC,CAAC,CAAC;QACrE,CAAC;QAED,IAAI,MAAM,CAAC,WAAW,KAAK,SAAS,IAAI,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;YAC/E,MAAM,IAAI,KAAK,CAAC,WAAW,GAAG,iCAAiC,CAAC,CAAC;QACnE,CAAC;QAED,SAAS,CAAC,GAAG,CAAC,GAAG;YACf,IAAI,EAAE,MAAM,CAAC,IAAc;YAC3B,SAAS,EAAE,MAAM,CAAC,SAAmB;YACrC,YAAY,EAAE,MAAM,CAAC,YAAsB;YAC3C,gBAAgB,EAAE,MAAM,CAAC,gBAA0B;YACnD,gBAAgB,EAAE,MAAM,CAAC,gBAAwC;YACjE,oBAAoB,EAAE,MAAM,CAAC,oBAA4C;YACzE,gBAAgB,EAAE,MAAM,CAAC,gBAAwC;YACjE,gBAAgB,EAAE,MAAM,CAAC,gBAAwC;YACjE,aAAa,EAAE,MAAM,CAAC,aAAmC;YACzD,WAAW,EAAE,MAAM,CAAC,WAAiC;SACtD,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAY;IAC3C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7C,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export interface ConsumerConfig {
|
|
2
|
+
apiUrl: string;
|
|
3
|
+
apiKey: string;
|
|
4
|
+
}
|
|
5
|
+
export declare function loadConsumerConfig(): Promise<ConsumerConfig | null>;
|
|
6
|
+
export declare function saveConsumerConfig(config: ConsumerConfig): Promise<void>;
|
|
7
|
+
export declare function resolveConsumerConfig(): Promise<ConsumerConfig>;
|
|
8
|
+
export declare function isConsumerMode(): boolean;
|
|
9
|
+
export declare function detectConsumerMode(): Promise<boolean>;
|
|
10
|
+
export declare function cmdInit(): Promise<void>;
|
|
11
|
+
//# sourceMappingURL=consumer-config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consumer-config.d.ts","sourceRoot":"","sources":["../src/consumer-config.ts"],"names":[],"mappings":"AAQA,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAQzE;AAED,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAG9E;AAED,wBAAsB,qBAAqB,IAAI,OAAO,CAAC,cAAc,CAAC,CAYrE;AAED,wBAAgB,cAAc,IAAI,OAAO,CAExC;AAED,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,OAAO,CAAC,CAI3D;AAED,wBAAsB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAuB7C"}
|