botrun-crawler-2 1.0.0

package/README.md

@@ -0,0 +1,126 @@
+# botrun-crawler
+
+智慧網頁爬蟲 - 支援動態下載連結偵測、MIME type 分析、政府網站相容
+
+## 安裝
+
+```bash
+npm install -g botrun-crawler
+```
+
+## CLI 使用方法
+
+### 基本爬取
+
+```bash
+# 單頁完整爬取
+botrun-crawler crawl "https://example.com"
+
+# JSON 格式輸出
+botrun-crawler crawl "https://example.com" --json
+
+# 安靜模式
+botrun-crawler crawl "https://example.com" -q
+```
+
+### 儲存結果
+
+```bash
+# 儲存 HTML
+botrun-crawler crawl "https://example.com" --save-html page.html
+
+# 儲存純文字（LLM 友善格式）
+botrun-crawler crawl "https://example.com" --save-text page.txt
+
+# 儲存下載連結清單
+botrun-crawler crawl "https://example.com" --save-downloads urls.txt
+
+# 全部儲存
+botrun-crawler crawl "https://example.com" \
+  --save-html page.html \
+  --save-text page.txt \
+  --save-downloads urls.txt
+```
+
+### 效能選項
+
+```bash
+# 跳過 MIME type 分析（加快速度）
+botrun-crawler crawl "https://example.com" --skip-mime
+```
+
+## 命令總覽
+
+| 命令 | 說明 |
+|------|------|
+| `crawl <url>` | 單頁完整爬取（推薦） |
+| `scrape <url>` | 簡易版抓取 |
+| `scrape-batch <file>` | 批次抓取多個網址 |
+| `process <file>` | 處理本地檔案 |
+| `help` | 顯示說明 |
+
+## crawl 選項
+
+| 選項 | 說明 |
+|------|------|
+| `--json` | JSON 格式輸出 |
+| `--save-html <file>` | 儲存原始 HTML |
+| `--save-text <file>` | 儲存純文字 |
+| `--save-downloads <file>` | 儲存下載連結清單 |
+| `--skip-mime` | 跳過 MIME type 分析 |
+| `-v, --verbose` | 詳細模式 |
+| `-q, --quiet` | 安靜模式 |
+
+## 輸出範例
+
+```
+========================================
+ 單頁完整爬取結果
+========================================
+網址: https://www.ida.gov.tw/...
+標題: 經濟部產業發展署
+
+連結統計
+├─ 下載連結: 475 個
+│   ├─ PDF: 120
+│   ├─ Word: 85
+│   └─ 動態連結: 270
+├─ 連外連結: 22 個
+└─ 內部連結: 675 個
+```
+
+## Proxy 設定
+
+```bash
+export HTTPS_PROXY="http://proxy:8080"
+export HTTP_PROXY="http://proxy:8080"
+export NO_PROXY="localhost,.internal.com"
+```
+
+## 程式庫使用
+
+```typescript
+import { crawlPage } from 'botrun-crawler/crawler';
+
+const result = await crawlPage('https://example.com');
+console.log(result.links.downloads.count);
+```
+
+### 子模組
+
+```typescript
+import { PageCrawler, crawlPage } from 'botrun-crawler/crawler';
+import { scrapeUrl } from 'botrun-crawler/scraper';
+import { fetchWithProxy, fetchMimeType } from 'botrun-crawler/lib';
+```
+
+## 特色
+
+- **動態下載連結偵測**：自動識別 `download.aspx`、`getFile.do` 等動態連結
+- **MIME Type 分析**：自動分類 PDF/Word/Excel 等檔案類型
+- **檔名亂碼修正**：自動修正 UTF-8 編碼問題
+- **政府網站相容**：針對台灣政府網站最佳化
+
+## License
+
+MIT

package/dist/cli.d.ts

@@ -0,0 +1,10 @@
+#!/usr/bin/env npx tsx
+/**
+ * 耳（Ear）- 任意格式處理器 CLI
+ * 零幻覺四部曲之一：將任意格式轉換為 AI 可讀文字
+ *
+ * @version 1.0.0
+ * @author 永恆知己系統
+ */
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;;;GAMG"}

package/dist/cli.js

@@ -0,0 +1,155 @@
+#!/usr/bin/env npx tsx
+/**
+ * 耳（Ear）- 任意格式處理器 CLI
+ * 零幻覺四部曲之一：將任意格式轉換為 AI 可讀文字
+ *
+ * @version 1.0.0
+ * @author 永恆知己系統
+ */
+import { Command } from "commander";
+import * as path from "path";
+import * as fs from "fs";
+import { ProcessorRouter } from "./processors/index.js";
+import { CacheService } from "./lib/cache.js";
+import { LoggerService } from "./lib/logger.js";
+import { AgentService } from "./lib/agent.js";
+import { runScrapeCommand, runScrapeBatchCommand } from "./scraper/cli.js";
+import { runCrawlCommand } from "./crawler/cli.js";
+// ============================================================
+// CLI 程式
+// ============================================================
+const program = new Command();
+program
+    .name("ear")
+    .description("耳（Ear）- 任意格式處理器\n零幻覺四部曲之一：將任意格式轉換為 AI 可讀文字")
+    .version("1.0.0");
+// ============================================================
+// process 命令
+// ============================================================
+program
+    .command("process <target>")
+    .description("處理單一檔案或目錄中的所有檔案")
+    .option("-o, --output <dir>", "指定輸出目錄")
+    .option("-f, --format <format>", "強制指定輸入格式")
+    .option("-m, --model <model>", "指定 AI 模型", "claude-haiku-4-5")
+    .option("-j, --jobs <n>", "並行處理數量", "4")
+    .option("--no-cache", "忽略快取，強制重新處理")
+    .option("--dry-run", "模擬執行，不實際處理")
+    .option("-v, --verbose", "顯示詳細日誌")
+    .action(async (target, options) => {
+    const logger = new LoggerService(options.verbose ?? false);
+    const cache = new CacheService(logger);
+    const agent = new AgentService(options.model ?? "claude-haiku-4-5", logger);
+    const router = new ProcessorRouter(cache, logger, agent);
+    try {
+        logger.info(`開始處理：${target}`);
+        const targetPath = path.resolve(target);
+        if (!fs.existsSync(targetPath)) {
+            logger.error(`檔案或目錄不存在：${targetPath}`);
+            process.exit(1);
+        }
+        const stats = fs.statSync(targetPath);
+        if (stats.isDirectory()) {
+            // 目錄處理
+            await router.processDirectory(targetPath, {
+                jobs: parseInt(String(options.jobs ?? 4), 10),
+                noCache: options.noCache ?? false,
+                dryRun: options.dryRun ?? false,
+                outputDir: options.output,
+            });
+        }
+        else {
+            // 單檔處理
+            const result = await router.processFile(targetPath, {
+                noCache: options.noCache ?? false,
+                dryRun: options.dryRun ?? false,
+                forceFormat: options.format,
+                outputDir: options.output,
+            });
+            if (result.success) {
+                logger.success(`處理完成：${result.outputPath}`);
+            }
+            else {
+                logger.error(`處理失敗：${result.error}`);
+                process.exit(1);
+            }
+        }
+    }
+    catch (error) {
+        logger.error(`處理錯誤：${error instanceof Error ? error.message : String(error)}`);
+        process.exit(1);
+    }
+});
+// ============================================================
+// ask 命令（自然語言驅動）
+// ============================================================
+program
+    .command("ask <query>")
+    .description("使用自然語言描述任務")
+    .option("-m, --model <model>", "指定 AI 模型", "claude-haiku-4-5")
+    .option("-v, --verbose", "顯示詳細日誌")
+    .action(async (query, options) => {
+    const logger = new LoggerService(options.verbose ?? false);
+    const agent = new AgentService(options.model ?? "claude-haiku-4-5", logger);
+    try {
+        logger.info(`自然語言查詢：${query}`);
+        const result = await agent.ask(query);
+        console.log("\n" + result);
+    }
+    catch (error) {
+        logger.error(`查詢錯誤：${error instanceof Error ? error.message : String(error)}`);
+        process.exit(1);
+    }
+});
+// ============================================================
+// crawl 命令（單頁完整爬取）
+// ============================================================
+program
+    .command("crawl <url>")
+    .description("單頁完整爬取（含結構分析、下載清單、變更追蹤）")
+    .option("--json", "以 JSON 格式輸出")
+    .option("--save-html <file>", "儲存原始 HTML")
+    .option("--save-text <file>", "儲存純文字（LLM 友善格式）")
+    .option("--save-downloads <file>", "儲存下載連結清單")
+    .option("--skip-mime", "跳過 MIME type 分析（預設會分析）")
+    .option("-v, --verbose", "顯示詳細資訊")
+    .option("-q, --quiet", "安靜模式")
+    .action(async (url, options) => {
+    await runCrawlCommand(url, options);
+});
+// ============================================================
+// scrape 命令（網頁抓取與分析）- 簡易版，向後相容
+// ============================================================
+program
+    .command("scrape <url>")
+    .description("抓取網頁並分析")
+    .option("--json", "以 JSON 格式輸出")
+    .option("--save-html <file>", "儲存原始 HTML")
+    .option("--save-text <file>", "儲存純文字（LLM 友善格式）")
+    .option("-v, --verbose", "顯示詳細資訊")
+    .option("-q, --quiet", "安靜模式")
+    .action(async (url, options) => {
+    await runScrapeCommand(url, options);
+});
+// ============================================================
+// scrape-batch 命令（批次抓取）
+// ============================================================
+program
+    .command("scrape-batch <urlsFile> [outputDir]")
+    .description("批次抓取多個網址")
+    .option("-q, --quiet", "安靜模式")
+    .action(async (urlsFile, outputDir = ".", options) => {
+    const filePath = path.resolve(urlsFile);
+    if (!fs.existsSync(filePath)) {
+        console.error(`找不到檔案: ${urlsFile}`);
+        process.exit(1);
+    }
+    const content = fs.readFileSync(filePath, "utf-8");
+    const urls = content.split("\n");
+    await runScrapeBatchCommand(urls, outputDir, options);
+});
+// ============================================================
+// 執行
+// ============================================================
+program.parse();
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;;;GAMG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAgBnD,+DAA+D;AAC/D,SAAS;AACT,+DAA+D;AAE/D,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,KAAK,CAAC;KACX,WAAW,CAAC,4CAA4C,CAAC;KACzD,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,+DAA+D;AAC/D,aAAa;AACb,+DAA+D;AAE/D,OAAO;KACJ,OAAO,CAAC,kBAAkB,CAAC;KAC3B,WAAW,CAAC,iBAAiB,CAAC;KAC9B,MAAM,CAAC,oBAAoB,EAAE,QAAQ,CAAC;KACtC,MAAM,CAAC,uBAAuB,EAAE,UAAU,CAAC;KAC3C,MAAM,CAAC,qBAAqB,EAAE,UAAU,EAAE,kBAAkB,CAAC;KAC7D,MAAM,CAAC,gBAAgB,EAAE,QAAQ,EAAE,GAAG,CAAC;KACvC,MAAM,CAAC,YAAY,EAAE,aAAa,CAAC;KACnC,MAAM,CAAC,WAAW,EAAE,YAAY,CAAC;KACjC,MAAM,CAAC,eAAe,EAAE,QAAQ,CAAC;KACjC,MAAM,CAAC,KAAK,EAAE,MAAc,EAAE,OAAuB,EAAE,EAAE;IACxD,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC,KAAK,IAAI,kBAAkB,EAAE,MAAM,CAAC,CAAC;IAC5E,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IAEzD,IAAI,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,QAAQ,MAAM,EAAE,CAAC,CAAC;QAE9B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAExC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,KAAK,CAAC,YAAY,UAAU,EAAE,CAAC,CAAC;YACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAEtC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,OAAO;YACP,MAAM,MAAM,CAAC,gBAAgB,CAAC,UAAU,EAAE;gBACxC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;gBAC7C,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,KAAK;gBACjC,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,KAAK;gBAC/B,SAAS,EAAE,OAAO,CAAC,MAAM;aAC1B,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,OAAO;YACP,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,UAAU,EAAE;gBAClD,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,KAAK;gBACjC,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,KAAK;gBAC/B,WAAW,EAAE,OAAO,CAAC,MAAM;gBAC3B,SAAS,EAAE,OAAO,CAAC,MAAM;aAC1B,CAAC,CAAC;YAEH,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,MAAM,CAAC,OAAO,CAAC,QAAQ,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;YAC9C,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,KAAK,CAAC,QAAQ,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;gBACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,QAAQ,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC/E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,+DAA+D;AAC/D,iBAAiB;AACjB,+DAA+D;AAE/D,OAAO;KACJ,OAAO,CAAC,aAAa,CAAC;KACtB,WAAW,CAAC,YAAY,CAAC;KACzB,MAAM,CAAC,qBAAqB,EAAE,UAAU,EAAE,kBAAkB,CAAC;KAC7D,MAAM,CAAC,eAAe,EAAE,QAAQ,CAAC;KACjC,MAAM,CAAC,KAAK,EAAE,KAAa,EAAE,OAA8C,EAAE,EAAE;IAC9E,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC,KAAK,IAAI,kBAAkB,EAAE,MAAM,CAAC,CAAC;IAE5E,IAAI,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,UAAU,KAAK,EAAE,CAAC,CAAC;QAE/B,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEtC,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,QAAQ,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC/E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,+DAA+D;AAC/D,mBAAmB;AACnB,+DAA+D;AAE/D,OAAO;KACJ,OAAO,CAAC,aAAa,CAAC;KACtB,WAAW,CAAC,yBAAyB,CAAC;KACtC,MAAM,CAAC,QAAQ,EAAE,aAAa,CAAC;KAC/B,MAAM,CAAC,oBAAoB,EAAE,WAAW,CAAC;KACzC,MAAM,CAAC,oBAAoB,EAAE,iBAAiB,CAAC;KAC/C,MAAM,CAAC,yBAAyB,EAAE,UAAU,CAAC;KAC7C,MAAM,CAAC,aAAa,EAAE,wBAAwB,CAAC;KAC/C,MAAM,CAAC,eAAe,EAAE,QAAQ,CAAC;KACjC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC;KAC7B,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,OAQ3B,EAAE,EAAE;IACH,MAAM,eAAe,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AACtC,CAAC,CAAC,CAAC;AAEL,+DAA+D;AAC/D,+BAA+B;AAC/B,+DAA+D;AAE/D,OAAO;KACJ,OAAO,CAAC,cAAc,CAAC;KACvB,WAAW,CAAC,SAAS,CAAC;KACtB,MAAM,CAAC,QAAQ,EAAE,aAAa,CAAC;KAC/B,MAAM,CAAC,oBAAoB,EAAE,WAAW,CAAC;KACzC,MAAM,CAAC,oBAAoB,EAAE,iBAAiB,CAAC;KAC/C,MAAM,CAAC,eAAe,EAAE,QAAQ,CAAC;KACjC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC;KAC7B,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,OAM3B,EAAE,EAAE;IACH,MAAM,gBAAgB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AACvC,CAAC,CAAC,CAAC;AAEL,+DAA+D;AAC/D,wBAAwB;AACxB,+DAA+D;AAE/D,OAAO;KACJ,OAAO,CAAC,qCAAqC,CAAC;KAC9C,WAAW,CAAC,UAAU,CAAC;KACvB,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC;KAC7B,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,YAAoB,GAAG,EAAE,OAA4B,EAAE,EAAE;IACxF,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,UAAU,QAAQ,EAAE,CAAC,CAAC;QACpC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEjC,MAAM,qBAAqB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;AACxD,CAAC,CAAC,CAAC;AAEL,+DAA+D;AAC/D,KAAK;AACL,+DAA+D;AAE/D,OAAO,CAAC,KAAK,EAAE,CAAC"}

package/dist/crawler/cli.d.ts

@@ -0,0 +1,19 @@
+import { CrawlResult } from "./index.js";
+export interface CrawlCliOptions {
+    json?: boolean;
+    saveHtml?: string;
+    saveText?: string;
+    saveDownloads?: string;
+    verbose?: boolean;
+    quiet?: boolean;
+    skipMime?: boolean;
+}
+/**
+ * 輸出人類可讀報告
+ */
+export declare function printCrawlReport(result: CrawlResult, verbose?: boolean): void;
+/**
+ * 執行 crawl 命令
+ */
+export declare function runCrawlCommand(url: string, options: CrawlCliOptions): Promise<void>;
+//# sourceMappingURL=cli.d.ts.map

package/dist/crawler/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../src/crawler/cli.ts"],"names":[],"mappings":"AAIA,OAAO,EAAe,WAAW,EAAc,MAAM,YAAY,CAAC;AAElE,MAAM,WAAW,eAAe;IAC9B,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,WAAW,EACnB,OAAO,GAAE,OAAe,GACvB,IAAI,CA8IN;AAED;;GAEG;AACH,wBAAsB,eAAe,CACnC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,IAAI,CAAC,CAyCf"}

package/dist/crawler/cli.js

@@ -0,0 +1,179 @@
+/**
+ * 單頁爬取 CLI 模組
+ */
+import * as fs from "fs";
+import { PageCrawler } from "./index.js";
+/**
+ * 輸出人類可讀報告
+ */
+export function printCrawlReport(result, verbose = false) {
+    console.log("=".repeat(60));
+    console.log("網頁分析報告");
+    console.log("=".repeat(60));
+    console.log(`網址: ${result.url}`);
+    if (result.title) {
+        console.log(`標題: ${result.title}`);
+    }
+    console.log(`爬取時間: ${result.crawled_at}`);
+    console.log(`頁面指紋: ${result.fingerprint.slice(0, 16)}...`);
+    // 頁面 MIME type（如果有）
+    if (result.pageMimeType) {
+        const mime = result.pageMimeType;
+        let mimeStr = mime.mimeType;
+        if (mime.charset)
+            mimeStr += `; charset=${mime.charset}`;
+        console.log(`頁面類型: ${mimeStr}`);
+    }
+    // Benchmark 計時
+    if (result.benchmark) {
+        const b = result.benchmark;
+        let benchmarkStr = `抓取: ${(b.fetch_ms / 1000).toFixed(2)}s, 解析: ${(b.parse_ms / 1000).toFixed(2)}s`;
+        if (b.mime_analysis_ms !== undefined) {
+            benchmarkStr += `, MIME分析: ${(b.mime_analysis_ms / 1000).toFixed(2)}s`;
+        }
+        benchmarkStr += `, 總計: ${(b.total_ms / 1000).toFixed(2)}s`;
+        console.log(`執行時間: ${benchmarkStr}`);
+    }
+    console.log();
+    console.log("【大小統計】");
+    console.log(`  原始 HTML: ${result.size.raw_html_readable}`);
+    console.log(`  純文字:    ${result.size.text_readable}`);
+    console.log(`  壓縮比:    ${result.size.compression_ratio} 節省`);
+    console.log();
+    // 下載檔案統計
+    const downloads = result.links.downloads;
+    const uniqueCount = downloads.unique_count || downloads.count;
+    const originalCount = downloads.original_count || downloads.count;
+    const hasDuplicates = originalCount > uniqueCount;
+    if (hasDuplicates) {
+        console.log(`【可下載檔案】共 ${uniqueCount} 個（原始 ${originalCount} 個，去重 ${originalCount - uniqueCount} 個）`);
+    }
+    else {
+        console.log(`【可下載檔案】共 ${result.links.downloads.count} 個`);
+    }
+    // 按檔案類別統計（人類可讀）
+    if (downloads.by_file_category && Object.keys(downloads.by_file_category).length > 0) {
+        console.log("  按檔案類型統計:");
+        const sorted = Object.entries(downloads.by_file_category)
+            .sort((a, b) => b[1] - a[1]);
+        for (const [category, count] of sorted) {
+            console.log(`    - ${category}: ${count} 個`);
+        }
+    }
+    else if (Object.keys(result.links.downloads.by_type).length > 0) {
+        // 如果沒有 MIME 分析，顯示按副檔名類型統計
+        console.log("  按副檔名統計:");
+        for (const [type, count] of Object.entries(result.links.downloads.by_type)
+            .sort((a, b) => b[1] - a[1])) {
+            console.log(`    - ${type.toUpperCase()}: ${count} 個`);
+        }
+    }
+    // 顯示真實 MIME type 統計（如果有且為 verbose 模式）
+    if (verbose && downloads.by_real_mime && Object.keys(downloads.by_real_mime).length > 0) {
+        console.log("  按真實 MIME type 統計:");
+        const sorted = Object.entries(downloads.by_real_mime)
+            .sort((a, b) => b[1] - a[1]);
+        for (const [mimeType, count] of sorted) {
+            console.log(`    - ${mimeType}: ${count} 個`);
+        }
+    }
+    if (verbose && result.links.downloads.items.length > 0) {
+        console.log("  下載清單:");
+        const items = result.links.downloads.items.slice(0, 20);
+        for (let i = 0; i < items.length; i++) {
+            const item = items[i];
+            console.log(`    ${i + 1}. [${item.type.toUpperCase()}] ${item.text.slice(0, 50)}`);
+            if (item.category) {
+                console.log(`       分類: ${item.category}`);
+            }
+        }
+        if (result.links.downloads.items.length > 20) {
+            console.log(`    ... 還有 ${result.links.downloads.items.length - 20} 個`);
+        }
+    }
+    console.log();
+    console.log(`【連外連結】共 ${result.links.external.count} 個`);
+    if (Object.keys(result.links.external.domains).length > 0) {
+        const sorted = Object.entries(result.links.external.domains)
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, 10);
+        for (const [domain, count] of sorted) {
+            console.log(`  - ${domain}: ${count} 個`);
+        }
+    }
+    // MIME type 統計（如果有）
+    if (result.links.external.by_mime_type && Object.keys(result.links.external.by_mime_type).length > 0) {
+        console.log("  按 MIME type 統計:");
+        const mimeTypeSorted = Object.entries(result.links.external.by_mime_type)
+            .sort((a, b) => b[1] - a[1]);
+        for (const [mimeType, count] of mimeTypeSorted) {
+            console.log(`    - ${mimeType}: ${count} 個`);
+        }
+    }
+    console.log();
+    console.log(`【內部連結】共 ${result.links.internal.count} 個`);
+    console.log();
+    if (result.content.headings.length > 0) {
+        console.log(`【頁面結構】共 ${result.content.headings.length} 個標題`);
+        if (verbose) {
+            for (const heading of result.content.headings.slice(0, 10)) {
+                console.log(`  - ${heading.slice(0, 60)}`);
+            }
+            if (result.content.headings.length > 10) {
+                console.log(`  ... 還有 ${result.content.headings.length - 10} 個`);
+            }
+        }
+        console.log();
+    }
+    if (result.content.forms && result.content.forms.length > 0) {
+        console.log(`【表單】共 ${result.content.forms.length} 個`);
+        if (verbose) {
+            for (const form of result.content.forms) {
+                console.log(`  - ${form.method} ${form.action || "(無 action)"}`);
+                console.log(`    欄位: ${form.fields.join(", ")}`);
+            }
+        }
+        console.log();
+    }
+}
+/**
+ * 執行 crawl 命令
+ */
+export async function runCrawlCommand(url, options) {
+    const crawler = new PageCrawler(url);
+    const success = await crawler.fetch();
+    if (!success) {
+        process.exit(1);
+    }
+    // 預設分析 MIME type（除非指定 --skip-mime）
+    const result = options.skipMime
+        ? crawler.analyze()
+        : await crawler.analyzeWithMimeTypes();
+    // 儲存檔案
+    if (options.saveHtml) {
+        fs.writeFileSync(options.saveHtml, crawler.getRawHtml(), "utf-8");
+        if (!options.quiet) {
+            console.error(`原始 HTML 已儲存: ${options.saveHtml}`);
+        }
+    }
+    if (options.saveText) {
+        fs.writeFileSync(options.saveText, crawler.getTextContent(), "utf-8");
+        if (!options.quiet) {
+            console.error(`純文字已儲存: ${options.saveText}`);
+        }
+    }
+    if (options.saveDownloads) {
+        fs.writeFileSync(options.saveDownloads, crawler.getDownloadList(), "utf-8");
+        if (!options.quiet) {
+            console.error(`下載清單已儲存: ${options.saveDownloads}`);
+        }
+    }
+    // 輸出
+    if (options.json) {
+        console.log(JSON.stringify(result, null, 2));
+    }
+    else {
+        printCrawlReport(result, options.verbose);
+    }
+}
+//# sourceMappingURL=cli.js.map

package/dist/crawler/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/crawler/cli.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,WAAW,EAA2B,MAAM,YAAY,CAAC;AAYlE;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAC9B,MAAmB,EACnB,UAAmB,KAAK;IAExB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,OAAO,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IACjC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,OAAO,CAAC,GAAG,CAAC,OAAO,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;IAC1C,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAE3D,oBAAoB;IACpB,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC;QACjC,IAAI,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC5B,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,aAAa,IAAI,CAAC,OAAO,EAAE,CAAC;QACzD,OAAO,CAAC,GAAG,CAAC,SAAS,OAAO,EAAE,CAAC,CAAC;IAClC,CAAC;IAED,eAAe;IACf,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC;QAC3B,IAAI,YAAY,GAAG,OAAO,CAAC,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;QACpG,IAAI,CAAC,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;YACrC,YAAY,IAAI,aAAa,CAAC,CAAC,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;QACzE,CAAC;QACD,YAAY,IAAI,SAAS,CAAC,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;QAC3D,OAAO,CAAC,GAAG,CAAC,SAAS,YAAY,EAAE,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtB,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;IAC3D,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,IAAI,CAAC,iBAAiB,KAAK,CAAC,CAAC;IAC7D,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,SAAS;IACT,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,SAAgB,CAAC;IAChD,MAAM,WAAW,GAAG,SAAS,CAAC,YAAY,IAAI,SAAS,CAAC,KAAK,CAAC;IAC9D,MAAM,aAAa,GAAG,SAAS,CAAC,cAAc,IAAI,SAAS,CAAC,KAAK,CAAC;IAClE,MAAM,aAAa,GAAG,aAAa,GAAG,WAAW,CAAC;IAElD,IAAI,aAAa,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,YAAY,WAAW,SAAS,aAAa,SAAS,aAAa,GAAG,WAAW,KAAK,CAAC,CAAC;IACtG,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,IAAI,CAAC,CAAC;IAC5D,CAAC;IAED,gBAAgB;IAChB,IAAI,SAAS,CAAC,gBAAgB,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrF,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,gBAA0C,CAAC;aAChF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;YACvC,OAAO,CAAC,GAAG,CAAC,SAAS,QAAQ,KAAK,KAAK,IAAI,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;SAAM,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClE,0BAA0B;QAC1B,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC;aACvE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,sCAAsC;IACtC,IAAI,OAAO,IAAI,SAAS,CAAC,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxF,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,YAAsC,CAAC;aAC5E,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;YACvC,OAAO,CAAC,GAAG,CAAC,SAAS,QAAQ,KAAK,KAAK,IAAI,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IACD,IAAI,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACvB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACxD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,OAAO,CAAC,GAAG,CACT,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CACvE,CAAC;YACF,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC7C,OAAO,CAAC,GAAG,CACT,cAAc,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,GAAG,EAAE,IAAI,CAC3D,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;IACxD,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1D,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC;aACzD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3B,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,OAAO,MAAM,KAAK,KAAK,IAAI,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,IAAI,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;aACtE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,cAAc,EAAE,CAAC;YAC/C,OAAO,CAAC,GAAG,CAAC,SAAS,QAAQ,KAAK,KAAK,IAAI,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,MAAM,CAAC,CAAC;QAC7D,IAAI,OAAO,EAAE,CAAC;YACZ,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;gBAC3D,OAAO,CAAC,GAAG,CAAC,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;YAC7C,CAAC;YACD,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;gBACxC,OAAO,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,IAAI,CAAC,CAAC;YACnE,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5D,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;QACtD,IAAI,OAAO,EAAE,CAAC;YACZ,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACxC,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,IAAI,YAAY,EAAE,CAAC,CAAC;gBACjE,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,GAAW,EACX,OAAwB;IAExB,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC;IAErC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACtC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,mCAAmC;IACnC,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ;QAC7B,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE;QACnB,CAAC,CAAC,MAAM,OAAO,CAAC,oBAAoB,EAAE,CAAC;IAEzC,OAAO;IACP,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,UAAU,EAAE,EAAE,OAAO,CAAC,CAAC;QAClE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACnB,OAAO,CAAC,KAAK,CAAC,gBAAgB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,cAAc,EAAE,EAAE,OAAO,CAAC,CAAC;QACtE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACnB,OAAO,CAAC,KAAK,CAAC,WAAW,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;QAC1B,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,eAAe,EAAE,EAAE,OAAO,CAAC,CAAC;QAC5E,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACnB,OAAO,CAAC,KAAK,CAAC,YAAY,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,KAAK;IACL,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC;SAAM,CAAC;QACN,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;AACH,CAAC"}

package/dist/crawler/index.d.ts

@@ -0,0 +1,146 @@
+/**
+ * 單頁完整爬取模組
+ *
+ * 滿足五個動機：
+ * 1. AI 知識庫建置 - 純文字 + metadata
+ * 2. 完整離線歸檔 - 下載清單 + 檔案分類
+ * 3. 法規變更追蹤 - 頁面指紋 + 時間戳記
+ * 4. 申請流程自動化 - 結構解析
+ * 5. 跨文件關聯分析 - 連結關係
+ *
+ * 支援 HTTPS Proxy（沙盒環境）
+ */
+import { MimeTypeInfo } from "../lib/http-client.js";
+export interface CrawlOptions {
+    timeout?: number;
+    userAgent?: string;
+}
+export interface DownloadLinkItem {
+    url: string;
+    text: string;
+    type: string;
+    category?: string;
+    mimeType?: MimeTypeInfo;
+    fileCategory?: string;
+}
+export interface ExternalLinkItem {
+    url: string;
+    text: string;
+    domain: string;
+    mimeType?: MimeTypeInfo;
+}
+export type { MimeTypeInfo };
+export interface LinkItem {
+    url: string;
+    text: string;
+}
+export interface FormInfo {
+    action: string;
+    method: string;
+    fields: string[];
+}
+export interface BenchmarkInfo {
+    fetch_ms: number;
+    parse_ms: number;
+    mime_analysis_ms?: number;
+    total_ms: number;
+}
+export interface CrawlResult {
+    url: string;
+    title: string;
+    description: string;
+    crawled_at: string;
+    fingerprint: string;
+    pageMimeType?: MimeTypeInfo;
+    benchmark?: BenchmarkInfo;
+    size: {
+        raw_html_bytes: number;
+        raw_html_readable: string;
+        text_bytes: number;
+        text_readable: string;
+        compression_ratio: string;
+    };
+    links: {
+        downloads: {
+            count: number;
+            unique_count?: number;
+            original_count?: number;
+            by_type: Record<string, number>;
+            by_real_mime?: Record<string, number>;
+            by_file_category?: Record<string, number>;
+            items: DownloadLinkItem[];
+        };
+        external: {
+            count: number;
+            domains: Record<string, number>;
+            by_mime_type?: Record<string, number>;
+            items: ExternalLinkItem[];
+        };
+        internal: {
+            count: number;
+            items: LinkItem[];
+        };
+    };
+    content: {
+        headings: string[];
+        sections: string[];
+        forms?: FormInfo[];
+    };
+}
+export declare function formatSize(bytes: number): string;
+/**
+ * 格式化為台灣時間 (UTC+8)
+ */
+export declare function formatTaiwanTime(date: Date): string;
+export declare function extractText(html: string): string;
+/**
+ * 下載連結分析器
+ * 用於判斷 URL 是否為下載連結，並分類
+ */
+export declare class DownloadAnalyzer {
+    analyze(url: string, linkText: string): {
+        isDownload: boolean;
+        type: string | null;
+        category: string | null;
+    };
+}
+export declare class PageCrawler {
+    private url;
+    private baseDomain;
+    private html;
+    private crawledAt;
+    private pageMimeType?;
+    private responseHeaders?;
+    private fetchStartTime;
+    private fetchEndTime;
+    constructor(url: string);
+    getFetchDuration(): number;
+    fetch(options?: CrawlOptions): Promise<boolean>;
+    private detectPageMimeType;
+    getPageMimeType(): MimeTypeInfo | undefined;
+    getRawHtml(): string;
+    getTextContent(): string;
+    getFingerprint(): string;
+    private downloadStats;
+    getDownloadableLinks(): DownloadLinkItem[];
+    getDownloadStats(): {
+        originalCount: number;
+        uniqueCount: number;
+    };
+    getExternalLinks(): ExternalLinkItem[];
+    getInternalLinks(): LinkItem[];
+    analyze(): CrawlResult;
+    /**
+     * 分析並獲取外部連結和下載連結的 MIME type（需要額外網路請求）
+     */
+    analyzeWithMimeTypes(): Promise<CrawlResult>;
+    /**
+     * 產生下載清單（每行一個 URL）
+     */
+    getDownloadList(): string;
+}
+/**
+ * 快速爬取單頁
+ */
+export declare function crawlPage(url: string, options?: CrawlOptions): Promise<CrawlResult | null>;
+//# sourceMappingURL=index.d.ts.map