ai-weekly 0.1.0

package/scripts/run-discovery.mjs

@@ -0,0 +1,375 @@
+import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
+import { resolve, join } from "node:path";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { pathToFileURL } from "node:url";
+import { createInterface } from "node:readline/promises";
+
+const GITHUB_TRENDING_URL = "https://github.com/trending?since=weekly";
+const BILIBILI_SEARCH_URL = "https://search.bilibili.com/all?keyword=";
+const execFileAsync = promisify(execFile);
+
+function usage() {
+  console.log(`
+用法：
+  npx ai-weekly discover [选项]
+
+选项：
+  --date YYYY-MM-DD             输出目录日期；默认今天
+  --project-limit N             GitHub 项目数量；默认 10
+  --candidate-limit N           每项目保留候选数；默认 5
+  --raw-limit N                 每个 Bilibili 查询读取的首屏视频数；默认 20
+  --max-duration-seconds N      最大时长；默认 1200（20 分钟）
+  --creator-labels PATH         人工博主标签；营销号不会进入候选
+  --profile PATH                专用 Chrome profile；默认 data/browser-profiles/bilibili
+  --headed                      显示浏览器窗口（首次登录时使用）
+  --wait-for-enter              打开浏览器后等待完成登录或验证并按回车
+  --github-only                 仅抓取 GitHub 周榜，不启动浏览器
+  --help                        显示本帮助
+
+示例：
+  npx --package=ai-weekly playwright install chromium
+  npx ai-weekly discover
+  npx ai-weekly discover --headed
+`);
+}
+
+function parseArgs(argv) {
+  const options = {
+    date: new Date().toLocaleDateString("en-CA", { timeZone: "Asia/Shanghai" }),
+    projectLimit: 10,
+    candidateLimit: 5,
+    rawLimit: 20,
+    maxDurationSeconds: 20 * 60,
+    headed: false,
+    waitForEnter: false,
+    githubOnly: false,
+    profile: "data/browser-profiles/bilibili",
+    creatorLabels: null
+  };
+  const valueOptions = new Map([
+    ["--date", "date"],
+    ["--project-limit", "projectLimit"],
+    ["--candidate-limit", "candidateLimit"],
+    ["--raw-limit", "rawLimit"],
+    ["--max-duration-seconds", "maxDurationSeconds"],
+    ["--profile", "profile"],
+    ["--creator-labels", "creatorLabels"]
+  ]);
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === "--help") return { help: true };
+    if (arg === "--headed" || arg === "--wait-for-enter") {
+      if (arg === "--headed") options.headed = true;
+      else options.waitForEnter = true;
+      continue;
+    }
+    if (arg === "--github-only") {
+      options.githubOnly = true;
+      continue;
+    }
+    const key = valueOptions.get(arg);
+    if (!key || index + 1 >= argv.length) throw new Error(`无效参数：${arg}`);
+    const value = argv[index + 1];
+    options[key] = ["projectLimit", "candidateLimit", "rawLimit", "maxDurationSeconds"].includes(key)
+      ? Number(value)
+      : value;
+    index += 1;
+  }
+  for (const key of ["projectLimit", "candidateLimit", "rawLimit", "maxDurationSeconds"]) {
+    if (!Number.isInteger(options[key]) || options[key] < 1) throw new Error(`${key} 必须是正整数`);
+  }
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(options.date)) throw new Error("--date 必须为 YYYY-MM-DD");
+  return options;
+}
+
+async function waitForEnter(message) {
+  const input = createInterface({ input: process.stdin, output: process.stdout });
+  try { await input.question(message); } finally { input.close(); }
+}
+
+async function getText(url) {
+  const { stdout } = await execFileAsync("curl", [
+    "-LfsS",
+    "--connect-timeout", "20",
+    "--max-time", "60",
+    "-A", "AIWeeklyDiscovery/0.1 (+local personal research)",
+    "--",
+    url
+  ], {
+    maxBuffer: 10 * 1024 * 1024
+  });
+  return stdout;
+}
+
+function decodeHtml(value) {
+  return value.replaceAll("&amp;", "&").replaceAll("&#39;", "'").replaceAll("&quot;", '"').replace(/<[^>]+>/g, "").replace(/\s+/g, " ").trim();
+}
+
+function parseTrending(html) {
+  const projects = [];
+  for (const match of html.matchAll(/<article class="Box-row">([\s\S]*?)<\/article>/g)) {
+    const article = match[1];
+    const repository = article.match(/href="\/([^"/]+\/[^"/]+)"[^>]*class="Link"/)?.[1];
+    const starsText = article.match(/([\d,]+)\s+stars this week/)?.[1];
+    if (!repository || !starsText) continue;
+    const descriptionMatch = article.match(/<p class="col-9 color-fg-muted my-1 pr-4">([\s\S]*?)<\/p>/);
+    projects.push({
+      repository,
+      weekly_stars: Number(starsText.replaceAll(",", "")),
+      description: descriptionMatch ? decodeHtml(descriptionMatch[1]) : null
+    });
+  }
+  return projects.sort((left, right) => right.weekly_stars - left.weekly_stars);
+}
+
+export function selectProjectsExcludingRepositories(projects, previousRepositories, projectLimit) {
+  return projects
+    .filter((project) => !previousRepositories.has(project.repository))
+    .slice(0, projectLimit)
+    .map((project, index) => ({ rank: index + 1, ...project }));
+}
+
+async function readPreviousTrendingRepositories(runsDirectory, currentRunDate) {
+  const repositories = new Set();
+  let entries;
+  try {
+    entries = await readdir(runsDirectory, { withFileTypes: true });
+  } catch (error) {
+    if (error.code === "ENOENT") return repositories;
+    throw error;
+  }
+
+  for (const entry of entries) {
+    if (!entry.isDirectory() || entry.name === currentRunDate) continue;
+    try {
+      const snapshot = JSON.parse(await readFile(join(runsDirectory, entry.name, "github-trending.json"), "utf8"));
+      for (const project of snapshot.projects || []) {
+        if (typeof project.repository === "string") repositories.add(project.repository);
+      }
+    } catch (error) {
+      if (error.code !== "ENOENT") console.warn(`忽略无法读取的历史 Trending 快照：${entry.name}/github-trending.json`);
+    }
+  }
+  return repositories;
+}
+
+function parseDuration(value) {
+  const parts = value.split(":").map(Number);
+  if (parts.some(Number.isNaN) || parts.length < 2 || parts.length > 3) return null;
+  return parts.reduce((seconds, part) => seconds * 60 + part, 0);
+}
+
+function parseViewCount(value) {
+  const normalized = value.replaceAll(",", "").trim();
+  const wan = normalized.match(/^([\d.]+)万$/);
+  if (wan) return Math.round(Number(wan[1]) * 10_000);
+  const amount = Number(normalized);
+  return Number.isFinite(amount) ? amount : null;
+}
+
+export function parseSearchCard(text, url, creatorProfile = {}) {
+  const lines = text.split("\n").map((line) => line.trim()).filter(Boolean);
+  const durationIndex = lines.findIndex((line) => /^\d{1,2}:\d{2}(?::\d{2})?$/.test(line));
+  if (durationIndex < 0 || !lines[durationIndex + 1]) return null;
+  const creatorAndDate = lines.slice(durationIndex + 2).join(" ").match(/^(.*?)\s*·\s*(.+)$/);
+  return {
+    title: lines[durationIndex + 1],
+    url,
+    creator: creatorAndDate?.[1]?.trim() || null,
+    creator_id: creatorProfile.creator_id || null,
+    creator_url: creatorProfile.creator_url || null,
+    cover_url: creatorProfile.cover_url || null,
+    published: creatorAndDate?.[2]?.trim() || null,
+    views_text: lines[0] || null,
+    views: parseViewCount(lines[0] || ""),
+    danmaku_text: lines[1] || null,
+    duration_seconds: parseDuration(lines[durationIndex])
+  };
+}
+
+function normalize(value) {
+  return value.toLowerCase().replace(/[^\p{L}\p{N}]/gu, "");
+}
+
+function freshnessPoints(published, runDate) {
+  if (!published) return 0;
+  if (/小时前|刚刚/.test(published)) return 15;
+  if (published === "昨天") return 14;
+  const match = published.match(/^(?:(\d{4})-)?(\d{1,2})-(\d{1,2})$/);
+  if (!match) return 3;
+  const [year, month, day] = [Number(match[1] || runDate.slice(0, 4)), Number(match[2]), Number(match[3])];
+  const publishedAt = new Date(`${year}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}T00:00:00+08:00`);
+  const runAt = new Date(`${runDate}T00:00:00+08:00`);
+  const ageDays = Math.max(0, Math.floor((runAt - publishedAt) / 86_400_000));
+  if (ageDays <= 2) return 15;
+  if (ageDays <= 7) return 12;
+  if (ageDays <= 30) return 8;
+  return 3;
+}
+
+function scoreCandidate(candidate, project, runDate) {
+  const slug = project.repository.split("/")[1];
+  const normalizedTitle = normalize(candidate.title);
+  const normalizedSlug = normalize(slug);
+  const normalizedRepository = normalize(project.repository);
+  const relevance = normalizedTitle.includes(normalizedRepository) || normalizedTitle.includes(normalizedSlug)
+    ? 55
+    : 35;
+  const duration = candidate.duration_seconds <= 5 * 60 ? 15 : candidate.duration_seconds <= 10 * 60 ? 13 : 10;
+  const freshness = freshnessPoints(candidate.published, runDate);
+  const popularity = candidate.views >= 50_000 ? 15 : candidate.views >= 10_000 ? 12 : candidate.views >= 1_000 ? 8 : candidate.views >= 100 ? 5 : 2;
+  return { total: relevance + duration + freshness + popularity, relevance, duration, freshness, popularity };
+}
+
+export function selectCandidates(rawResults, project, options) {
+  const eligibleResults = rawResults
+    .filter((candidate) => candidate.duration_seconds !== null && candidate.duration_seconds <= options.maxDurationSeconds)
+    .map((candidate) => ({ ...candidate, score: scoreCandidate(candidate, project, options.date) }))
+    .sort((left, right) => right.score.total - left.score.total);
+  const marketingCreators = options.marketingCreators || new Set();
+  const filteredResults = eligibleResults.filter((candidate) => !marketingCreators.has(candidate.creator_id || candidate.creator));
+  return { eligibleResults, filteredResults };
+}
+
+const pageExtraction = async ({ url, rawLimit }) => {
+  const response = await fetch(url, { credentials: "include" });
+  const html = await response.text();
+  const extract = (document) => [...document.querySelectorAll('a[href*="/video/BV"]')]
+    .filter((anchor, index, anchors) => anchors.findIndex((item) => item.href === anchor.href) === index)
+    .slice(0, rawLimit)
+    .map((anchor) => {
+      const card = anchor.parentElement?.parentElement;
+      const creatorAnchor = card?.querySelector('a[href*="space.bilibili.com"], a[href*="/space/"]');
+      const creatorId = creatorAnchor?.href?.match(/(?:space\.bilibili\.com|\/space)\/(\d+)/)?.[1] || null;
+      const creatorUrl = creatorId ? `https://space.bilibili.com/${creatorId}` : null;
+      const cover = card?.querySelector("img");
+      const coverUrl = cover?.currentSrc || cover?.src || cover?.getAttribute("data-src") || cover?.getAttribute("data-lazy-src") || null;
+      return { url: anchor.href, text: (card?.innerText || card?.textContent || "").trim(), creator_url: creatorUrl, creator_id: creatorId, cover_url: coverUrl?.startsWith("//") ? `https:${coverUrl}` : coverUrl };
+    });
+  const document = new DOMParser().parseFromString(html, "text/html");
+  return { status: response.status, html, captcha: html.includes("验证码_哔哩哔哩"), cards: extract(document) };
+};
+
+async function searchBilibili(page, project, options, rawDirectory) {
+  const query = project.repository.split("/")[1];
+  const url = `${BILIBILI_SEARCH_URL}${encodeURIComponent(query)}`;
+  let extracted = await page.evaluate(pageExtraction, { url, rawLimit: options.rawLimit });
+  let extractionMode = "authenticated_fetch";
+  if (extracted.captcha) throw new Error(`${project.repository} 的 Bilibili 请求遇到验证码；请用 --headed 登录或稍后重试`);
+  let rawResults = extracted.cards.map(({ text, url: cardUrl, creator_url: creatorUrl, creator_id: creatorId, cover_url: coverUrl }) => parseSearchCard(text, cardUrl, { creator_url: creatorUrl, creator_id: creatorId, cover_url: coverUrl })).filter(Boolean);
+  if (rawResults.length === 0) {
+    extractionMode = "dom_fallback";
+    await page.goto(url, { waitUntil: "domcontentloaded" });
+    await page.waitForTimeout(1_500);
+    extracted.captcha = (await page.title()).includes("验证码");
+    extracted.cards = await page.evaluate(({ rawLimit }) => [...document.querySelectorAll('a[href*="/video/BV"]')]
+      .filter((anchor, index, anchors) => anchors.findIndex((item) => item.href === anchor.href) === index)
+      .slice(0, rawLimit)
+      .map((anchor) => {
+        const card = anchor.parentElement?.parentElement;
+        const creatorAnchor = card?.querySelector('a[href*="space.bilibili.com"], a[href*="/space/"]');
+        const creatorId = creatorAnchor?.href?.match(/(?:space\.bilibili\.com|\/space)\/(\d+)/)?.[1] || null;
+        const creatorUrl = creatorId ? `https://space.bilibili.com/${creatorId}` : null;
+        const cover = card?.querySelector("img");
+        const coverUrl = cover?.currentSrc || cover?.src || cover?.getAttribute("data-src") || cover?.getAttribute("data-lazy-src") || null;
+        return { url: anchor.href, text: (card?.innerText || "").trim(), creator_url: creatorUrl, creator_id: creatorId, cover_url: coverUrl?.startsWith("//") ? `https:${coverUrl}` : coverUrl };
+      }), { rawLimit: options.rawLimit });
+    rawResults = extracted.cards.map(({ text, url: cardUrl, creator_url: creatorUrl, creator_id: creatorId, cover_url: coverUrl }) => parseSearchCard(text, cardUrl, { creator_url: creatorUrl, creator_id: creatorId, cover_url: coverUrl })).filter(Boolean);
+  }
+  if (extracted.captcha) throw new Error(`${project.repository} 的 Bilibili 页面遇到验证码；请降低访问频率后重试`);
+  await writeFile(join(rawDirectory, `${query}.html`), extracted.html);
+  const eligibleResults = rawResults
+    .filter((candidate) => candidate.duration_seconds !== null && candidate.duration_seconds <= options.maxDurationSeconds)
+    .map((candidate) => ({ ...candidate, score: scoreCandidate(candidate, project, options.date) }))
+    .sort((left, right) => right.score.total - left.score.total);
+  const marketingCreators = options.marketingCreators || new Set();
+  const filteredResults = eligibleResults.filter((candidate) => !marketingCreators.has(candidate.creator_id || candidate.creator));
+  return {
+    query,
+    url,
+    extraction_mode: extractionMode,
+    raw_cards_read: rawResults.length,
+    eligible_cards: eligibleResults.length,
+    marketing_filtered_cards: eligibleResults.length - filteredResults.length,
+    raw_results: rawResults,
+    recommended_candidates: filteredResults.slice(0, options.candidateLimit)
+  };
+}
+
+async function readMarketingCreators(path) {
+  if (!path) return new Set();
+  try {
+    const labels = JSON.parse(await readFile(path, "utf8"));
+    return new Set((labels.creators || []).flatMap((creator) => creator.marketing_label === "营销号" || creator.is_marketing === true ? [creator.creator_id, creator.creator].filter(Boolean) : []));
+  } catch (error) {
+    if (error.code === "ENOENT") return new Set();
+    throw error;
+  }
+}
+
+async function main() {
+  const options = parseArgs(process.argv.slice(2));
+  if (options.help) return usage();
+  const runDirectory = resolve("data", "runs", options.date);
+  options.marketingCreators = await readMarketingCreators(options.creatorLabels || join(runDirectory, "creator-labels.json"));
+  const rawDirectory = join(runDirectory, "bilibili-raw");
+  await mkdir(rawDirectory, { recursive: true });
+
+  const trendingHtml = await getText(GITHUB_TRENDING_URL);
+  await writeFile(join(runDirectory, "github-trending.html"), trendingHtml);
+  const trendingProjects = parseTrending(trendingHtml);
+  const previousRepositories = await readPreviousTrendingRepositories(resolve("data", "runs"), options.date);
+  const projects = selectProjectsExcludingRepositories(trendingProjects, previousRepositories, options.projectLimit);
+  const skippedPreviousProjects = trendingProjects.filter((project) => previousRepositories.has(project.repository)).length;
+  if (skippedPreviousProjects > 0) console.log(`已跳过历史周榜重复项目 ${skippedPreviousProjects} 个，本次保留 ${projects.length} 个。`);
+  await writeFile(join(runDirectory, "github-trending.json"), `${JSON.stringify({ source: GITHUB_TRENDING_URL, projects }, null, 2)}\n`);
+
+  const output = {
+    schema_version: "1.1",
+    run: {
+      run_date: options.date,
+      source_window: `Bilibili 综合搜索首屏前 ${options.rawLimit} 条视频卡片`,
+      status: options.githubOnly ? "github_only" : "completed"
+    },
+    sources: { github_trending_weekly: GITHUB_TRENDING_URL, bilibili_search_base: BILIBILI_SEARCH_URL },
+    selection_policy: {
+      max_duration_seconds: options.maxDurationSeconds,
+      candidate_limit_per_project: options.candidateLimit,
+      score_weights: { relevance: 55, duration: 15, freshness: 15, popularity: 15 },
+      github_trending_deduplication: "排除其他运行日期已有 github-trending.json 中的 repository；按本次周榜排序从后续项目补足。",
+      marketing_creator_policy: "人工标记为营销号的博主不会进入推荐候选；候选不足时仅从同一抓取结果中的其他合格视频补位。"
+    },
+    projects
+  };
+  if (!options.githubOnly) {
+    const { chromium } = await import("playwright");
+    const context = await chromium.launchPersistentContext(resolve(options.profile), {
+      channel: "chrome",
+      headless: !options.headed,
+      viewport: { width: 1440, height: 960 }
+    });
+    try {
+      const page = context.pages()[0] || await context.newPage();
+      await page.goto("https://search.bilibili.com/all?keyword=AI", { waitUntil: "domcontentloaded" });
+      if (options.waitForEnter) await waitForEnter("请在浏览器中完成 Bilibili 登录或验证，然后按 Enter 继续：");
+      if ((await page.title()).includes("验证码")) throw new Error("Bilibili 要求验证；请以 --headed 模式登录或完成验证后重新运行");
+      for (const project of output.projects) {
+        project.bilibili_search = await searchBilibili(page, project, options, rawDirectory);
+        console.log(`已完成：${project.repository}`);
+        await page.waitForTimeout(1_000);
+      }
+    } finally {
+      await context.close();
+    }
+  }
+  await writeFile(join(runDirectory, "discovery.json"), `${JSON.stringify(output, null, 2)}\n`);
+  console.log(`输出已保存：${join(runDirectory, "discovery.json")}`);
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
+  main().catch((error) => {
+    console.error(`运行失败：${error.message}`);
+    process.exitCode = 1;
+  });
+}

package/scripts/run-weekly.mjs

@@ -0,0 +1,88 @@
+#!/usr/bin/env node
+
+import { spawn } from "node:child_process";
+import { realpathSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+const PACKAGE_ROOT = resolve(fileURLToPath(new URL("..", import.meta.url)));
+
+function usage() {
+  console.log(`
+用法：
+  npx ai-weekly [--date YYYY-MM-DD]
+
+依次执行发现、字幕、Luna 摘要、汇总、人工复核和离线报告。
+省略 --date 时使用上海时区的当天日期。
+仅在 Bilibili 登录受限时打开浏览器等待回车；人工复核保存后按回车继续。
+`);
+}
+
+export function parseArgs(argv) {
+  const options = { date: new Date().toLocaleDateString("en-CA", { timeZone: "Asia/Shanghai" }) };
+  for (let index = 0; index < argv.length; index += 1) {
+    if (argv[index] === "--help") return { help: true };
+    if (argv[index] !== "--date" || index + 1 >= argv.length) throw new Error(`无效参数：${argv[index]}`);
+    options.date = argv[index + 1];
+    index += 1;
+  }
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(options.date)) throw new Error("--date 必须为 YYYY-MM-DD");
+  return options;
+}
+
+function runCommand(script, args) {
+  return new Promise((resolvePromise, reject) => {
+    const child = spawn(process.execPath, [resolve(PACKAGE_ROOT, script), ...args], { stdio: ["inherit", "pipe", "pipe"] });
+    let output = "";
+    for (const stream of [child.stdout, child.stderr]) stream.on("data", (chunk) => {
+      output += chunk;
+      (stream === child.stdout ? process.stdout : process.stderr).write(chunk);
+    });
+    child.once("error", reject);
+    child.once("close", (code, signal) => code === 0 ? resolvePromise() : reject(new Error(`${script} 退出码 ${code ?? signal}${output ? `：${output.slice(-500)}` : ""}`)));
+  });
+}
+
+async function readJson(path) {
+  return JSON.parse(await readFile(path, "utf8"));
+}
+
+export async function runWeekly(options, { run = runCommand, read = readJson, log = console.log } = {}) {
+  const dateArgs = ["--date", options.date];
+  try {
+    await run("scripts/run-discovery.mjs", dateArgs);
+  } catch (error) {
+    if (!/Bilibili.*(验证码|验证)|验证码|验证/.test(error.message)) throw error;
+    log("Bilibili 发现阶段需要人工登录或验证，已打开浏览器。");
+    await run("scripts/run-discovery.mjs", [...dateArgs, "--headed", "--wait-for-enter"]);
+  }
+
+  await run("scripts/fetch-subtitles.mjs", [...dateArgs, "--headless"]);
+  const subtitlesPath = resolve(join("data", "runs", options.date, "subtitles.json"));
+  let subtitles = await read(subtitlesPath);
+  if ((subtitles.summary?.login_required || 0) > 0) {
+    log("字幕读取需要 Bilibili 登录或验证，已打开浏览器。");
+    await run("scripts/fetch-subtitles.mjs", [...dateArgs, "--headed"]);
+    subtitles = await read(subtitlesPath);
+    if ((subtitles.summary?.login_required || 0) > 0) throw new Error("Bilibili 登录或验证尚未完成，已停止后续流程。");
+  }
+
+  await run("scripts/summarize-videos.mjs", [...dateArgs, "--concurrency", "2"]);
+  await run("scripts/synthesize-projects.mjs", dateArgs);
+  await run("scripts/review-creators.mjs", [...dateArgs, "--wait-for-enter"]);
+  await run("scripts/synthesize-projects.mjs", dateArgs);
+  await run("scripts/download-report-covers.mjs", dateArgs);
+  await run("scripts/build-report.mjs", dateArgs);
+  log(`完整周报已生成：${resolve(join("data", "runs", options.date, "report", "index.html"))}`);
+}
+
+async function main() {
+  const options = parseArgs(process.argv.slice(2));
+  if (options.help) return usage();
+  await runWeekly(options);
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href) {
+  main().catch((error) => { console.error(`完整流程失败：${error.message}`); process.exitCode = 1; });
+}