autocrew 0.1.0

package/src/modules/intel/collectors/rss.ts

@@ -0,0 +1,70 @@
+import RssParser from "rss-parser";
+import type { IntelItem } from "../../../storage/pipeline-store.js";
+import type { Collector, CollectorOptions, CollectorResult } from "../collector.js";
+import { loadSourceConfig } from "../source-config.js";
+
+// ─── RSS Item Parser ────────────────────────────────────────────────────────
+
+export interface RssItem {
+  title?: string;
+  link?: string;
+  contentSnippet?: string;
+  content?: string;
+  isoDate?: string;
+  categories?: string[];
+  creator?: string;
+}
+
+export function parseRssItems(
+  items: RssItem[],
+  domain: string,
+  tags?: string[],
+): IntelItem[] {
+  return items
+    .filter((item) => item.title)
+    .map((item) => ({
+      title: item.title!,
+      domain,
+      source: "rss" as const,
+      sourceUrl: item.link,
+      collectedAt: item.isoDate ?? new Date().toISOString(),
+      relevance: 40,
+      tags: [...(tags ?? []), ...(item.categories ?? [])],
+      expiresAfter: 14,
+      summary: item.contentSnippet ?? item.content?.slice(0, 200) ?? "",
+      keyPoints: [],
+      topicPotential: "",
+    }));
+}
+
+// ─── RSS Collector ──────────────────────────────────────────────────────────
+
+export function createRssCollector(): Collector {
+  const parser = new RssParser();
+
+  return {
+    id: "rss",
+    async collect(opts: CollectorOptions): Promise<CollectorResult> {
+      const config = await loadSourceConfig(opts.dataDir);
+      const items: IntelItem[] = [];
+      const errors: string[] = [];
+
+      for (const feed of config.rss) {
+        try {
+          const parsed = await parser.parseURL(feed.url);
+          const feedItems = parseRssItems(
+            parsed.items as RssItem[],
+            feed.domain,
+            feed.tags,
+          );
+          items.push(...feedItems);
+        } catch (err: unknown) {
+          const msg = err instanceof Error ? err.message : String(err);
+          errors.push(`RSS feed ${feed.url} failed: ${msg}`);
+        }
+      }
+
+      return { items, source: "rss", errors };
+    },
+  };
+}

package/src/modules/intel/collectors/trends.test.ts

@@ -0,0 +1,80 @@
+import { describe, it, expect, vi } from "vitest";
+import { buildTrendQueries, createTrendCollector } from "./trends.js";
+import type { TrendSource } from "../source-config.js";
+
+describe("buildTrendQueries", () => {
+  it("builds correct queries for each platform", () => {
+    const sources: TrendSource[] = [
+      { source: "hackernews", min_score: 100 },
+      { source: "weibo_hot" },
+      { source: "reddit", subreddits: ["ChatGPT", "LocalLLaMA"] },
+    ];
+
+    const queries = buildTrendQueries(sources, ["AI"]);
+    expect(queries).toHaveLength(3);
+    expect(queries[0]).toContain("Hacker News");
+    expect(queries[0]).toContain("score>100");
+    expect(queries[1]).toContain("微博热搜");
+    expect(queries[2]).toContain("r/ChatGPT");
+    expect(queries[2]).toContain("r/LocalLLaMA");
+  });
+
+  it("skips disabled sources", () => {
+    const sources: TrendSource[] = [
+      { source: "hackernews", enabled: false },
+      { source: "weibo_hot", enabled: true },
+      { source: "google_trends" },
+    ];
+
+    const queries = buildTrendQueries(sources, ["test"]);
+    expect(queries).toHaveLength(2);
+    expect(queries.every((q) => !q.includes("Hacker News"))).toBe(true);
+  });
+
+  it("includes keywords in all queries", () => {
+    const sources: TrendSource[] = [
+      { source: "zhihu_hot" },
+      { source: "douyin_hot" },
+    ];
+
+    const queries = buildTrendQueries(sources, ["LLM", "Agent"]);
+    for (const q of queries) {
+      expect(q).toContain("LLM");
+      expect(q).toContain("Agent");
+    }
+  });
+
+  it("handles arxiv with categories", () => {
+    const sources: TrendSource[] = [
+      { source: "arxiv", categories: ["cs.AI", "cs.CL"] },
+    ];
+
+    const queries = buildTrendQueries(sources, ["transformer"]);
+    expect(queries[0]).toContain("arXiv");
+    expect(queries[0]).toContain("cs.AI");
+    expect(queries[0]).toContain("cs.CL");
+  });
+
+  it("handles twitter_trending with region", () => {
+    const sources: TrendSource[] = [
+      { source: "twitter_trending", region: "US" },
+    ];
+
+    const queries = buildTrendQueries(sources, ["tech"]);
+    expect(queries[0]).toContain("Twitter");
+    expect(queries[0]).toContain("US");
+  });
+});
+
+describe("createTrendCollector", () => {
+  it("uses trend source with 7-day expiry", async () => {
+    const mockSearch = vi.fn().mockResolvedValue([
+      { title: "Trending Topic", url: "https://example.com/t", snippet: "Hot" },
+    ]);
+    const collector = createTrendCollector(mockSearch);
+
+    // This will try to loadSourceConfig which needs real fs — we test the
+    // query building separately, so just verify the collector has correct id
+    expect(collector.id).toBe("trend");
+  });
+});

package/src/modules/intel/collectors/trends.ts

@@ -0,0 +1,107 @@
+import type { IntelItem } from "../../../storage/pipeline-store.js";
+import type { Collector, CollectorOptions, CollectorResult } from "../collector.js";
+import type { TrendSource } from "../source-config.js";
+import { loadSourceConfig } from "../source-config.js";
+import type { SearchFn, SearchResult } from "./web-search.js";
+
+// ─── Platform Query Templates ───────────────────────────────────────────────
+
+const PLATFORM_QUERY_MAP: Record<string, (src: TrendSource, keywords: string[]) => string> = {
+  hackernews: (src, keywords) =>
+    `Hacker News top stories today ${keywords.join(" ")}${src.min_score ? ` score>${src.min_score}` : ""}`,
+  producthunt: (_src, keywords) =>
+    `Product Hunt trending today ${keywords.join(" ")}`,
+  github_trending: (_src, keywords) =>
+    `GitHub trending repositories today ${keywords.join(" ")}`,
+  weibo_hot: (_src, keywords) =>
+    `微博热搜 今日 ${keywords.join(" ")}`,
+  zhihu_hot: (_src, keywords) =>
+    `知乎热榜 今日 ${keywords.join(" ")}`,
+  douyin_hot: (_src, keywords) =>
+    `抖音热榜 今日 ${keywords.join(" ")}`,
+  bilibili_hot: (_src, keywords) =>
+    `B站热搜 今日 ${keywords.join(" ")}`,
+  toutiao_hot: (_src, keywords) =>
+    `今日头条热榜 ${keywords.join(" ")}`,
+  twitter_trending: (src, keywords) =>
+    `Twitter trending${src.region ? ` ${src.region}` : ""} ${keywords.join(" ")}`,
+  reddit: (src, keywords) => {
+    const subs = src.subreddits?.length
+      ? `r/${src.subreddits.join(" r/")}`
+      : "";
+    return `Reddit trending ${subs} ${keywords.join(" ")}`.trim();
+  },
+  google_trends: (_src, keywords) =>
+    `Google Trends today ${keywords.join(" ")}`,
+  arxiv: (src, keywords) => {
+    const cats = src.categories?.length
+      ? src.categories.join(" ")
+      : "";
+    return `arXiv latest papers ${cats} ${keywords.join(" ")}`.trim();
+  },
+  youtube_trending: (_src, keywords) =>
+    `YouTube trending videos today ${keywords.join(" ")}`,
+};
+
+// ─── Query Builder ──────────────────────────────────────────────────────────
+
+export function buildTrendQueries(
+  sources: TrendSource[],
+  keywords: string[],
+): string[] {
+  return sources
+    .filter((src) => src.enabled !== false)
+    .map((src) => {
+      const buildQuery = PLATFORM_QUERY_MAP[src.source];
+      if (!buildQuery) return `${src.source} trending ${keywords.join(" ")}`;
+      return buildQuery(src, keywords);
+    });
+}
+
+// ─── Trend Collector ────────────────────────────────────────────────────────
+
+function trendResultToIntel(result: SearchResult, source: string): IntelItem {
+  return {
+    title: result.title,
+    domain: source,
+    source: "trend",
+    sourceUrl: result.url,
+    collectedAt: new Date().toISOString(),
+    relevance: 50,
+    tags: ["trend", source],
+    expiresAfter: 7,
+    summary: result.snippet,
+    keyPoints: [],
+    topicPotential: "",
+  };
+}
+
+export function createTrendCollector(searchFn: SearchFn): Collector {
+  return {
+    id: "trend",
+    async collect(opts: CollectorOptions): Promise<CollectorResult> {
+      const config = await loadSourceConfig(opts.dataDir);
+      const items: IntelItem[] = [];
+      const errors: string[] = [];
+      const seen = new Set<string>();
+
+      const queries = buildTrendQueries(config.trends, opts.keywords);
+
+      for (const query of queries) {
+        try {
+          const results = await searchFn(query);
+          for (const result of results) {
+            if (seen.has(result.url)) continue;
+            seen.add(result.url);
+            items.push(trendResultToIntel(result, "trend"));
+          }
+        } catch (err: unknown) {
+          const msg = err instanceof Error ? err.message : String(err);
+          errors.push(`Trend query "${query}" failed: ${msg}`);
+        }
+      }
+
+      return { items, source: "trend", errors };
+    },
+  };
+}

package/src/modules/intel/collectors/web-search.test.ts

@@ -0,0 +1,85 @@
+import { describe, it, expect, vi } from "vitest";
+import { buildMultiDimensionQueries, createWebSearchCollector } from "./web-search.js";
+import type { SearchResult } from "./web-search.js";
+
+describe("buildMultiDimensionQueries", () => {
+  it("generates 4+ queries with expected keywords", () => {
+    const queries = buildMultiDimensionQueries("AI Agent", "科技", ["小红书", "抖音"]);
+    expect(queries.length).toBeGreaterThanOrEqual(4);
+    // Every query should contain the keyword
+    for (const q of queries) {
+      expect(q).toContain("AI Agent");
+    }
+  });
+
+  it("includes dimension-specific terms", () => {
+    const queries = buildMultiDimensionQueries("ChatGPT", "AI", []);
+    const joined = queries.join(" ");
+    expect(joined).toContain("行业动态");
+    expect(joined).toContain("争议");
+    expect(joined).toContain("数据报告");
+    expect(joined).toContain("实操教程");
+    expect(joined).toContain("最新趋势");
+  });
+
+  it("adds platform-specific queries", () => {
+    const queries = buildMultiDimensionQueries("LLM", "科技", ["B站"]);
+    expect(queries.some((q) => q.includes("B站"))).toBe(true);
+  });
+});
+
+describe("createWebSearchCollector", () => {
+  it("collects items from search results", async () => {
+    const mockResults: SearchResult[] = [
+      { title: "Test Article", url: "https://example.com/1", snippet: "A test" },
+      { title: "Another Article", url: "https://example.com/2", snippet: "Another" },
+    ];
+    const searchFn = vi.fn().mockResolvedValue(mockResults);
+    const collector = createWebSearchCollector(searchFn);
+
+    const result = await collector.collect({
+      keywords: ["test"],
+      industry: "科技",
+      platforms: [],
+    });
+
+    expect(result.source).toBe("web_search");
+    expect(result.items.length).toBeGreaterThan(0);
+    expect(result.items[0].source).toBe("web_search");
+    expect(result.errors).toEqual([]);
+  });
+
+  it("deduplicates by URL", async () => {
+    const sameResult: SearchResult[] = [
+      { title: "Dup", url: "https://example.com/same", snippet: "dup" },
+    ];
+    const searchFn = vi.fn().mockResolvedValue(sameResult);
+    const collector = createWebSearchCollector(searchFn);
+
+    const result = await collector.collect({
+      keywords: ["test"],
+      industry: "科技",
+      platforms: [],
+    });
+
+    // All queries return the same URL, should be deduped to 1
+    const urls = result.items.map((i) => i.sourceUrl);
+    const unique = new Set(urls);
+    expect(unique.size).toBe(1);
+  });
+
+  it("captures errors without crashing", async () => {
+    const searchFn = vi.fn().mockRejectedValue(new Error("Network error"));
+    const collector = createWebSearchCollector(searchFn);
+
+    const result = await collector.collect({
+      keywords: ["fail"],
+      industry: "科技",
+      platforms: [],
+    });
+
+    expect(result.items).toEqual([]);
+    expect(result.errors.length).toBeGreaterThan(0);
+    expect(result.errors[0]).toContain("Network error");
+  });
+});

package/src/modules/intel/collectors/web-search.ts

@@ -0,0 +1,81 @@
+import type { IntelItem } from "../../../storage/pipeline-store.js";
+import type { Collector, CollectorOptions, CollectorResult } from "../collector.js";
+
+export interface SearchResult {
+  title: string;
+  url: string;
+  snippet: string;
+}
+
+export type SearchFn = (query: string) => Promise<SearchResult[]>;
+
+// ─── Query Builder ──────────────────────────────────────────────────────────
+
+export function buildMultiDimensionQueries(
+  keyword: string,
+  industry: string,
+  platforms: string[],
+): string[] {
+  const queries: string[] = [
+    `${keyword} ${industry} 行业动态 最新`,
+    `${keyword} 争议 观点 讨论`,
+    `${keyword} 数据报告 研究 ${new Date().getFullYear()}`,
+    `${keyword} 实操教程 方法论`,
+    `${keyword} 最新趋势 ${industry}`,
+  ];
+
+  // Platform-specific queries
+  for (const platform of platforms) {
+    queries.push(`${keyword} ${platform} 热门内容`);
+  }
+
+  return queries;
+}
+
+// ─── Web Search Collector ───────────────────────────────────────────────────
+
+function searchResultToIntel(result: SearchResult, keyword: string): IntelItem {
+  return {
+    title: result.title,
+    domain: keyword,
+    source: "web_search",
+    sourceUrl: result.url,
+    collectedAt: new Date().toISOString(),
+    relevance: 50,
+    tags: [keyword],
+    expiresAfter: 14,
+    summary: result.snippet,
+    keyPoints: [],
+    topicPotential: "",
+  };
+}
+
+export function createWebSearchCollector(searchFn: SearchFn): Collector {
+  return {
+    id: "web_search",
+    async collect(opts: CollectorOptions): Promise<CollectorResult> {
+      const items: IntelItem[] = [];
+      const errors: string[] = [];
+      const seen = new Set<string>();
+
+      for (const keyword of opts.keywords) {
+        const queries = buildMultiDimensionQueries(keyword, opts.industry, opts.platforms);
+        for (const query of queries) {
+          try {
+            const results = await searchFn(query);
+            for (const result of results) {
+              if (seen.has(result.url)) continue;
+              seen.add(result.url);
+              items.push(searchResultToIntel(result, keyword));
+            }
+          } catch (err: unknown) {
+            const msg = err instanceof Error ? err.message : String(err);
+            errors.push(`Query "${query}" failed: ${msg}`);
+          }
+        }
+      }
+
+      return { items, source: "web_search", errors };
+    },
+  };
+}

package/src/modules/intel/integration.test.ts

@@ -0,0 +1,203 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import {
+  initPipeline,
+  saveIntel,
+  listIntel,
+  saveTopic,
+  listTopics,
+  startProject,
+  addDraftVersion,
+  advanceProject,
+  getProjectMeta,
+  trashProject,
+  restoreProject,
+  listProjects,
+  slugify,
+  stagePath,
+  type IntelItem,
+  type TopicCandidate,
+} from "../../storage/pipeline-store.js";
+
+let testDir: string;
+
+beforeEach(async () => {
+  testDir = await fs.mkdtemp(
+    path.join(os.tmpdir(), "autocrew-integration-test-"),
+  );
+});
+
+afterEach(async () => {
+  await fs.rm(testDir, { recursive: true, force: true });
+});
+
+// ─── Fixtures ───────────────────────────────────────────────────────────────
+
+function makeIntel(overrides: Partial<IntelItem> = {}): IntelItem {
+  return {
+    title: "AI内容创作新趋势",
+    domain: "ai-content",
+    source: "web_search",
+    collectedAt: new Date().toISOString(),
+    relevance: 85,
+    tags: ["AI", "内容创作"],
+    expiresAfter: 7,
+    summary: "AI正在改变内容创作流程",
+    keyPoints: ["效率提升3倍", "质量可控"],
+    topicPotential: "可做系列教程选题",
+    ...overrides,
+  };
+}
+
+function makeTopic(overrides: Partial<TopicCandidate> = {}): TopicCandidate {
+  return {
+    title: "AI写作工具评测",
+    domain: "ai-content",
+    score: { heat: 80, differentiation: 70, audienceFit: 90, overall: 80 },
+    formats: ["video", "article"],
+    suggestedPlatforms: ["小红书", "B站"],
+    createdAt: new Date().toISOString(),
+    intelRefs: ["2024-01-15-ai-content-trend.md"],
+    angles: ["横向对比", "实操演示"],
+    audienceResonance: "目标用户对AI工具有强烈兴趣",
+    references: ["https://example.com/ai-tools"],
+    ...overrides,
+  };
+}
+
+// ─── Full Pipeline Flow ─────────────────────────────────────────────────────
+
+describe("Pipeline Integration — full flow", () => {
+  it("walks intel → topic → project → production → published", async () => {
+    // 1. Init pipeline
+    await initPipeline(testDir);
+    for (const stage of ["intel", "topics", "drafting", "production", "published", "trash"] as const) {
+      const stat = await fs.stat(stagePath(stage, testDir));
+      expect(stat.isDirectory()).toBe(true);
+    }
+
+    // 2. Save intel → list confirms it's there
+    const intel = makeIntel();
+    await saveIntel(intel, testDir);
+    const intelItems = await listIntel(undefined, testDir);
+    expect(intelItems.length).toBe(1);
+    expect(intelItems[0].title).toBe(intel.title);
+
+    // 3. Save topic → list confirms sorted by score
+    const topicHigh = makeTopic({
+      title: "高分选题",
+      score: { heat: 95, differentiation: 90, audienceFit: 95, overall: 95 },
+    });
+    const topicLow = makeTopic({
+      title: "低分选题",
+      score: { heat: 40, differentiation: 30, audienceFit: 50, overall: 40 },
+    });
+    await saveTopic(topicLow, testDir);
+    await saveTopic(topicHigh, testDir);
+
+    const topics = await listTopics(undefined, testDir);
+    expect(topics.length).toBe(2);
+    expect(topics[0].title).toBe("高分选题");
+    expect(topics[1].title).toBe("低分选题");
+
+    // 4. Start project from high-score topic → topic consumed, project dir created
+    const projectDir = await startProject("高分选题", testDir);
+    expect(projectDir).toContain("drafting");
+
+    const remainingTopics = await listTopics(undefined, testDir);
+    expect(remainingTopics.length).toBe(1);
+    expect(remainingTopics[0].title).toBe("低分选题");
+
+    const files = await fs.readdir(projectDir);
+    expect(files).toContain("meta.yaml");
+    expect(files).toContain("draft-v1.md");
+    expect(files).toContain("draft.md");
+
+    const projectName = slugify("高分选题");
+
+    // 5. Add draft version → meta.yaml updated with v2
+    await addDraftVersion(
+      projectName,
+      "# 高分选题\n\n第二版内容",
+      "improved draft",
+      testDir,
+    );
+
+    let meta = await getProjectMeta(projectName, testDir);
+    expect(meta).not.toBeNull();
+    expect(meta!.versions.length).toBe(2);
+    expect(meta!.current).toBe("draft-v2.md");
+    expect(meta!.versions[1].note).toBe("improved draft");
+
+    // 6. Advance: drafting → production → published
+    await advanceProject(projectName, testDir);
+    meta = await getProjectMeta(projectName, testDir);
+    expect(meta!.history.at(-1)!.stage).toBe("production");
+
+    await advanceProject(projectName, testDir);
+    meta = await getProjectMeta(projectName, testDir);
+    expect(meta!.history.at(-1)!.stage).toBe("published");
+
+    // 7. Verify history has all stage entries
+    expect(meta!.history.length).toBe(3);
+    expect(meta!.history[0].stage).toBe("drafting");
+    expect(meta!.history[1].stage).toBe("production");
+    expect(meta!.history[2].stage).toBe("published");
+  });
+});
+
+// ─── Trash & Restore ────────────────────────────────────────────────────────
+
+describe("Pipeline Integration — trash and restore", () => {
+  it("trashes and restores a project preserving state", async () => {
+    await initPipeline(testDir);
+
+    // Set up a project in drafting
+    await saveTopic(makeTopic({ title: "回收还原测试" }), testDir);
+    await startProject("回收还原测试", testDir);
+    const projectName = slugify("回收还原测试");
+
+    // Add a draft version so there's real state
+    await addDraftVersion(
+      projectName,
+      "# 重要内容\n\n不能丢失",
+      "v2",
+      testDir,
+    );
+
+    // Trash it
+    await trashProject(projectName, testDir);
+
+    const trashList = await listProjects("trash", testDir);
+    expect(trashList).toContain(projectName);
+
+    const draftingList = await listProjects("drafting", testDir);
+    expect(draftingList).not.toContain(projectName);
+
+    // Restore it
+    const restoredDir = await restoreProject(projectName, testDir);
+    expect(restoredDir).toContain("drafting");
+
+    // Verify state is preserved
+    const meta = await getProjectMeta(projectName, testDir);
+    expect(meta).not.toBeNull();
+    expect(meta!.versions.length).toBe(2);
+    expect(meta!.current).toBe("draft-v2.md");
+    expect(meta!.title).toBe("回收还原测试");
+
+    // History should show: drafting → trash → drafting
+    expect(meta!.history.length).toBe(3);
+    expect(meta!.history[0].stage).toBe("drafting");
+    expect(meta!.history[1].stage).toBe("trash");
+    expect(meta!.history[2].stage).toBe("drafting");
+
+    // Content should be intact
+    const draftContent = await fs.readFile(
+      path.join(restoredDir, "draft.md"),
+      "utf-8",
+    );
+    expect(draftContent).toBe("# 重要内容\n\n不能丢失");
+  });
+});