autocrew 0.1.0

package/src/modules/intel/intel-engine.test.ts

@@ -0,0 +1,103 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import fs from "node:fs/promises";
+import path from "node:path";
+import os from "node:os";
+import { runIntelPull } from "./intel-engine.js";
+import type { SearchResult } from "./collectors/web-search.js";
+
+describe("runIntelPull", () => {
+  let tmpDir: string;
+
+  beforeEach(async () => {
+    tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "intel-engine-test-"));
+  });
+
+  afterEach(async () => {
+    await fs.rm(tmpDir, { recursive: true, force: true });
+  });
+
+  it("collects and saves intel from web_search source", async () => {
+    const mockResults: SearchResult[] = [
+      { title: "AI Breakthrough", url: "https://example.com/1", snippet: "Big news in AI" },
+      { title: "LLM Update", url: "https://example.com/2", snippet: "New model released" },
+    ];
+    const searchFn = vi.fn().mockResolvedValue(mockResults);
+
+    const result = await runIntelPull({
+      keywords: ["AI"],
+      industry: "科技",
+      platforms: ["小红书"],
+      dataDir: tmpDir,
+      searchFn,
+      skipBrowser: true,
+      sources: ["web_search"],
+    });
+
+    expect(result.totalCollected).toBeGreaterThan(0);
+    expect(result.totalSaved).toBeGreaterThan(0);
+    expect(result.bySource.web_search).toBeGreaterThan(0);
+    expect(result.errors).toEqual([]);
+
+    // Verify files were saved
+    const intelDir = path.join(tmpDir, "pipeline", "intel", "AI");
+    const files = await fs.readdir(intelDir);
+    expect(files.length).toBeGreaterThan(0);
+  });
+
+  it("handles collector errors gracefully", async () => {
+    const searchFn = vi.fn().mockRejectedValue(new Error("Network down"));
+
+    const result = await runIntelPull({
+      keywords: ["fail"],
+      industry: "科技",
+      platforms: [],
+      dataDir: tmpDir,
+      searchFn,
+      skipBrowser: true,
+      sources: ["web_search"],
+    });
+
+    expect(result.totalCollected).toBe(0);
+    expect(result.totalSaved).toBe(0);
+    expect(result.errors.length).toBeGreaterThan(0);
+  });
+
+  it("filters to specific sources", async () => {
+    const searchFn = vi.fn().mockResolvedValue([
+      { title: "Test", url: "https://example.com/x", snippet: "test" },
+    ]);
+
+    const result = await runIntelPull({
+      keywords: ["test"],
+      industry: "科技",
+      platforms: [],
+      dataDir: tmpDir,
+      searchFn,
+      skipBrowser: true,
+      sources: ["web_search"],
+    });
+
+    // Only web_search source should be present
+    expect(Object.keys(result.bySource)).toEqual(["web_search"]);
+  });
+
+  it("runs multiple collectors in parallel", async () => {
+    const searchFn = vi.fn().mockResolvedValue([
+      { title: "Multi", url: "https://example.com/m", snippet: "multi" },
+    ]);
+
+    const result = await runIntelPull({
+      keywords: ["test"],
+      industry: "科技",
+      platforms: [],
+      dataDir: tmpDir,
+      searchFn,
+      skipBrowser: true,
+      // No sources filter — runs web_search, rss, trend (skips competitor due to skipBrowser)
+    });
+
+    // web_search and trend both use searchFn, rss may return 0 (no config)
+    expect(result.totalCollected).toBeGreaterThan(0);
+    expect(result.totalSaved).toBeGreaterThan(0);
+  });
+});

package/src/modules/intel/intel-engine.ts

@@ -0,0 +1,96 @@
+import type { IntelItem } from "../../storage/pipeline-store.js";
+import { saveIntel } from "../../storage/pipeline-store.js";
+import type { Collector, CollectorOptions } from "./collector.js";
+import { createWebSearchCollector } from "./collectors/web-search.js";
+import type { SearchFn } from "./collectors/web-search.js";
+import { createRssCollector } from "./collectors/rss.js";
+import { createTrendCollector } from "./collectors/trends.js";
+import { createCompetitorCollector } from "./collectors/competitor.js";
+
+// ─── Types ──────────────────────────────────────────────────────────────────
+
+export interface IntelPullOptions {
+  keywords: string[];
+  industry: string;
+  platforms: string[];
+  dataDir?: string;
+  searchFn: SearchFn;
+  skipBrowser?: boolean;
+  sources?: string[]; // filter to specific sources
+}
+
+export interface IntelPullResult {
+  totalCollected: number;
+  totalSaved: number;
+  bySource: Record<string, number>;
+  errors: string[];
+}
+
+// ─── Orchestrator ───────────────────────────────────────────────────────────
+
+function buildCollectors(opts: IntelPullOptions): Collector[] {
+  const all: Collector[] = [
+    createWebSearchCollector(opts.searchFn),
+    createRssCollector(),
+    createTrendCollector(opts.searchFn),
+  ];
+
+  if (!opts.skipBrowser) {
+    all.push(createCompetitorCollector());
+  }
+
+  if (opts.sources?.length) {
+    const allowed = new Set(opts.sources);
+    return all.filter((c) => allowed.has(c.id));
+  }
+
+  return all;
+}
+
+export async function runIntelPull(opts: IntelPullOptions): Promise<IntelPullResult> {
+  const collectors = buildCollectors(opts);
+
+  const collectorOpts: CollectorOptions = {
+    keywords: opts.keywords,
+    industry: opts.industry,
+    platforms: opts.platforms,
+    dataDir: opts.dataDir,
+  };
+
+  const settled = await Promise.allSettled(
+    collectors.map((c) => c.collect(collectorOpts)),
+  );
+
+  const allItems: IntelItem[] = [];
+  const errors: string[] = [];
+  const bySource: Record<string, number> = {};
+
+  for (const result of settled) {
+    if (result.status === "fulfilled") {
+      const { items, source, errors: collectorErrors } = result.value;
+      allItems.push(...items);
+      bySource[source] = (bySource[source] ?? 0) + items.length;
+      errors.push(...collectorErrors);
+    } else {
+      errors.push(`Collector failed: ${result.reason}`);
+    }
+  }
+
+  let totalSaved = 0;
+  for (const item of allItems) {
+    try {
+      await saveIntel(item, opts.dataDir);
+      totalSaved++;
+    } catch (err: unknown) {
+      const msg = err instanceof Error ? err.message : String(err);
+      errors.push(`Failed to save "${item.title}": ${msg}`);
+    }
+  }
+
+  return {
+    totalCollected: allItems.length,
+    totalSaved,
+    bySource,
+    errors,
+  };
+}

package/src/modules/intel/source-config.test.ts

@@ -0,0 +1,113 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import yaml from "js-yaml";
+import { loadSourceConfig, getRecommendedSources } from "./source-config.js";
+
+let testDir: string;
+
+beforeEach(async () => {
+  testDir = await fs.mkdtemp(
+    path.join(os.tmpdir(), "autocrew-source-config-test-"),
+  );
+  // Create pipeline/intel/_sources directory
+  await fs.mkdir(
+    path.join(testDir, "pipeline", "intel", "_sources"),
+    { recursive: true },
+  );
+});
+
+afterEach(async () => {
+  await fs.rm(testDir, { recursive: true, force: true });
+});
+
+describe("loadSourceConfig", () => {
+  it("loads RSS config from _sources/*.yaml", async () => {
+    const configData = {
+      rss: [
+        { url: "https://example.com/feed", domain: "科技" },
+        { url: "https://other.com/rss", domain: "AI", tags: ["ml"] },
+      ],
+      keywords: ["AI", "LLM"],
+    };
+    await fs.writeFile(
+      path.join(testDir, "pipeline", "intel", "_sources", "feeds.yaml"),
+      yaml.dump(configData),
+    );
+
+    const config = await loadSourceConfig(testDir);
+    expect(config.rss).toHaveLength(2);
+    expect(config.rss[0].url).toBe("https://example.com/feed");
+    expect(config.rss[1].tags).toEqual(["ml"]);
+    expect(config.keywords).toEqual(["AI", "LLM"]);
+  });
+
+  it("returns empty arrays when no config files exist", async () => {
+    const config = await loadSourceConfig(testDir);
+    expect(config.rss).toEqual([]);
+    expect(config.trends).toEqual([]);
+    expect(config.accounts).toEqual([]);
+    expect(config.keywords).toEqual([]);
+  });
+
+  it("returns empty arrays when _sources dir does not exist", async () => {
+    const emptyDir = await fs.mkdtemp(
+      path.join(os.tmpdir(), "autocrew-empty-test-"),
+    );
+    const config = await loadSourceConfig(emptyDir);
+    expect(config.rss).toEqual([]);
+    expect(config.trends).toEqual([]);
+    await fs.rm(emptyDir, { recursive: true, force: true });
+  });
+
+  it("merges multiple yaml files", async () => {
+    const sourcesDir = path.join(testDir, "pipeline", "intel", "_sources");
+    await fs.writeFile(
+      path.join(sourcesDir, "rss.yaml"),
+      yaml.dump({ rss: [{ url: "https://a.com/feed", domain: "A" }] }),
+    );
+    await fs.writeFile(
+      path.join(sourcesDir, "trends.yaml"),
+      yaml.dump({ trends: [{ source: "hackernews", min_score: 100 }] }),
+    );
+
+    const config = await loadSourceConfig(testDir);
+    expect(config.rss).toHaveLength(1);
+    expect(config.trends).toHaveLength(1);
+    expect(config.trends[0].source).toBe("hackernews");
+  });
+});
+
+describe("getRecommendedSources", () => {
+  it("recommends sources by exact industry match", async () => {
+    const result = await getRecommendedSources("科技");
+    expect(result.trends.length).toBeGreaterThan(0);
+    expect(result.trends.some((t) => t.source === "hackernews")).toBe(true);
+    expect(result.rssSuggestions.length).toBeGreaterThan(0);
+  });
+
+  it("different industries get different sources", async () => {
+    const tech = await getRecommendedSources("科技");
+    const beauty = await getRecommendedSources("美妆");
+
+    // 科技 has hackernews, 美妆 does not
+    expect(tech.trends.some((t) => t.source === "hackernews")).toBe(true);
+    expect(beauty.trends.some((t) => t.source === "hackernews")).toBe(false);
+
+    // 美妆 has douyin_hot, 科技 does not
+    expect(beauty.trends.some((t) => t.source === "douyin_hot")).toBe(true);
+  });
+
+  it("falls back to _default for unknown industry", async () => {
+    const result = await getRecommendedSources("未知行业");
+    expect(result.trends.length).toBeGreaterThan(0);
+    expect(result.trends.some((t) => t.source === "weibo_hot")).toBe(true);
+  });
+
+  it("AI industry includes arxiv and reddit sources", async () => {
+    const result = await getRecommendedSources("AI");
+    expect(result.trends.some((t) => t.source === "arxiv")).toBe(true);
+    expect(result.trends.some((t) => t.source === "reddit")).toBe(true);
+  });
+});

package/src/modules/intel/source-config.ts

@@ -0,0 +1,131 @@
+import path from "node:path";
+import fs from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import yaml from "js-yaml";
+import { pipelinePath } from "../../storage/pipeline-store.js";
+
+// ─── Types ──────────────────────────────────────────────────────────────────
+
+export interface RssFeed {
+  url: string;
+  domain: string;
+  tags?: string[];
+}
+
+export interface TrendSource {
+  source: string;
+  enabled?: boolean;
+  region?: string;
+  subreddits?: string[];
+  min_score?: number;
+  keywords?: string[];
+  categories?: string[];
+}
+
+export interface CompetitorAccount {
+  platform: string;
+  name: string;
+  id: string;
+  domain: string;
+}
+
+export interface SourceConfig {
+  rss: RssFeed[];
+  trends: TrendSource[];
+  accounts: CompetitorAccount[];
+  keywords: string[];
+}
+
+export interface RecommendedSources {
+  trends: TrendSource[];
+  rssSuggestions: RssFeed[];
+}
+
+// ─── Source Config Loader ───────────────────────────────────────────────────
+
+export async function loadSourceConfig(dataDir?: string): Promise<SourceConfig> {
+  const sourcesDir = path.join(pipelinePath(dataDir), "intel", "_sources");
+  const config: SourceConfig = {
+    rss: [],
+    trends: [],
+    accounts: [],
+    keywords: [],
+  };
+
+  let files: string[];
+  try {
+    files = (await fs.readdir(sourcesDir)).filter((f) => f.endsWith(".yaml") || f.endsWith(".yml"));
+  } catch {
+    return config;
+  }
+
+  for (const file of files) {
+    const content = await fs.readFile(path.join(sourcesDir, file), "utf-8");
+    const data = yaml.load(content) as Record<string, unknown> | null;
+    if (!data) continue;
+
+    if (Array.isArray(data.rss)) {
+      config.rss.push(...(data.rss as RssFeed[]));
+    }
+    if (Array.isArray(data.trends)) {
+      config.trends.push(...(data.trends as TrendSource[]));
+    }
+    if (Array.isArray(data.accounts)) {
+      config.accounts.push(...(data.accounts as CompetitorAccount[]));
+    }
+    if (Array.isArray(data.keywords)) {
+      config.keywords.push(...(data.keywords as string[]));
+    }
+  }
+
+  return config;
+}
+
+// ─── Recommended Sources ────────────────────────────────────────────────────
+
+interface PresetEntry {
+  trends: TrendSource[];
+  rss_suggestions: RssFeed[];
+}
+
+interface PresetsFile {
+  presets: Record<string, PresetEntry>;
+}
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const PRESETS_PATH = path.resolve(__dirname, "../../data/source-presets.yaml");
+
+export async function getRecommendedSources(industry: string): Promise<RecommendedSources> {
+  const content = await fs.readFile(PRESETS_PATH, "utf-8");
+  const file = yaml.load(content) as PresetsFile;
+  const presets = file.presets;
+
+  // Exact match
+  if (presets[industry]) {
+    const preset = presets[industry];
+    return {
+      trends: preset.trends ?? [],
+      rssSuggestions: preset.rss_suggestions ?? [],
+    };
+  }
+
+  // Partial match — find first key that contains the industry string or vice versa
+  for (const key of Object.keys(presets)) {
+    if (key === "_default") continue;
+    if (key.includes(industry) || industry.includes(key)) {
+      const preset = presets[key];
+      return {
+        trends: preset.trends ?? [],
+        rssSuggestions: preset.rss_suggestions ?? [],
+      };
+    }
+  }
+
+  // Default fallback
+  const fallback = presets._default;
+  return {
+    trends: fallback?.trends ?? [],
+    rssSuggestions: fallback?.rss_suggestions ?? [],
+  };
+}

package/src/modules/learnings/diff-tracker.test.ts

@@ -0,0 +1,144 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import {
+  detectPatterns,
+  recordDiff,
+  listDiffs,
+  getPatternFrequency,
+} from "../learnings/diff-tracker.js";
+
+let testDir: string;
+
+beforeEach(async () => {
+  testDir = await fs.mkdtemp(path.join(os.tmpdir(), "autocrew-diff-test-"));
+});
+
+afterEach(async () => {
+  await fs.rm(testDir, { recursive: true, force: true });
+});
+
+describe("detectPatterns", () => {
+  it("detects remove_progression_words when 首先/其次/最后 are removed", () => {
+    const patterns = detectPatterns(
+      "首先我们来看。其次分析。最后总结。",
+      "我们来看，分析，总结。",
+    );
+    expect(patterns).toContain("remove_progression_words");
+  });
+
+  it("detects shorten_content when text gets significantly shorter", () => {
+    const long = "这是一个非常非常非常长的句子，包含了很多很多很多的内容，需要被大幅度缩短。这里还有更多内容。";
+    const short = "短。";
+    const patterns = detectPatterns(long, short);
+    expect(patterns).toContain("shorten_content");
+  });
+
+  it("detects add_emoji when 3+ emoji added", () => {
+    const patterns = detectPatterns("这是一段文字", "这是一段文字 🎉🔥✨💡");
+    expect(patterns).toContain("add_emoji");
+  });
+
+  it("detects reduce_we_pronoun when 2+ 我们 are removed", () => {
+    const patterns = detectPatterns(
+      "我们来看看这个问题，我们分析原因，我们给出结论。",
+      "来看看这个问题，分析原因，给出结论。",
+    );
+    expect(patterns).toContain("reduce_we_pronoun");
+  });
+
+  it("returns empty array when no patterns detected", () => {
+    const patterns = detectPatterns("今天天气不错", "今天天气不错，出去走走");
+    // Minor addition — no strong pattern
+    expect(Array.isArray(patterns)).toBe(true);
+  });
+
+  it("can detect multiple patterns at once", () => {
+    const patterns = detectPatterns(
+      "首先我们来看看这个非常非常长的问题，其次我们分析原因，最后我们给出结论。",
+      "来看这问题 🎉，分析原因，给结论。",
+    );
+    expect(patterns.length).toBeGreaterThanOrEqual(2);
+  });
+});
+
+describe("recordDiff", () => {
+  it("saves a diff file and returns the diff object", async () => {
+    const diff = await recordDiff("content-001", "body", "原始文本", "修改后文本", testDir);
+    expect(diff.id).toMatch(/^diff-/);
+    expect(diff.contentId).toBe("content-001");
+    expect(diff.before).toBe("原始文本");
+    expect(diff.after).toBe("修改后文本");
+    expect(diff.createdAt).toBeTruthy();
+  });
+
+  it("truncates very long before/after to 2000 chars", async () => {
+    const longText = "x".repeat(5000);
+    const diff = await recordDiff("content-002", "body", longText, "short", testDir);
+    expect(diff.before.length).toBeLessThanOrEqual(2000);
+  });
+
+  it("persists to disk", async () => {
+    await recordDiff("content-003", "title", "旧标题", "新标题", testDir);
+    const diffs = await listDiffs(undefined, testDir);
+    expect(diffs.length).toBeGreaterThanOrEqual(1);
+  });
+});
+
+describe("listDiffs", () => {
+  it("returns empty array when no diffs exist", async () => {
+    const diffs = await listDiffs(undefined, testDir);
+    expect(diffs).toEqual([]);
+  });
+
+  it("returns all recorded diffs", async () => {
+    await recordDiff("c1", "body", "a", "b", testDir);
+    await recordDiff("c2", "body", "c", "d", testDir);
+    const diffs = await listDiffs(undefined, testDir);
+    expect(diffs.length).toBe(2);
+  });
+
+  it("filters by contentId", async () => {
+    await recordDiff("c1", "body", "a", "b", testDir);
+    await recordDiff("c2", "body", "c", "d", testDir);
+    const diffs = await listDiffs({ contentId: "c1" }, testDir);
+    expect(diffs.length).toBe(1);
+    expect(diffs[0].contentId).toBe("c1");
+  });
+});
+
+describe("getPatternFrequency", () => {
+  it("returns empty array when no diffs", async () => {
+    const freq = await getPatternFrequency(testDir);
+    expect(freq).toEqual([]);
+  });
+
+  it("counts pattern occurrences across diffs", async () => {
+    // Record 3 diffs that all trigger remove_progression_words
+    for (let i = 0; i < 3; i++) {
+      await recordDiff(
+        `c${i}`,
+        "body",
+        "首先看，其次分析，最后总结。",
+        "看，分析，总结。",
+        testDir,
+      );
+    }
+    const freq = await getPatternFrequency(testDir);
+    const entry = freq.find((f) => f.pattern === "remove_progression_words");
+    expect(entry).toBeDefined();
+    expect(entry!.count).toBe(3);
+  });
+
+  it("sorts by frequency descending", async () => {
+    // 3x pattern A, 1x pattern B
+    for (let i = 0; i < 3; i++) {
+      await recordDiff(`ca${i}`, "body", "首先看，其次分析，最后总结。", "看，分析，总结。", testDir);
+    }
+    await recordDiff("cb1", "body", "我们来看", "来看", testDir);
+
+    const freq = await getPatternFrequency(testDir);
+    expect(freq[0].count).toBeGreaterThanOrEqual(freq[1]?.count ?? 0);
+  });
+});