autocrew 0.1.0

package/src/modules/filter/sensitive-words.ts

@@ -0,0 +1,212 @@
+/**
+ * Sensitive Words Filter — scans text for sensitive/restricted words
+ *
+ * Sources:
+ * 1. Built-in word list: src/data/sensitive-words-builtin.json
+ * 2. User custom list:   ~/.autocrew/sensitive-words/custom.txt (one word per line)
+ * 3. Platform-specific restricted words with replacement suggestions
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+
+// --- Types ---
+
+export interface ScanHit {
+  word: string;
+  category: string;
+  /** Suggested replacement (if available) */
+  suggestion?: string;
+  /** All positions where this word appears */
+  positions: number[];
+}
+
+export interface ScanResult {
+  ok: boolean;
+  /** Total number of distinct sensitive words found */
+  hitCount: number;
+  /** Detailed hits */
+  hits: ScanHit[];
+  /** Quick summary for display */
+  summary: string;
+  /** Auto-fixed text (with suggestions applied) */
+  autoFixedText?: string;
+}
+
+interface BuiltinData {
+  categories: Record<string, { description: string; words: string[] }>;
+  platform_specific: Record<
+    string,
+    {
+      description: string;
+      restricted: string[];
+      suggestions?: Record<string, string>;
+    }
+  >;
+}
+
+// --- Loader ---
+
+let _builtinCache: BuiltinData | null = null;
+
+import { createRequire } from "node:module";
+import { fileURLToPath } from "node:url";
+
+const _require = createRequire(import.meta.url);
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+async function loadBuiltin(): Promise<BuiltinData> {
+  if (_builtinCache) return _builtinCache;
+  const filePath = path.resolve(__dirname, "../../data/sensitive-words-builtin.json");
+  const raw = await fs.readFile(filePath, "utf-8");
+  _builtinCache = JSON.parse(raw) as BuiltinData;
+  return _builtinCache;
+}
+
+async function loadCustomWords(dataDir?: string): Promise<string[]> {
+  const home = process.env.HOME || process.env.USERPROFILE || "~";
+  const customPath = path.join(dataDir || path.join(home, ".autocrew"), "sensitive-words", "custom.txt");
+  try {
+    const raw = await fs.readFile(customPath, "utf-8");
+    return raw
+      .split("\n")
+      .map((l) => l.trim())
+      .filter((l) => l && !l.startsWith("#"));
+  } catch {
+    return [];
+  }
+}
+
+// --- Core scan ---
+
+/**
+ * Scan text for sensitive words.
+ * @param text - The text to scan
+ * @param platform - Optional platform for platform-specific checks
+ * @param dataDir - Optional custom data directory
+ */
+export async function scanText(
+  text: string,
+  platform?: string,
+  dataDir?: string,
+): Promise<ScanResult> {
+  if (!text || !text.trim()) {
+    return { ok: true, hitCount: 0, hits: [], summary: "空文本，无需检查" };
+  }
+
+  const builtin = await loadBuiltin();
+  const customWords = await loadCustomWords(dataDir);
+  const hits: ScanHit[] = [];
+
+  // 1. Scan built-in categories
+  for (const [category, data] of Object.entries(builtin.categories)) {
+    for (const word of data.words) {
+      const positions = findAllPositions(text, word);
+      if (positions.length > 0) {
+        hits.push({ word, category, positions });
+      }
+    }
+  }
+
+  // 2. Scan platform-specific restricted words
+  if (platform && builtin.platform_specific[platform]) {
+    const platformData = builtin.platform_specific[platform];
+    for (const word of platformData.restricted) {
+      const positions = findAllPositions(text, word);
+      if (positions.length > 0) {
+        const suggestion = platformData.suggestions?.[word];
+        hits.push({
+          word,
+          category: `platform:${platform}`,
+          suggestion,
+          positions,
+        });
+      }
+    }
+  }
+
+  // 3. Scan custom words
+  for (const word of customWords) {
+    const positions = findAllPositions(text, word);
+    if (positions.length > 0) {
+      hits.push({ word, category: "custom", positions });
+    }
+  }
+
+  // Deduplicate by word
+  const deduped = deduplicateHits(hits);
+
+  // Build auto-fixed text
+  let autoFixedText: string | undefined;
+  const fixableHits = deduped.filter((h) => h.suggestion);
+  if (fixableHits.length > 0) {
+    autoFixedText = text;
+    for (const hit of fixableHits) {
+      autoFixedText = autoFixedText.replaceAll(hit.word, hit.suggestion!);
+    }
+  }
+
+  const summary = buildSummary(deduped);
+
+  return {
+    ok: deduped.length === 0,
+    hitCount: deduped.length,
+    hits: deduped,
+    summary,
+    autoFixedText,
+  };
+}
+
+// --- Helpers ---
+
+function findAllPositions(text: string, word: string): number[] {
+  const positions: number[] = [];
+  const lower = text.toLowerCase();
+  const target = word.toLowerCase();
+  let idx = lower.indexOf(target);
+  while (idx !== -1) {
+    positions.push(idx);
+    idx = lower.indexOf(target, idx + 1);
+  }
+  return positions;
+}
+
+function deduplicateHits(hits: ScanHit[]): ScanHit[] {
+  const map = new Map<string, ScanHit>();
+  for (const hit of hits) {
+    const existing = map.get(hit.word);
+    if (existing) {
+      // Merge positions, keep first category
+      existing.positions = [...new Set([...existing.positions, ...hit.positions])];
+      if (!existing.suggestion && hit.suggestion) {
+        existing.suggestion = hit.suggestion;
+      }
+    } else {
+      map.set(hit.word, { ...hit });
+    }
+  }
+  return Array.from(map.values());
+}
+
+function buildSummary(hits: ScanHit[]): string {
+  if (hits.length === 0) return "✅ 未检测到敏感词";
+
+  const byCat = new Map<string, number>();
+  for (const h of hits) {
+    byCat.set(h.category, (byCat.get(h.category) || 0) + 1);
+  }
+
+  const parts: string[] = [];
+  for (const [cat, count] of byCat) {
+    parts.push(`${cat}: ${count} 个`);
+  }
+
+  const fixable = hits.filter((h) => h.suggestion).length;
+  const fixNote = fixable > 0 ? `，其中 ${fixable} 个可自动替换` : "";
+
+  return `⚠️ 检测到 ${hits.length} 个敏感词（${parts.join("、")}）${fixNote}`;
+}
+
+/** Reset the built-in cache (for testing) */
+export function _resetCache(): void {
+  _builtinCache = null;
+}

package/src/modules/humanizer/zh.test.ts

@@ -0,0 +1,75 @@
+import { describe, it, expect } from "vitest";
+import { humanizeZh } from "../humanizer/zh.js";
+
+describe("humanizeZh", () => {
+  it("removes filler phrases", () => {
+    const result = humanizeZh({ text: "值得一提的是，这个功能很好用。" });
+    expect(result.ok).toBe(true);
+    expect(result.humanizedText).not.toContain("值得一提的是");
+    expect(result.changes.length).toBeGreaterThan(0);
+  });
+
+  it("replaces corporate buzzwords", () => {
+    const result = humanizeZh({ text: "这个产品可以赋能用户，助力企业发展。" });
+    expect(result.humanizedText).not.toContain("赋能");
+    expect(result.humanizedText).not.toContain("助力");
+    expect(result.humanizedText).toContain("帮");
+  });
+
+  it("replaces 闭环 with 跑通", () => {
+    const result = humanizeZh({ text: "我们需要形成完整的闭环。" });
+    expect(result.humanizedText).toContain("跑通");
+    expect(result.humanizedText).not.toContain("闭环");
+  });
+
+  it("removes summary openers", () => {
+    const result = humanizeZh({ text: "综上所述，这是一个好方案。" });
+    expect(result.humanizedText).not.toContain("综上所述");
+  });
+
+  it("removes 总而言之 and 总的来说", () => {
+    const r1 = humanizeZh({ text: "总而言之，效果不错。" });
+    const r2 = humanizeZh({ text: "总的来说，还可以。" });
+    expect(r1.humanizedText).not.toContain("总而言之");
+    expect(r2.humanizedText).not.toContain("总的来说");
+  });
+
+  it("removes vague adjectives", () => {
+    const result = humanizeZh({ text: "这是一个深度分析，全方位覆盖，多维度解读。" });
+    expect(result.humanizedText).not.toContain("深度");
+    expect(result.humanizedText).not.toContain("全方位");
+    expect(result.humanizedText).not.toContain("多维度");
+  });
+
+  it("returns original text unchanged when no AI patterns found", () => {
+    const clean = "今天天气不错，出去走走吧。";
+    const result = humanizeZh({ text: clean });
+    expect(result.ok).toBe(true);
+    expect(result.humanizedText).toBe(clean);
+    expect(result.changeCount).toBe(0);
+  });
+
+  it("returns ok:true and summary on success", () => {
+    const result = humanizeZh({ text: "赋能用户，助力发展。" });
+    expect(result.ok).toBe(true);
+    expect(result.summary).toContain("humanizer-zh");
+  });
+
+  it("handles empty string", () => {
+    const result = humanizeZh({ text: "" });
+    expect(result.ok).toBe(true);
+    expect(result.humanizedText).toBe("");
+  });
+
+  it("handles multiple replacements in one pass", () => {
+    const result = humanizeZh({
+      text: "值得一提的是，我们需要赋能用户，打通渠道，形成闭环。综上所述，这是深度分析。",
+    });
+    expect(result.humanizedText).not.toContain("值得一提的是");
+    expect(result.humanizedText).not.toContain("赋能");
+    expect(result.humanizedText).not.toContain("闭环");
+    expect(result.humanizedText).not.toContain("综上所述");
+    expect(result.humanizedText).not.toContain("深度");
+    expect(result.changes.length).toBeGreaterThanOrEqual(5);
+  });
+});

package/src/modules/humanizer/zh.ts

@@ -0,0 +1,175 @@
+export interface HumanizeZhOptions {
+  text: string;
+}
+
+export interface HumanizeZhResult {
+  ok: boolean;
+  originalText: string;
+  humanizedText: string;
+  changes: string[];
+  changeCount: number;
+  summary: string;
+}
+
+const DIRECT_REPLACEMENTS: Array<{ pattern: RegExp; replacement: string; note: string }> = [
+  { pattern: /值得一提的是/g, replacement: "", note: "删除空转折词“值得一提的是”" },
+  { pattern: /需要注意的是/g, replacement: "", note: "删除空提醒词“需要注意的是”" },
+  { pattern: /综上所述|总而言之|总的来说/g, replacement: "", note: "删除套路化总结句式" },
+  { pattern: /可以说|毫不夸张地说/g, replacement: "", note: "删除夸张前缀，直接表达判断" },
+  { pattern: /赋能/g, replacement: "帮", note: "把“赋能”改成具体动作词" },
+  { pattern: /助力/g, replacement: "帮", note: "把“助力”改成具体动作词" },
+  { pattern: /打通/g, replacement: "连接", note: "把“打通”改成更具体表达" },
+  { pattern: /闭环/g, replacement: "跑通", note: "把“闭环”改成更口语化表达" },
+  { pattern: /深度/g, replacement: "", note: "删除空泛形容词“深度”" },
+  { pattern: /全方位/g, replacement: "", note: "删除空泛形容词“全方位”" },
+  { pattern: /多维度/g, replacement: "", note: "删除空泛形容词“多维度”" },
+];
+
+function normalizeWhitespace(text: string): string {
+  return text
+    .replace(/[ \t]+\n/g, "\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .replace(/[ \t]{2,}/g, " ")
+    .trim();
+}
+
+function replaceWithTracking(
+  text: string,
+  pattern: RegExp,
+  replacement: string,
+): { text: string; count: number } {
+  let count = 0;
+  const nextText = text.replace(pattern, () => {
+    count += 1;
+    return replacement;
+  });
+  return { text: nextText, count };
+}
+
+function breakLongClauses(text: string): { text: string; count: number } {
+  const lines = text.split("\n");
+  let count = 0;
+  const nextLines = lines.map((line) => {
+    const trimmed = line.trim();
+    if (!trimmed) return line;
+    const chineseLength = (trimmed.match(/[\u4e00-\u9fff]/g) || []).length;
+    if (chineseLength <= 40) return line;
+
+    const replaced = line
+      .replace(/，(?=[^，。！？]{10,})/, "。")
+      .replace(/；(?=[^；。！？]{8,})/, "。");
+    if (replaced !== line) {
+      count += 1;
+      return replaced;
+    }
+    return line;
+  });
+  return { text: nextLines.join("\n"), count };
+}
+
+function simplifyProgressionPhrases(text: string): { text: string; count: number } {
+  let count = 0;
+  let next = text.replace(/首先[，,]?/g, () => {
+    count += 1;
+    return "";
+  });
+  next = next.replace(/其次[，,]?/g, () => {
+    count += 1;
+    return "";
+  });
+  next = next.replace(/最后[，,]?/g, () => {
+    count += 1;
+    return "最后，";
+  });
+  return { text: next, count };
+}
+
+function reduceWeOpenings(text: string): { text: string; count: number } {
+  const lines = text.split("\n");
+  let weCount = 0;
+  for (const line of lines) {
+    if (line.trim().startsWith("我们")) {
+      weCount += 1;
+    }
+  }
+  if (weCount <= 2) {
+    return { text, count: 0 };
+  }
+
+  let changed = 0;
+  const nextLines = lines.map((line) => {
+    if (changed >= weCount - 2) return line;
+    if (line.trim().startsWith("我们")) {
+      changed += 1;
+      return line.replace("我们", "你");
+    }
+    return line;
+  });
+  return { text: nextLines.join("\n"), count: changed };
+}
+
+function addRhythmPhraseIfNeeded(text: string): { text: string; count: number } {
+  if (/说白了|你想啊|问题来了/.test(text)) {
+    return { text, count: 0 };
+  }
+
+  const paragraphs = text.split(/\n{2,}/);
+  if (paragraphs.length < 2) {
+    return { text, count: 0 };
+  }
+
+  const next = [...paragraphs];
+  next.splice(1, 0, "说白了，这件事拼的不是工具数量，而是表达和执行。");
+  return { text: next.join("\n\n"), count: 1 };
+}
+
+export function humanizeZh(options: HumanizeZhOptions): HumanizeZhResult {
+  const originalText = options.text || "";
+  let humanizedText = originalText;
+  const changes: string[] = [];
+
+  for (const replacement of DIRECT_REPLACEMENTS) {
+    const result = replaceWithTracking(humanizedText, replacement.pattern, replacement.replacement);
+    if (result.count > 0) {
+      humanizedText = result.text;
+      changes.push(`${replacement.note} × ${result.count}`);
+    }
+  }
+
+  const progression = simplifyProgressionPhrases(humanizedText);
+  if (progression.count > 0) {
+    humanizedText = progression.text;
+    changes.push(`打散“首先/其次/最后”顺序词 × ${progression.count}`);
+  }
+
+  const longClauses = breakLongClauses(humanizedText);
+  if (longClauses.count > 0) {
+    humanizedText = longClauses.text;
+    changes.push(`拆开过长句子 × ${longClauses.count}`);
+  }
+
+  const weOpenings = reduceWeOpenings(humanizedText);
+  if (weOpenings.count > 0) {
+    humanizedText = weOpenings.text;
+    changes.push(`减少“我们”开头句子 × ${weOpenings.count}`);
+  }
+
+  const rhythm = addRhythmPhraseIfNeeded(humanizedText);
+  if (rhythm.count > 0) {
+    humanizedText = rhythm.text;
+    changes.push("补入 1 处口语化节奏句");
+  }
+
+  humanizedText = normalizeWhitespace(humanizedText);
+  return {
+    ok: true,
+    originalText,
+    humanizedText,
+    changes,
+    changeCount: changes.length,
+    summary:
+      changes.length > 0
+        ? `humanizer-zh 完成：修改了 ${changes.length} 类问题`
+        : "humanizer-zh 完成：确认无明显 AI 痕迹",
+  };
+}

package/src/modules/intel/collector.ts

@@ -0,0 +1,19 @@
+import type { IntelItem } from "../../storage/pipeline-store.js";
+
+export interface CollectorResult {
+  items: IntelItem[];
+  source: string;
+  errors: string[];
+}
+
+export interface Collector {
+  id: string;
+  collect(opts: CollectorOptions): Promise<CollectorResult>;
+}
+
+export interface CollectorOptions {
+  keywords: string[];
+  industry: string;
+  platforms: string[];
+  dataDir?: string;
+}

package/src/modules/intel/collectors/competitor.test.ts

@@ -0,0 +1,71 @@
+import { describe, it, expect } from "vitest";
+import { transformCompetitorResults } from "./competitor.js";
+import type { ResearchItem } from "../../../adapters/browser/types.js";
+
+describe("transformCompetitorResults", () => {
+  it("produces correct IntelItems with competitor tags", () => {
+    const results: ResearchItem[] = [
+      {
+        title: "Competitor Post 1",
+        summary: "Their latest product launch",
+        url: "https://xiaohongshu.com/note/123",
+        platform: "xiaohongshu",
+        source: "browser_cdp",
+      },
+      {
+        title: "Competitor Post 2",
+        url: "https://xiaohongshu.com/note/456",
+        platform: "xiaohongshu",
+        source: "browser_cdp",
+      },
+    ];
+
+    const items = transformCompetitorResults(results, "竞品A", "美妆", "xiaohongshu");
+    expect(items).toHaveLength(2);
+    expect(items[0].source).toBe("competitor");
+    expect(items[0].domain).toBe("美妆");
+    expect(items[0].tags).toContain("competitor:竞品A");
+    expect(items[0].tags).toContain("xiaohongshu");
+    expect(items[0].title).toBe("Competitor Post 1");
+    expect(items[0].summary).toBe("Their latest product launch");
+    expect(items[0].sourceUrl).toBe("https://xiaohongshu.com/note/123");
+    expect(items[0].relevance).toBe(60);
+  });
+
+  it("filters out items without titles", () => {
+    const results: ResearchItem[] = [
+      {
+        title: "",
+        platform: "xiaohongshu",
+        source: "browser_cdp",
+      },
+      {
+        title: "Valid Title",
+        platform: "xiaohongshu",
+        source: "browser_cdp",
+      },
+    ];
+
+    const items = transformCompetitorResults(results, "竞品B", "科技", "xiaohongshu");
+    expect(items).toHaveLength(1);
+    expect(items[0].title).toBe("Valid Title");
+  });
+
+  it("handles empty results array", () => {
+    const items = transformCompetitorResults([], "竞品C", "教育", "bilibili");
+    expect(items).toEqual([]);
+  });
+
+  it("uses empty string for missing summary", () => {
+    const results: ResearchItem[] = [
+      {
+        title: "No Summary",
+        platform: "douyin",
+        source: "browser_cdp",
+      },
+    ];
+
+    const items = transformCompetitorResults(results, "竞品D", "美食", "douyin");
+    expect(items[0].summary).toBe("");
+  });
+});

package/src/modules/intel/collectors/competitor.ts

@@ -0,0 +1,65 @@
+import type { IntelItem } from "../../../storage/pipeline-store.js";
+import type { Collector, CollectorOptions, CollectorResult } from "../collector.js";
+import type { ResearchItem, BrowserPlatform } from "../../../adapters/browser/types.js";
+import { browserCdpAdapter } from "../../../adapters/browser/browser-cdp.js";
+import { loadSourceConfig } from "../source-config.js";
+
+// ─── Result Transformer ────────────────────────────────────────────────────
+
+export function transformCompetitorResults(
+  results: ResearchItem[],
+  accountName: string,
+  domain: string,
+  platform: string,
+): IntelItem[] {
+  return results
+    .filter((r) => r.title)
+    .map((r) => ({
+      title: r.title,
+      domain,
+      source: "competitor" as const,
+      sourceUrl: r.url,
+      collectedAt: new Date().toISOString(),
+      relevance: 60,
+      tags: [`competitor:${accountName}`, platform],
+      expiresAfter: 14,
+      summary: r.summary ?? "",
+      keyPoints: [],
+      topicPotential: "",
+    }));
+}
+
+// ─── Competitor Collector ───────────────────────────────────────────────────
+
+export function createCompetitorCollector(): Collector {
+  return {
+    id: "competitor",
+    async collect(opts: CollectorOptions): Promise<CollectorResult> {
+      const config = await loadSourceConfig(opts.dataDir);
+      const items: IntelItem[] = [];
+      const errors: string[] = [];
+
+      for (const account of config.accounts) {
+        try {
+          const results = await browserCdpAdapter.research({
+            platform: account.platform as BrowserPlatform,
+            keyword: account.name,
+            limit: 10,
+          });
+          const transformed = transformCompetitorResults(
+            results,
+            account.name,
+            account.domain,
+            account.platform,
+          );
+          items.push(...transformed);
+        } catch (err: unknown) {
+          const msg = err instanceof Error ? err.message : String(err);
+          errors.push(`Competitor ${account.name} (${account.platform}) failed: ${msg}`);
+        }
+      }
+
+      return { items, source: "competitor", errors };
+    },
+  };
+}

package/src/modules/intel/collectors/rss.test.ts

@@ -0,0 +1,56 @@
+import { describe, it, expect } from "vitest";
+import { parseRssItems } from "./rss.js";
+import type { RssItem } from "./rss.js";
+
+describe("parseRssItems", () => {
+  it("transforms RSS items into IntelItems", () => {
+    const items: RssItem[] = [
+      {
+        title: "AI Breakthrough",
+        link: "https://example.com/article",
+        contentSnippet: "Major AI advancement announced today",
+        isoDate: "2024-01-15T10:00:00Z",
+        categories: ["tech", "ai"],
+      },
+    ];
+
+    const result = parseRssItems(items, "科技", ["rss-source"]);
+    expect(result).toHaveLength(1);
+    expect(result[0].title).toBe("AI Breakthrough");
+    expect(result[0].domain).toBe("科技");
+    expect(result[0].source).toBe("rss");
+    expect(result[0].sourceUrl).toBe("https://example.com/article");
+    expect(result[0].summary).toBe("Major AI advancement announced today");
+    expect(result[0].tags).toEqual(["rss-source", "tech", "ai"]);
+    expect(result[0].collectedAt).toBe("2024-01-15T10:00:00Z");
+  });
+
+  it("handles missing fields gracefully", () => {
+    const items: RssItem[] = [
+      { title: "No Link Article" },
+      { title: undefined }, // should be filtered out
+      { title: "Minimal", content: "Full content here that is quite long and should be sliced" },
+    ];
+
+    const result = parseRssItems(items, "默认");
+    expect(result).toHaveLength(2);
+    expect(result[0].title).toBe("No Link Article");
+    expect(result[0].sourceUrl).toBeUndefined();
+    expect(result[0].summary).toBe("");
+    expect(result[0].tags).toEqual([]);
+    expect(result[1].title).toBe("Minimal");
+    expect(result[1].summary).toContain("Full content");
+  });
+
+  it("uses isoDate when available, falls back to now", () => {
+    const withDate: RssItem[] = [{ title: "Dated", isoDate: "2024-06-01T00:00:00Z" }];
+    const withoutDate: RssItem[] = [{ title: "Undated" }];
+
+    const dated = parseRssItems(withDate, "test");
+    const undated = parseRssItems(withoutDate, "test");
+
+    expect(dated[0].collectedAt).toBe("2024-06-01T00:00:00Z");
+    // Should be a valid ISO date (now-ish)
+    expect(new Date(undated[0].collectedAt).getTime()).toBeGreaterThan(0);
+  });
+});