@web-auto/webauto 0.1.4 → 0.1.7

package/modules/collection-manager/index.ts

@@ -0,0 +1,309 @@
+/**
+ * Collection Data Manager
+ * 
+ * Unified data management for all platforms and collection types.
+ * Handles:
+ * - Deduplication via Bloom Filter
+ * - Fresh/Incremental modes
+ * - File storage with human-readable paths
+ * - Stats and persistence
+ */
+
+import * as os from 'os';
+import * as path from 'path';
+import { BloomFilter } from './bloom-filter';
+import { CollectionStorage } from './storage';
+import { getCurrentTimestamp, parsePlatformDate } from './date-utils';
+import type { 
+  Platform, 
+  CollectionSource, 
+  CollectionMode, 
+  CollectionMeta,
+  CollectionIdSpec,
+  PostRecord,
+  CommentRecord 
+} from './types';
+import { buildCollectionId, parseCollectionId } from './types';
+
+export interface CollectionManagerOptions {
+  platform: Platform;
+  env: string;
+  spec: CollectionIdSpec;
+  mode?: CollectionMode;
+  baseDir?: string;
+}
+
+export interface CollectionStats {
+  totalPosts: number;
+  totalComments: number;
+  newPosts: number;
+  newComments: number;
+  duplicatesSkipped: number;
+}
+
+export class CollectionDataManager {
+  private platform: Platform;
+  private env: string;
+  private spec: CollectionIdSpec;
+  private mode: CollectionMode;
+  private collectionId: string;
+  private storage: CollectionStorage;
+  private bloomFilter: BloomFilter;
+  private meta: CollectionMeta | null = null;
+  private stats: CollectionStats = {
+    totalPosts: 0,
+    totalComments: 0,
+    newPosts: 0,
+    newComments: 0,
+    duplicatesSkipped: 0
+  };
+
+  constructor(options: CollectionManagerOptions) {
+    this.platform = options.platform;
+    this.env = options.env;
+    this.spec = options.spec;
+    this.mode = options.mode || 'incremental';
+    
+    // Build human-readable collection ID
+    this.collectionId = buildCollectionId(options.spec);
+    
+    // Default base directory
+    const baseDir = options.baseDir || path.join(os.homedir(), '.webauto', 'download');
+    
+    this.storage = new CollectionStorage(baseDir, this.platform, this.env, this.collectionId);
+    this.bloomFilter = new BloomFilter(500000, 0.001); // 500k items, 0.1% false positive
+  }
+
+  /**
+   * Initialize collection manager
+   * - Load existing meta if available
+   * - Load existing bloom filter for incremental mode
+   * - Clear data for fresh mode
+   */
+  async init(): Promise<void> {
+    await this.storage.init();
+    
+    const existingMeta = await this.storage.readMeta();
+    
+    if (this.mode === 'fresh') {
+      // Clear all existing data
+      await this.storage.clear();
+      this.meta = this.createMeta();
+      await this.storage.writeMeta(this.meta);
+    } else if (existingMeta) {
+      // Incremental mode with existing data
+      this.meta = existingMeta;
+      this.meta.updatedAt = new Date().toISOString();
+      
+      // Load existing bloom filter state if available; fallback to rebuild from posts.
+      let loadedFromMeta = false;
+      if (existingMeta.bloomFilter) {
+        try {
+          this.bloomFilter = BloomFilter.import(existingMeta.bloomFilter, 500000);
+          loadedFromMeta = true;
+        } catch {
+          loadedFromMeta = false;
+        }
+      }
+
+      if (!loadedFromMeta) {
+        const posts = await this.storage.readPosts();
+        for (const post of posts) {
+          this.bloomFilter.add(post.id);
+        }
+      }
+      
+      await this.storage.writeMeta(this.meta);
+    } else {
+      // Incremental mode, but no existing data
+      this.meta = this.createMeta();
+      await this.storage.writeMeta(this.meta);
+    }
+    
+    // Initialize stats from existing data
+    const storageStats = await this.storage.getStats();
+    this.stats.totalPosts = storageStats.postCount;
+    this.stats.totalComments = storageStats.commentCount;
+  }
+
+  private createMeta(): CollectionMeta {
+    return {
+      platform: this.platform,
+      env: this.env,
+      collectionId: this.collectionId,
+      source: this.spec.source,
+      createdAt: new Date().toISOString(),
+      updatedAt: new Date().toISOString(),
+      totalPosts: 0,
+      totalComments: 0,
+      mode: this.mode,
+      ...(this.spec.source === 'search' && { keyword: (this.spec as { keyword: string }).keyword }),
+      ...(this.spec.source === 'user' && { 
+        userId: (this.spec as { userId: string }).userId,
+        userName: (this.spec as { userName?: string }).userName 
+      })
+    };
+  }
+
+  /**
+   * Check if post ID already exists (via bloom filter)
+   */
+  hasPost(postId: string): boolean {
+    return this.bloomFilter.mightContain(postId);
+  }
+
+  /**
+   * Add a post if not duplicate
+   * Returns true if added, false if duplicate
+   */
+  async addPost(post: PostRecord): Promise<boolean> {
+    // Check bloom filter
+    if (this.bloomFilter.mightContain(post.id)) {
+      this.stats.duplicatesSkipped++;
+      return false;
+    }
+    
+    // Auto-fill collectedAt fields if not provided
+    if (!post.collectedAt) {
+      const ts = getCurrentTimestamp();
+      post.collectedAt = ts.collectedAt;
+      post.collectedAtLocal = ts.collectedAtLocal;
+      post.collectedDate = ts.collectedDate;
+    }
+    
+    // Add to bloom filter and storage
+    this.bloomFilter.add(post.id);
+    await this.storage.appendPost(post);
+    
+    this.stats.totalPosts++;
+    this.stats.newPosts++;
+    
+    return true;
+  }
+
+  /**
+   * Add a comment (no deduplication for comments within same post)
+   * But we use postId:commentId as the key to dedupe
+   */
+  async addComment(comment: CommentRecord): Promise<boolean> {
+    const commentKey = `${comment.postId}:${comment.id}`;
+    
+    if (this.bloomFilter.mightContain(commentKey)) {
+      return false;
+    }
+    
+    this.bloomFilter.add(commentKey);
+    await this.storage.appendComment(comment);
+    
+    this.stats.totalComments++;
+    this.stats.newComments++;
+    
+    return true;
+  }
+
+  /**
+   * Persist metadata and bloom filter state
+   */
+  async persist(): Promise<void> {
+    if (!this.meta) return;
+    
+    this.meta.updatedAt = new Date().toISOString();
+    this.meta.totalPosts = this.stats.totalPosts;
+    this.meta.totalComments = this.stats.totalComments;
+    this.meta.bloomFilter = this.bloomFilter.export();
+    
+    await this.storage.writeMeta(this.meta);
+  }
+
+  /**
+   * Get current stats
+   */
+  getStats(): CollectionStats {
+    return { ...this.stats };
+  }
+
+  /**
+   * Get collection metadata
+   */
+  getMeta(): CollectionMeta | null {
+    return this.meta;
+  }
+
+  /**
+   * Get collection ID
+   */
+  getCollectionId(): string {
+    return this.collectionId;
+  }
+
+  /**
+   * Get storage paths for external use (e.g., logging)
+   */
+  getPaths(): {
+    collectionDir: string;
+    postsPath: string;
+    commentsPath: string;
+    linksPath: string;
+    runLogPath: string;
+  } {
+    return {
+      collectionDir: (this.storage as any).collectionDir,
+      postsPath: this.storage.getPostsPath(),
+      commentsPath: this.storage.getCommentsPath(),
+      linksPath: this.storage.getLinksPath(),
+      runLogPath: this.storage.getRunLogPath()
+    };
+  }
+
+  /**
+   * List all collections for this platform
+   */
+  static async listCollections(
+    platform: Platform, 
+    env: string, 
+    baseDir?: string
+  ): Promise<Array<{ collectionId: string; spec: CollectionIdSpec | null }>> {
+    const dir = baseDir || path.join(os.homedir(), '.webauto', 'download');
+    const ids = await CollectionStorage.listCollections(dir, platform, env);
+    
+    return ids.map(id => ({
+      collectionId: id,
+      spec: parseCollectionId(id)
+    }));
+  }
+
+  /**
+   * Merge multiple collections into one
+   * Useful for combining timeline data from multiple dates
+   */
+  static async mergeCollections(
+    sourceIds: string[],
+    targetId: string,
+    platform: Platform,
+    env: string,
+    baseDir?: string
+  ): Promise<void> {
+    const dir = baseDir || path.join(os.homedir(), '.webauto', 'download');
+    const targetStorage = new CollectionStorage(dir, platform, env, targetId);
+    await targetStorage.init();
+    
+    const seenPostIds = new Set<string>();
+    
+    for (const sourceId of sourceIds) {
+      const sourceStorage = new CollectionStorage(dir, platform, env, sourceId);
+      const posts = await sourceStorage.readPosts();
+      
+      for (const post of posts) {
+        if (!seenPostIds.has(post.id)) {
+          seenPostIds.add(post.id);
+          await targetStorage.appendPost(post);
+        }
+      }
+    }
+  }
+}
+
+export { buildCollectionId, parseCollectionId };
+export type { CollectionIdSpec, CollectionMeta, PostRecord, CommentRecord };
+
+export { getCurrentTimestamp, parsePlatformDate, extractWeiboPostDate } from "./date-utils.js";

package/modules/collection-manager/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "@webauto/collection-manager",
+  "version": "1.0.0",
+  "main": "index.ts",
+  "types": "index.ts",
+  "dependencies": {},
+  "devDependencies": {
+    "@types/node": "^20.0.0"
+  }
+}

package/modules/collection-manager/storage.ts

@@ -0,0 +1,174 @@
+/**
+ * File storage utilities for collection data
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import type { CollectionMeta, PostRecord, CommentRecord } from './types';
+
+export class CollectionStorage {
+  private baseDir: string;
+  private collectionDir: string;
+  private platform: string;
+  private env: string;
+  private collectionId: string;
+
+  constructor(baseDir: string, platform: string, env: string, collectionId: string) {
+    this.baseDir = baseDir;
+    this.platform = platform;
+    this.env = env;
+    this.collectionId = collectionId;
+    
+    // Directory structure: baseDir/platform/env/collectionId/
+    this.collectionDir = path.join(baseDir, platform, env, collectionId);
+  }
+
+  /**
+   * Initialize storage directories
+   */
+  async init(): Promise<void> {
+    await fs.promises.mkdir(this.collectionDir, { recursive: true });
+  }
+
+  /**
+   * Get paths for different files
+   */
+  getMetaPath(): string {
+    return path.join(this.collectionDir, 'collection-meta.json');
+  }
+
+  getPostsPath(): string {
+    return path.join(this.collectionDir, 'posts.jsonl');
+  }
+
+  getCommentsPath(): string {
+    return path.join(this.collectionDir, 'comments.jsonl');
+  }
+
+  getLinksPath(): string {
+    return path.join(this.collectionDir, 'links.jsonl');
+  }
+
+  getRunLogPath(): string {
+    return path.join(this.collectionDir, 'run.log');
+  }
+
+  getEventsPath(): string {
+    return path.join(this.collectionDir, 'run-events.jsonl');
+  }
+
+  /**
+   * Read/write meta
+   */
+  async readMeta(): Promise<CollectionMeta | null> {
+    const metaPath = this.getMetaPath();
+    try {
+      const data = await fs.promises.readFile(metaPath, 'utf-8');
+      return JSON.parse(data);
+    } catch {
+      return null;
+    }
+  }
+
+  async writeMeta(meta: CollectionMeta): Promise<void> {
+    const metaPath = this.getMetaPath();
+    await fs.promises.writeFile(metaPath, JSON.stringify(meta, null, 2), 'utf-8');
+  }
+
+  /**
+   * Append post to JSONL
+   */
+  async appendPost(post: PostRecord): Promise<void> {
+    const line = JSON.stringify(post) + '\n';
+    await fs.promises.appendFile(this.getPostsPath(), line, 'utf-8');
+  }
+
+  /**
+   * Append comment to JSONL
+   */
+  async appendComment(comment: CommentRecord): Promise<void> {
+    const line = JSON.stringify(comment) + '\n';
+    await fs.promises.appendFile(this.getCommentsPath(), line, 'utf-8');
+  }
+
+  /**
+   * Read all posts
+   */
+  async readPosts(): Promise<PostRecord[]> {
+    try {
+      const data = await fs.promises.readFile(this.getPostsPath(), 'utf-8');
+      return data.trim().split('\n').map(line => JSON.parse(line));
+    } catch {
+      return [];
+    }
+  }
+
+  /**
+   * Read all comments
+   */
+  async readComments(): Promise<CommentRecord[]> {
+    try {
+      const data = await fs.promises.readFile(this.getCommentsPath(), 'utf-8');
+      return data.trim().split('\n').map(line => JSON.parse(line));
+    } catch {
+      return [];
+    }
+  }
+
+  /**
+   * Count lines in JSONL file
+   */
+  async countLines(filePath: string): Promise<number> {
+    try {
+      const data = await fs.promises.readFile(filePath, 'utf-8');
+      return data.trim().split('\n').filter(line => line.length > 0).length;
+    } catch {
+      return 0;
+    }
+  }
+
+  /**
+   * Get collection stats
+   */
+  async getStats(): Promise<{ postCount: number; commentCount: number }> {
+    return {
+      postCount: await this.countLines(this.getPostsPath()),
+      commentCount: await this.countLines(this.getCommentsPath())
+    };
+  }
+
+  /**
+   * Clear all collection data (for fresh mode)
+   */
+  async clear(): Promise<void> {
+    const files = [
+      this.getPostsPath(),
+      this.getCommentsPath(),
+      this.getLinksPath(),
+      this.getEventsPath()
+    ];
+    
+    for (const file of files) {
+      try {
+        await fs.promises.unlink(file);
+      } catch {
+        // File might not exist
+      }
+    }
+  }
+
+  /**
+   * List all collections for a platform/env
+   */
+  static async listCollections(baseDir: string, platform: string, env: string): Promise<string[]> {
+    const dir = path.join(baseDir, platform, env);
+    try {
+      const entries = await fs.promises.readdir(dir, { withFileTypes: true });
+      return entries
+        .filter(e => e.isDirectory())
+        .map(e => e.name);
+    } catch {
+      return [];
+    }
+  }
+}

package/modules/collection-manager/types.ts

@@ -0,0 +1,156 @@
+/** Collection types for unified data management across platforms */
+
+export type Platform = 'xiaohongshu' | 'weibo' | '1688';
+
+export type CollectionSource = 'search' | 'timeline' | 'user' | 'note' | 'product';
+
+export type CollectionMode = 'fresh' | 'incremental';
+
+export interface CollectionMeta {
+  platform: Platform;
+  env: string;
+  /** Human-readable identifier - varies by source type */
+  collectionId: string;
+  /** Source type */
+  source: CollectionSource;
+  /** Creation timestamp */
+  createdAt: string;
+  /** Last update timestamp */
+  updatedAt: string;
+  /** Total posts collected */
+  totalPosts: number;
+  /** Total comments collected */
+  totalComments: number;
+  /** Collection mode */
+  mode: CollectionMode;
+  /** Bloom filter state (base64) */
+  bloomFilter?: string;
+  /** Optional: search keyword */
+  keyword?: string;
+  /** Optional: user ID for user monitoring */
+  userId?: string;
+  /** Optional: user name for display */
+  userName?: string;
+}
+
+export interface PostRecord {
+  /** Unique ID from platform */
+  id: string;
+  /** Post URL */
+  url: string;
+  /** Author ID */
+  authorId?: string;
+  /** Author name */
+  authorName?: string;
+  /** Post content */
+  content?: string;
+  /** Collected timestamp (ISO 8601 UTC) */
+  collectedAt: string;
+  /** Collected timestamp (local with timezone) */
+  collectedAtLocal?: string;
+  /** Collected date (YYYY-MM-DD, local) */
+  collectedDate?: string;
+  /** Published date extracted from content (YYYY-MM-DD) */
+  publishedDate?: string;
+  /** Published time extracted from content (HH:MM) */
+  publishedTime?: string;
+  /** Full published datetime text */
+  publishedAtText?: string;
+  /** Raw data from platform */
+  raw?: Record<string, unknown>;
+  /** Comments count */
+  commentsCount?: number;
+  /** Likes count */
+  likesCount?: number;
+}
+
+export interface CommentRecord {
+  /** Unique ID */
+  id: string;
+  /** Parent post ID */
+  postId: string;
+  /** Author ID */
+  authorId?: string;
+  /** Author name */
+  authorName?: string;
+  /** Comment content */
+  content?: string;
+  /** Parent comment ID (for nested comments) */
+  parentId?: string;
+  /** Reply to user name */
+  replyToName?: string;
+  /** Collected timestamp */
+  collectedAt: string;
+  /** Raw data */
+  raw?: Record<string, unknown>;
+}
+
+/**
+ * Collection ID naming conventions:
+ * 
+ * search: "search:<keyword>"           -> search:春晚
+ * timeline: "timeline:<date>"          -> timeline:2026-02-20
+ * user: "user:<userId>:<userName>"     -> user:1234567890:张三
+ * note: "note:<noteId>"                -> note:abc123
+ * product: "product:<keyword>"         -> product:女装
+ */
+export type CollectionIdSpec = {
+  source: 'search';
+  keyword: string;
+} | {
+  source: 'timeline';
+  date: string; // YYYY-MM-DD
+} | {
+  source: 'user';
+  userId: string;
+  userName?: string;
+} | {
+  source: 'note';
+  noteId: string;
+} | {
+  source: 'product';
+  keyword: string;
+};
+
+/**
+ * Build collection ID from spec
+ */
+export function buildCollectionId(spec: CollectionIdSpec): string {
+  switch (spec.source) {
+    case 'search':
+      return `search:${spec.keyword}`;
+    case 'timeline':
+      return `timeline:${spec.date}`;
+    case 'user':
+      const safeName = spec.userName?.replace(/[\/\\:*?"<>|]/g, '_') || 'unknown';
+      return `user:${spec.userId}:${safeName}`;
+    case 'note':
+      return `note:${spec.noteId}`;
+    case 'product':
+      return `product:${spec.keyword}`;
+  }
+}
+
+/**
+ * Parse collection ID back to spec
+ */
+export function parseCollectionId(id: string): CollectionIdSpec | null {
+  const parts = id.split(':');
+  if (parts.length < 2) return null;
+  
+  const source = parts[0] as CollectionSource;
+  switch (source) {
+    case 'search':
+      return { source: 'search', keyword: parts.slice(1).join(':') };
+    case 'timeline':
+      return { source: 'timeline', date: parts[1] };
+    case 'user':
+      return { source: 'user', userId: parts[1], userName: parts[2] };
+    case 'note':
+      return { source: 'note', noteId: parts[1] };
+    case 'product':
+      return { source: 'product', keyword: parts.slice(1).join(':') };
+    default:
+      return null;
+  }
+}

package/modules/logging/src/index.ts

@@ -53,7 +53,7 @@ export function logDebug(module: string, event: string, data: Record<string, any
     data,
   };
   try {
-    fs.appendFileSync(DEBUG_LOG_FILE, `${JSON.stringify(entry)}\n`);
+    fs.appendFileSync(DEBUG_LOG_FILE, `${JSON.stringify(entry)}\n`, 'utf8');
   } catch {
     // ignore
   }