@web-auto/webauto 0.1.4 → 0.1.6

package/dist/modules/collection-manager/bloom-filter.js

@@ -0,0 +1,91 @@
+"use strict";
+/**
+ * Simple Bloom Filter for memory-efficient deduplication
+ * Uses non-cryptographic hash functions for speed
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BloomFilter = void 0;
+class BloomFilter {
+    bitmap;
+    size;
+    hashCount;
+    count = 0;
+    constructor(expectedItems = 100000, falsePositiveRate = 0.01) {
+        // Calculate optimal size and hash count
+        this.size = Math.ceil(-expectedItems * Math.log(falsePositiveRate) / Math.pow(Math.LN2, 2));
+        this.hashCount = Math.ceil((this.size / expectedItems) * Math.LN2);
+        this.bitmap = new Uint8Array(Math.ceil(this.size / 8));
+    }
+    hash(str, seed) {
+        // FNV-1a variant with seed
+        let hash = 2166136261 ^ seed;
+        for (let i = 0; i < str.length; i++) {
+            hash ^= str.charCodeAt(i);
+            hash = Math.imul(hash, 16777619);
+        }
+        return Math.abs(hash) % this.size;
+    }
+    add(item) {
+        for (let i = 0; i < this.hashCount; i++) {
+            const bit = this.hash(item, i);
+            const byteIndex = Math.floor(bit / 8);
+            const bitIndex = bit % 8;
+            this.bitmap[byteIndex] |= (1 << bitIndex);
+        }
+        this.count++;
+    }
+    mightContain(item) {
+        for (let i = 0; i < this.hashCount; i++) {
+            const bit = this.hash(item, i);
+            const byteIndex = Math.floor(bit / 8);
+            const bitIndex = bit % 8;
+            if ((this.bitmap[byteIndex] & (1 << bitIndex)) === 0) {
+                return false;
+            }
+        }
+        return true;
+    }
+    getCount() {
+        return this.count;
+    }
+    /**
+     * Export to base64 string for persistence
+     */
+    export() {
+        const buffer = Buffer.from(this.bitmap);
+        return buffer.toString('base64');
+    }
+    /**
+     * Import from base64 string
+     */
+    static import(base64, expectedItems = 100000) {
+        const filter = new BloomFilter(expectedItems);
+        const buffer = Buffer.from(base64, 'base64');
+        filter.bitmap = new Uint8Array(buffer);
+        return filter;
+    }
+    /**
+     * Serialize to JSON
+     */
+    toJSON() {
+        return {
+            bitmap: this.export(),
+            size: this.size,
+            hashCount: this.hashCount,
+            count: this.count
+        };
+    }
+    /**
+     * Deserialize from JSON
+     */
+    static fromJSON(json) {
+        const filter = new BloomFilter(1000); // Dummy, will be overwritten
+        filter.bitmap = new Uint8Array(Buffer.from(json.bitmap, 'base64'));
+        filter.size = json.size;
+        filter.hashCount = json.hashCount;
+        filter.count = json.count;
+        return filter;
+    }
+}
+exports.BloomFilter = BloomFilter;
+//# sourceMappingURL=bloom-filter.js.map

package/dist/modules/collection-manager/date-utils.js

@@ -0,0 +1,275 @@
+"use strict";
+/**
+ * Date extraction utilities for social media posts
+ * Handles various time formats from platforms like Weibo, Xiaohongshu
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parsePlatformDate = parsePlatformDate;
+exports.getCurrentTimestamp = getCurrentTimestamp;
+exports.extractWeiboPostDate = extractWeiboPostDate;
+/**
+ * Parse relative time strings to absolute date
+ *
+ * Supported formats:
+ * - "刚刚" / "刚刚来自..." → now
+ * - "5分钟前" / "30秒前" → relative to now
+ * - "今天 08:30" / "今天08:30" → today
+ * - "昨天 14:20" / "昨天14:20" → yesterday
+ * - "前天 10:00" → 2 days ago
+ * - "01-15" / "1月15日" → this year
+ * - "2025-12-01" / "2025年12月01日" → exact date
+ * - "12-01 15:30" → this year with time
+ */
+function parsePlatformDate(text, options = {}) {
+    const { now = new Date(), timezone = 'Asia/Shanghai' } = options;
+    const trimmed = text.trim();
+    if (!trimmed)
+        return null;
+    // Get current date in specified timezone
+    const currentDate = new Date(now.toLocaleString('en-US', { timeZone: timezone }));
+    const currentYear = currentDate.getFullYear();
+    // "刚刚" / "刚刚来自..."
+    if (trimmed.includes('刚刚')) {
+        return {
+            date: formatDate(currentDate),
+            time: formatTime(currentDate),
+            fullText: formatDateTime(currentDate)
+        };
+    }
+    // "X分钟前" / "X秒前" / "X小时前"
+    const relativeMatch = trimmed.match(/(\d+)\s*(秒|分钟|小时)前/);
+    if (relativeMatch) {
+        const amount = parseInt(relativeMatch[1], 10);
+        const unit = relativeMatch[2];
+        const result = new Date(currentDate);
+        if (unit === '秒')
+            result.setSeconds(result.getSeconds() - amount);
+        else if (unit === '分钟')
+            result.setMinutes(result.getMinutes() - amount);
+        else if (unit === '小时')
+            result.setHours(result.getHours() - amount);
+        return {
+            date: formatDate(result),
+            time: formatTime(result),
+            fullText: formatDateTime(result)
+        };
+    }
+    // "今天 08:30" / "今天08:30"
+    const todayMatch = trimmed.match(/今天\s*(\d{1,2}):(\d{2})/);
+    if (todayMatch) {
+        const hour = parseInt(todayMatch[1], 10);
+        const minute = parseInt(todayMatch[2], 10);
+        const result = new Date(currentDate);
+        result.setHours(hour, minute, 0, 0);
+        return {
+            date: formatDate(result),
+            time: formatTime(result),
+            fullText: formatDateTime(result)
+        };
+    }
+    // "昨天 14:20" / "昨天14:20"
+    const yesterdayMatch = trimmed.match(/昨天\s*(\d{1,2}):(\d{2})/);
+    if (yesterdayMatch) {
+        const hour = parseInt(yesterdayMatch[1], 10);
+        const minute = parseInt(yesterdayMatch[2], 10);
+        const result = new Date(currentDate);
+        result.setDate(result.getDate() - 1);
+        result.setHours(hour, minute, 0, 0);
+        return {
+            date: formatDate(result),
+            time: formatTime(result),
+            fullText: formatDateTime(result)
+        };
+    }
+    // "前天 10:00"
+    const dayBeforeYesterdayMatch = trimmed.match(/前天\s*(\d{1,2}):(\d{2})/);
+    if (dayBeforeYesterdayMatch) {
+        const hour = parseInt(dayBeforeYesterdayMatch[1], 10);
+        const minute = parseInt(dayBeforeYesterdayMatch[2], 10);
+        const result = new Date(currentDate);
+        result.setDate(result.getDate() - 2);
+        result.setHours(hour, minute, 0, 0);
+        return {
+            date: formatDate(result),
+            time: formatTime(result),
+            fullText: formatDateTime(result)
+        };
+    }
+    // "2天前" / "3天前"
+    const daysAgoMatch = trimmed.match(/(\d+)\s*天前/);
+    if (daysAgoMatch) {
+        const days = parseInt(daysAgoMatch[1], 10);
+        const result = new Date(currentDate);
+        result.setDate(result.getDate() - days);
+        return {
+            date: formatDate(result),
+            time: '',
+            fullText: formatDate(result)
+        };
+    }
+    // "2025-12-01" / "2025年12月01日" - Full date (must match before MM-DD)
+    const fullDateMatch = trimmed.match(/(\d{4})[-年](\d{1,2})[-月](\d{1,2})/);
+    if (fullDateMatch) {
+        const year = parseInt(fullDateMatch[1], 10);
+        const month = parseInt(fullDateMatch[2], 10);
+        const day = parseInt(fullDateMatch[3], 10);
+        const result = new Date(year, month - 1, day);
+        // Check if there's time info
+        const timeMatch = trimmed.match(/(\d{1,2}):(\d{2})/);
+        if (timeMatch) {
+            result.setHours(parseInt(timeMatch[1], 10), parseInt(timeMatch[2], 10), 0, 0);
+            if (result.getTime() > currentDate.getTime()) {
+                result.setFullYear(currentYear - 1);
+            }
+            return {
+                date: formatDate(result),
+                time: formatTime(result),
+                fullText: formatDateTime(result)
+            };
+        }
+        return {
+            date: formatDate(result),
+            time: '',
+            fullText: formatDate(result)
+        };
+    }
+    // "01-15" / "1月15日" (this year, fallback to previous year when parsed date is in future)
+    const monthDayMatch = trimmed.match(/(\d{1,2})[-月](\d{1,2})日?/);
+    if (monthDayMatch && !trimmed.includes('年')) {
+        const month = parseInt(monthDayMatch[1], 10);
+        const day = parseInt(monthDayMatch[2], 10);
+        const result = new Date(currentYear, month - 1, day);
+        // Check if there's time info
+        const timeMatch = trimmed.match(/(\d{1,2}):(\d{2})/);
+        if (timeMatch) {
+            result.setHours(parseInt(timeMatch[1], 10), parseInt(timeMatch[2], 10), 0, 0);
+            if (result.getTime() > currentDate.getTime()) {
+                result.setFullYear(currentYear - 1);
+            }
+            return {
+                date: formatDate(result),
+                time: formatTime(result),
+                fullText: formatDateTime(result)
+            };
+        }
+        if (result.getTime() > currentDate.getTime()) {
+            result.setFullYear(currentYear - 1);
+        }
+        return {
+            date: formatDate(result),
+            time: '',
+            fullText: formatDate(result)
+        };
+    }
+    return null;
+}
+/**
+ * Format date as YYYY-MM-DD
+ */
+function formatDate(date) {
+    const year = date.getFullYear();
+    const month = String(date.getMonth() + 1).padStart(2, '0');
+    const day = String(date.getDate()).padStart(2, '0');
+    return `${year}-${month}-${day}`;
+}
+/**
+ * Format time as HH:MM
+ */
+function formatTime(date) {
+    const hour = String(date.getHours()).padStart(2, '0');
+    const minute = String(date.getMinutes()).padStart(2, '0');
+    return `${hour}:${minute}`;
+}
+/**
+ * Format datetime as YYYY-MM-DD HH:MM
+ */
+function formatDateTime(date) {
+    return `${formatDate(date)} ${formatTime(date)}`;
+}
+/**
+ * Get current timestamp in ISO format with timezone
+ */
+function getCurrentTimestamp(timezone = 'Asia/Shanghai') {
+    const now = new Date();
+    // UTC ISO string
+    const collectedAt = now.toISOString();
+    // Local time with timezone
+    const formatter = new Intl.DateTimeFormat('en-US', {
+        timeZone: timezone,
+        year: 'numeric',
+        month: '2-digit',
+        day: '2-digit',
+        hour: '2-digit',
+        minute: '2-digit',
+        second: '2-digit',
+        fractionalSecondDigits: 3,
+        hour12: false
+    });
+    const parts = formatter.formatToParts(now);
+    const get = (type) => parts.find(p => p.type === type)?.value || '00';
+    const year = get('year');
+    const month = get('month');
+    const day = get('day');
+    const hour = get('hour');
+    const minute = get('minute');
+    const second = get('second');
+    const ms = get('fractionalSecond');
+    // Get timezone offset
+    const tzOffset = new Intl.DateTimeFormat('en-US', {
+        timeZone: timezone,
+        timeZoneName: 'shortOffset'
+    }).format(now).split(' ').pop() || '+08:00';
+    const collectedAtLocal = `${year}-${month}-${day} ${hour}:${minute}:${second}.${ms} ${tzOffset}`;
+    const collectedDate = `${year}-${month}-${day}`;
+    return {
+        collectedAt,
+        collectedAtLocal,
+        collectedDate
+    };
+}
+/**
+ * Weibo-specific date extraction from post element
+ */
+function extractWeiboPostDate(postElement, now = new Date()) {
+    // Weibo post time is usually in:
+    // - <a class="head-info_time_..."> or similar
+    // - Element with from, time info
+    const timeSelectors = [
+        'a[class*="time"]',
+        'a[class*="date"]',
+        'span[class*="time"]',
+        '.from a',
+        'a[href*="weibo.com"]'
+    ];
+    for (const selector of timeSelectors) {
+        const timeEl = postElement.querySelector(selector);
+        if (timeEl) {
+            const text = timeEl.textContent?.trim();
+            if (text) {
+                const parsed = parsePlatformDate(text, { now });
+                if (parsed)
+                    return parsed;
+            }
+        }
+    }
+    // Fallback: search all text content for date patterns
+    const allText = postElement.textContent || '';
+    const datePatterns = [
+        /刚刚/,
+        /\d+\s*(秒|分钟|小时)前/,
+        /今天\s*\d{1,2}:\d{2}/,
+        /昨天\s*\d{1,2}:\d{2}/,
+        /\d{1,2}-\d{1,2}/,
+        /\d{4}-\d{1,2}-\d{1,2}/
+    ];
+    for (const pattern of datePatterns) {
+        const match = allText.match(pattern);
+        if (match) {
+            const parsed = parsePlatformDate(match[0], { now });
+            if (parsed)
+                return parsed;
+        }
+    }
+    return null;
+}
+//# sourceMappingURL=date-utils.js.map

package/dist/modules/collection-manager/index.js

@@ -0,0 +1,258 @@
+"use strict";
+/**
+ * Collection Data Manager
+ *
+ * Unified data management for all platforms and collection types.
+ * Handles:
+ * - Deduplication via Bloom Filter
+ * - Fresh/Incremental modes
+ * - File storage with human-readable paths
+ * - Stats and persistence
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseCollectionId = exports.buildCollectionId = exports.CollectionDataManager = void 0;
+const os = __importStar(require("os"));
+const path = __importStar(require("path"));
+const bloom_filter_1 = require("./bloom-filter");
+const storage_1 = require("./storage");
+const types_1 = require("./types");
+Object.defineProperty(exports, "buildCollectionId", { enumerable: true, get: function () { return types_1.buildCollectionId; } });
+Object.defineProperty(exports, "parseCollectionId", { enumerable: true, get: function () { return types_1.parseCollectionId; } });
+class CollectionDataManager {
+    platform;
+    env;
+    spec;
+    mode;
+    collectionId;
+    storage;
+    bloomFilter;
+    meta = null;
+    stats = {
+        totalPosts: 0,
+        totalComments: 0,
+        newPosts: 0,
+        newComments: 0,
+        duplicatesSkipped: 0
+    };
+    constructor(options) {
+        this.platform = options.platform;
+        this.env = options.env;
+        this.spec = options.spec;
+        this.mode = options.mode || 'incremental';
+        // Build human-readable collection ID
+        this.collectionId = (0, types_1.buildCollectionId)(options.spec);
+        // Default base directory
+        const baseDir = options.baseDir || path.join(os.homedir(), '.webauto', 'download');
+        this.storage = new storage_1.CollectionStorage(baseDir, this.platform, this.env, this.collectionId);
+        this.bloomFilter = new bloom_filter_1.BloomFilter(500000, 0.001); // 500k items, 0.1% false positive
+    }
+    /**
+     * Initialize collection manager
+     * - Load existing meta if available
+     * - Load existing bloom filter for incremental mode
+     * - Clear data for fresh mode
+     */
+    async init() {
+        await this.storage.init();
+        const existingMeta = await this.storage.readMeta();
+        if (this.mode === 'fresh') {
+            // Clear all existing data
+            await this.storage.clear();
+            this.meta = this.createMeta();
+            await this.storage.writeMeta(this.meta);
+        }
+        else if (existingMeta) {
+            // Incremental mode with existing data
+            this.meta = existingMeta;
+            this.meta.updatedAt = new Date().toISOString();
+            // Load existing posts into bloom filter
+            const posts = await this.storage.readPosts();
+            for (const post of posts) {
+                this.bloomFilter.add(post.id);
+            }
+            // Load existing bloom filter state if available
+            if (existingMeta.bloomFilter) {
+                try {
+                    this.bloomFilter = bloom_filter_1.BloomFilter.import(existingMeta.bloomFilter, 500000);
+                }
+                catch {
+                    // Ignore if bloom filter is corrupted
+                }
+            }
+            await this.storage.writeMeta(this.meta);
+        }
+        else {
+            // Incremental mode, but no existing data
+            this.meta = this.createMeta();
+            await this.storage.writeMeta(this.meta);
+        }
+        // Initialize stats from existing data
+        const storageStats = await this.storage.getStats();
+        this.stats.totalPosts = storageStats.postCount;
+        this.stats.totalComments = storageStats.commentCount;
+    }
+    createMeta() {
+        return {
+            platform: this.platform,
+            env: this.env,
+            collectionId: this.collectionId,
+            source: this.spec.source,
+            createdAt: new Date().toISOString(),
+            updatedAt: new Date().toISOString(),
+            totalPosts: 0,
+            totalComments: 0,
+            mode: this.mode,
+            ...(this.spec.source === 'search' && { keyword: this.spec.keyword }),
+            ...(this.spec.source === 'user' && {
+                userId: this.spec.userId,
+                userName: this.spec.userName
+            })
+        };
+    }
+    /**
+     * Check if post ID already exists (via bloom filter)
+     */
+    hasPost(postId) {
+        return this.bloomFilter.mightContain(postId);
+    }
+    /**
+     * Add a post if not duplicate
+     * Returns true if added, false if duplicate
+     */
+    async addPost(post) {
+        // Check bloom filter
+        if (this.bloomFilter.mightContain(post.id)) {
+            this.stats.duplicatesSkipped++;
+            return false;
+        }
+        // Add to bloom filter and storage
+        this.bloomFilter.add(post.id);
+        await this.storage.appendPost(post);
+        this.stats.totalPosts++;
+        this.stats.newPosts++;
+        return true;
+    }
+    /**
+     * Add a comment (no deduplication for comments within same post)
+     * But we use postId:commentId as the key to dedupe
+     */
+    async addComment(comment) {
+        const commentKey = `${comment.postId}:${comment.id}`;
+        if (this.bloomFilter.mightContain(commentKey)) {
+            return false;
+        }
+        this.bloomFilter.add(commentKey);
+        await this.storage.appendComment(comment);
+        this.stats.totalComments++;
+        this.stats.newComments++;
+        return true;
+    }
+    /**
+     * Persist metadata and bloom filter state
+     */
+    async persist() {
+        if (!this.meta)
+            return;
+        this.meta.updatedAt = new Date().toISOString();
+        this.meta.totalPosts = this.stats.totalPosts;
+        this.meta.totalComments = this.stats.totalComments;
+        this.meta.bloomFilter = this.bloomFilter.export();
+        await this.storage.writeMeta(this.meta);
+    }
+    /**
+     * Get current stats
+     */
+    getStats() {
+        return { ...this.stats };
+    }
+    /**
+     * Get collection metadata
+     */
+    getMeta() {
+        return this.meta;
+    }
+    /**
+     * Get collection ID
+     */
+    getCollectionId() {
+        return this.collectionId;
+    }
+    /**
+     * Get storage paths for external use (e.g., logging)
+     */
+    getPaths() {
+        return {
+            collectionDir: this.storage.collectionDir,
+            postsPath: this.storage.getPostsPath(),
+            commentsPath: this.storage.getCommentsPath(),
+            linksPath: this.storage.getLinksPath(),
+            runLogPath: this.storage.getRunLogPath()
+        };
+    }
+    /**
+     * List all collections for this platform
+     */
+    static async listCollections(platform, env, baseDir) {
+        const dir = baseDir || path.join(os.homedir(), '.webauto', 'download');
+        const ids = await storage_1.CollectionStorage.listCollections(dir, platform, env);
+        return ids.map(id => ({
+            collectionId: id,
+            spec: (0, types_1.parseCollectionId)(id)
+        }));
+    }
+    /**
+     * Merge multiple collections into one
+     * Useful for combining timeline data from multiple dates
+     */
+    static async mergeCollections(sourceIds, targetId, platform, env, baseDir) {
+        const dir = baseDir || path.join(os.homedir(), '.webauto', 'download');
+        const targetStorage = new storage_1.CollectionStorage(dir, platform, env, targetId);
+        await targetStorage.init();
+        const seenPostIds = new Set();
+        for (const sourceId of sourceIds) {
+            const sourceStorage = new storage_1.CollectionStorage(dir, platform, env, sourceId);
+            const posts = await sourceStorage.readPosts();
+            for (const post of posts) {
+                if (!seenPostIds.has(post.id)) {
+                    seenPostIds.add(post.id);
+                    await targetStorage.appendPost(post);
+                }
+            }
+        }
+    }
+}
+exports.CollectionDataManager = CollectionDataManager;
+//# sourceMappingURL=index.js.map