@skills-store/rednote 0.1.12 → 0.1.14

package/dist/rednote/persistence.js

@@ -0,0 +1,578 @@
+import path from 'node:path';
+import { createClient } from '@libsql/client';
+import { nanoid } from 'nanoid';
+import { DataSource, EntitySchema, In } from 'typeorm';
+import { REDNOTE_DATABASE_PATH, ensureDir } from '../utils/browser-core.js';
+const rednotePostSchema = new EntitySchema({
+    name: 'RednotePostRecord',
+    tableName: 'rednote_posts',
+    columns: {
+        id: {
+            type: String,
+            primary: true,
+            length: 16
+        },
+        noteId: {
+            name: 'noteid',
+            type: String
+        },
+        title: {
+            type: String,
+            nullable: true
+        },
+        url: {
+            type: String,
+            nullable: true
+        },
+        image: {
+            type: String,
+            nullable: true
+        },
+        likeCount: {
+            name: 'like_count',
+            type: String,
+            nullable: true
+        },
+        commentCount: {
+            name: 'comment_count',
+            type: String,
+            nullable: true
+        },
+        collectedCount: {
+            name: 'collected_count',
+            type: String,
+            nullable: true
+        },
+        sharedCount: {
+            name: 'shared_count',
+            type: String,
+            nullable: true
+        },
+        authorId: {
+            name: 'author_id',
+            type: String,
+            nullable: true
+        },
+        authorNickname: {
+            name: 'author_nickname',
+            type: String,
+            nullable: true
+        },
+        modelType: {
+            name: 'model_type',
+            type: String,
+            nullable: true
+        },
+        xsecToken: {
+            name: 'xsec_token',
+            type: String,
+            nullable: true
+        },
+        instanceName: {
+            name: 'instance_name',
+            type: String
+        },
+        raw: {
+            type: 'simple-json',
+            nullable: true
+        },
+        createdAt: {
+            name: 'created_at',
+            type: Date,
+            createDate: true
+        },
+        updatedAt: {
+            name: 'updated_at',
+            type: Date,
+            updateDate: true
+        }
+    },
+    indices: [
+        {
+            name: 'IDX_rednote_posts_note_instance',
+            columns: [
+                'noteId',
+                'instanceName'
+            ],
+            unique: true
+        },
+        {
+            name: 'IDX_rednote_posts_instance',
+            columns: [
+                'instanceName'
+            ]
+        }
+    ]
+});
+const rednotePostDetailSchema = new EntitySchema({
+    name: 'RednotePostDetailRecord',
+    tableName: 'rednote_post_details',
+    columns: {
+        id: {
+            type: String,
+            primary: true,
+            length: 16
+        },
+        noteId: {
+            name: 'noteid',
+            type: String
+        },
+        title: {
+            type: String,
+            nullable: true
+        },
+        content: {
+            type: 'text',
+            nullable: true
+        },
+        tags: {
+            type: 'simple-json',
+            nullable: true
+        },
+        imageList: {
+            name: 'image_list',
+            type: 'simple-json',
+            nullable: true
+        },
+        videoUrl: {
+            name: 'video_url',
+            type: String,
+            nullable: true
+        },
+        likeCount: {
+            name: 'like_count',
+            type: String,
+            nullable: true
+        },
+        commentCount: {
+            name: 'comment_count',
+            type: String,
+            nullable: true
+        },
+        collectedCount: {
+            name: 'collected_count',
+            type: String,
+            nullable: true
+        },
+        shareCount: {
+            name: 'share_count',
+            type: String,
+            nullable: true
+        },
+        authorId: {
+            name: 'author_id',
+            type: String,
+            nullable: true
+        },
+        authorNickname: {
+            name: 'author_nickname',
+            type: String,
+            nullable: true
+        },
+        noteType: {
+            name: 'note_type',
+            type: String,
+            nullable: true
+        },
+        instanceName: {
+            name: 'instance_name',
+            type: String
+        },
+        raw: {
+            type: 'simple-json',
+            nullable: true
+        },
+        createdAt: {
+            name: 'created_at',
+            type: Date,
+            createDate: true
+        },
+        updatedAt: {
+            name: 'updated_at',
+            type: Date,
+            updateDate: true
+        }
+    },
+    indices: [
+        {
+            name: 'IDX_rednote_post_details_note_instance',
+            columns: [
+                'noteId',
+                'instanceName'
+            ],
+            unique: true
+        },
+        {
+            name: 'IDX_rednote_post_details_instance',
+            columns: [
+                'instanceName'
+            ]
+        }
+    ]
+});
+const rednotePostCommentSchema = new EntitySchema({
+    name: 'RednotePostCommentRecord',
+    tableName: 'rednote_post_comments',
+    columns: {
+        id: {
+            type: String,
+            primary: true,
+            length: 16
+        },
+        commentId: {
+            name: 'commentid',
+            type: String
+        },
+        noteId: {
+            name: 'noteid',
+            type: String
+        },
+        content: {
+            type: 'text',
+            nullable: true
+        },
+        likeCount: {
+            name: 'like_count',
+            type: String,
+            nullable: true
+        },
+        replyCount: {
+            name: 'reply_count',
+            type: String,
+            nullable: true
+        },
+        authorId: {
+            name: 'author_id',
+            type: String,
+            nullable: true
+        },
+        authorNickname: {
+            name: 'author_nickname',
+            type: String,
+            nullable: true
+        },
+        instanceName: {
+            name: 'instance_name',
+            type: String
+        },
+        raw: {
+            type: 'simple-json',
+            nullable: true
+        },
+        createdAt: {
+            name: 'created_at',
+            type: Date,
+            createDate: true
+        },
+        updatedAt: {
+            name: 'updated_at',
+            type: Date,
+            updateDate: true
+        }
+    },
+    indices: [
+        {
+            name: 'IDX_rednote_post_comments_unique',
+            columns: [
+                'commentId',
+                'noteId',
+                'instanceName'
+            ],
+            unique: true
+        },
+        {
+            name: 'IDX_rednote_post_comments_note_instance',
+            columns: [
+                'noteId',
+                'instanceName'
+            ]
+        }
+    ]
+});
+let dataSourcePromise = null;
+function createRecordId() {
+    return nanoid(16);
+}
+function uniqueStrings(values) {
+    return [
+        ...new Set(values.filter((value)=>typeof value === 'string' && value.length > 0))
+    ];
+}
+function firstNonEmpty(...values) {
+    return values.find((value)=>typeof value === 'string' && value.length > 0) ?? null;
+}
+function toCountString(value) {
+    if (value === null || value === undefined || value === '') {
+        return null;
+    }
+    return String(value);
+}
+function coalesceValue(nextValue, fallbackValue) {
+    return nextValue ?? fallbackValue ?? null;
+}
+function extractPrimaryImage(post) {
+    const cover = post.noteCard.cover;
+    return firstNonEmpty(cover.urlDefault, cover.urlPre, cover.url, ...post.noteCard.imageList.flatMap((image)=>image.infoList.map((info)=>info.url)));
+}
+function extractNoteIdFromUrl(url) {
+    try {
+        const parsed = new URL(url);
+        const segments = parsed.pathname.split('/').filter(Boolean);
+        return segments.at(-1) ?? null;
+    } catch  {
+        return null;
+    }
+}
+function extractAuthorFromRawNote(rawNote) {
+    if (!rawNote || typeof rawNote !== 'object' || Array.isArray(rawNote)) {
+        return {
+            userId: null,
+            nickname: null
+        };
+    }
+    const note = rawNote;
+    const user = note.user ?? note.userInfo ?? note.user_info;
+    return {
+        userId: user?.userId ?? user?.user_id ?? null,
+        nickname: user?.nickname ?? user?.nickName ?? user?.nick_name ?? null
+    };
+}
+function extractCommentId(comment) {
+    const value = comment?.id ?? comment?.commentId ?? comment?.comment_id;
+    if (typeof value === 'string' || typeof value === 'number') {
+        return String(value);
+    }
+    const fallbackUserId = comment?.userInfo?.userId ?? comment?.user_info?.user_id ?? 'unknown';
+    const fallbackCreateTime = comment?.createTime ?? comment?.create_time ?? 'unknown';
+    const fallbackContent = comment?.content ?? '';
+    return `${fallbackUserId}:${fallbackCreateTime}:${fallbackContent}`;
+}
+async function initializeDataSource() {
+    ensureDir(path.dirname(REDNOTE_DATABASE_PATH));
+    const client = createClient({
+        url: `file:${REDNOTE_DATABASE_PATH}`
+    });
+    await client.execute('SELECT 1');
+    client.close();
+    const dataSource = new DataSource({
+        type: 'better-sqlite3',
+        database: REDNOTE_DATABASE_PATH,
+        entities: [
+            rednotePostSchema,
+            rednotePostDetailSchema,
+            rednotePostCommentSchema
+        ],
+        synchronize: true,
+        logging: false,
+        prepareDatabase: (database)=>{
+            database.pragma('journal_mode = WAL');
+            database.pragma('foreign_keys = ON');
+        }
+    });
+    return await dataSource.initialize();
+}
+export async function initializeRednoteDatabase() {
+    if (!dataSourcePromise) {
+        dataSourcePromise = initializeDataSource().catch((error)=>{
+            dataSourcePromise = null;
+            throw error;
+        });
+    }
+    return await dataSourcePromise;
+}
+async function persistPosts(instanceName, inputs) {
+    if (inputs.length === 0) {
+        return;
+    }
+    const dataSource = await initializeRednoteDatabase();
+    const repository = dataSource.getRepository(rednotePostSchema);
+    const noteIds = uniqueStrings(inputs.map((input)=>input.post.id));
+    const existingRows = noteIds.length > 0 ? await repository.find({
+        where: {
+            instanceName,
+            noteId: In(noteIds)
+        }
+    }) : [];
+    const existingMap = new Map(existingRows.map((row)=>[
+            row.noteId,
+            row
+        ]));
+    const entities = inputs.map(({ post, raw })=>{
+        const existing = existingMap.get(post.id);
+        const image = extractPrimaryImage(post);
+        const authorNickname = firstNonEmpty(post.noteCard.user.nickname, post.noteCard.user.nickName);
+        return repository.create({
+            id: existing?.id ?? createRecordId(),
+            noteId: post.id,
+            title: coalesceValue(post.noteCard.displayTitle, existing?.title),
+            url: coalesceValue(post.url, existing?.url),
+            image: coalesceValue(image, existing?.image),
+            likeCount: coalesceValue(toCountString(post.noteCard.interactInfo.likedCount), existing?.likeCount),
+            commentCount: coalesceValue(toCountString(post.noteCard.interactInfo.commentCount), existing?.commentCount),
+            collectedCount: coalesceValue(toCountString(post.noteCard.interactInfo.collectedCount), existing?.collectedCount),
+            sharedCount: coalesceValue(toCountString(post.noteCard.interactInfo.sharedCount), existing?.sharedCount),
+            authorId: coalesceValue(post.noteCard.user.userId, existing?.authorId),
+            authorNickname: coalesceValue(authorNickname, existing?.authorNickname),
+            modelType: coalesceValue(post.modelType, existing?.modelType),
+            xsecToken: coalesceValue(post.xsecToken, existing?.xsecToken),
+            instanceName,
+            raw,
+            ...existing?.createdAt ? {
+                createdAt: existing.createdAt
+            } : {}
+        });
+    });
+    await repository.save(entities);
+}
+export async function persistHomePosts(instanceName, inputs) {
+    await persistPosts(instanceName, inputs);
+}
+export async function persistSearchPosts(instanceName, inputs) {
+    await persistPosts(instanceName, inputs);
+}
+export async function persistFeedDetail(input) {
+    const noteId = input.note.noteId ?? extractNoteIdFromUrl(input.url);
+    if (!noteId) {
+        return;
+    }
+    const dataSource = await initializeRednoteDatabase();
+    await dataSource.transaction(async (manager)=>{
+        const postRepository = manager.getRepository(rednotePostSchema);
+        const detailRepository = manager.getRepository(rednotePostDetailSchema);
+        const commentRepository = manager.getRepository(rednotePostCommentSchema);
+        const [existingPost, existingDetail] = await Promise.all([
+            postRepository.findOne({
+                where: {
+                    instanceName: input.instanceName,
+                    noteId
+                }
+            }),
+            detailRepository.findOne({
+                where: {
+                    instanceName: input.instanceName,
+                    noteId
+                }
+            })
+        ]);
+        const author = extractAuthorFromRawNote(input.rawNote);
+        await postRepository.save(postRepository.create({
+            id: existingPost?.id ?? createRecordId(),
+            noteId,
+            title: coalesceValue(input.note.title, existingPost?.title),
+            url: coalesceValue(input.url, existingPost?.url),
+            image: coalesceValue(input.note.imageList[0] ?? null, existingPost?.image),
+            likeCount: coalesceValue(toCountString(input.note.likedCount), existingPost?.likeCount),
+            commentCount: coalesceValue(toCountString(input.note.commentCount), existingPost?.commentCount),
+            collectedCount: coalesceValue(toCountString(input.note.collectedCount), existingPost?.collectedCount),
+            sharedCount: coalesceValue(toCountString(input.note.shareCount), existingPost?.sharedCount),
+            authorId: coalesceValue(author.userId, existingPost?.authorId),
+            authorNickname: coalesceValue(author.nickname, existingPost?.authorNickname),
+            modelType: coalesceValue(input.note.type, existingPost?.modelType),
+            xsecToken: coalesceValue((()=>{
+                try {
+                    return new URL(input.url).searchParams.get('xsec_token');
+                } catch  {
+                    return null;
+                }
+            })(), existingPost?.xsecToken),
+            instanceName: input.instanceName,
+            raw: input.rawNote,
+            ...existingPost?.createdAt ? {
+                createdAt: existingPost.createdAt
+            } : {}
+        }));
+        await detailRepository.save(detailRepository.create({
+            id: existingDetail?.id ?? createRecordId(),
+            noteId,
+            title: coalesceValue(input.note.title, existingDetail?.title),
+            content: coalesceValue(input.note.desc, existingDetail?.content),
+            tags: input.note.tagList.length > 0 ? input.note.tagList : existingDetail?.tags ?? [],
+            imageList: input.note.imageList.length > 0 ? input.note.imageList : existingDetail?.imageList ?? [],
+            videoUrl: coalesceValue(input.note.video, existingDetail?.videoUrl),
+            likeCount: coalesceValue(toCountString(input.note.likedCount), existingDetail?.likeCount),
+            commentCount: coalesceValue(toCountString(input.note.commentCount), existingDetail?.commentCount),
+            collectedCount: coalesceValue(toCountString(input.note.collectedCount), existingDetail?.collectedCount),
+            shareCount: coalesceValue(toCountString(input.note.shareCount), existingDetail?.shareCount),
+            authorId: coalesceValue(author.userId, existingDetail?.authorId),
+            authorNickname: coalesceValue(author.nickname, existingDetail?.authorNickname),
+            noteType: coalesceValue(input.note.type, existingDetail?.noteType),
+            instanceName: input.instanceName,
+            raw: input.rawNote,
+            ...existingDetail?.createdAt ? {
+                createdAt: existingDetail.createdAt
+            } : {}
+        }));
+        const rawComments = Array.isArray(input.rawComments) ? input.rawComments : [];
+        if (rawComments.length === 0) {
+            return;
+        }
+        const commentIds = rawComments.map(extractCommentId);
+        const existingComments = await commentRepository.find({
+            where: {
+                instanceName: input.instanceName,
+                noteId,
+                commentId: In(commentIds)
+            }
+        });
+        const existingCommentMap = new Map(existingComments.map((comment)=>[
+                comment.commentId,
+                comment
+            ]));
+        const commentEntities = rawComments.map((comment)=>{
+            const commentId = extractCommentId(comment);
+            const existing = existingCommentMap.get(commentId);
+            return commentRepository.create({
+                id: existing?.id ?? createRecordId(),
+                commentId,
+                noteId,
+                content: coalesceValue(comment?.content ?? null, existing?.content),
+                likeCount: coalesceValue(toCountString(comment?.likeCount ?? comment?.like_count ?? comment?.interactInfo?.likedCount), existing?.likeCount),
+                replyCount: coalesceValue(toCountString(comment?.subCommentCount ?? comment?.sub_comment_count), existing?.replyCount),
+                authorId: coalesceValue(comment?.userInfo?.userId ?? comment?.user_info?.user_id ?? null, existing?.authorId),
+                authorNickname: coalesceValue(comment?.userInfo?.nickname ?? comment?.user_info?.nickname ?? null, existing?.authorNickname),
+                instanceName: input.instanceName,
+                raw: comment,
+                ...existing?.createdAt ? {
+                    createdAt: existing.createdAt
+                } : {}
+            });
+        });
+        await commentRepository.save(commentEntities);
+    });
+}
+export async function listPersistedPostSummaries(instanceName, noteIds) {
+    const uniqueNoteIds = uniqueStrings(noteIds);
+    if (uniqueNoteIds.length === 0) {
+        return [];
+    }
+    const dataSource = await initializeRednoteDatabase();
+    const repository = dataSource.getRepository(rednotePostSchema);
+    const rows = await repository.find({
+        where: {
+            instanceName,
+            noteId: In(uniqueNoteIds)
+        }
+    });
+    const rowMap = new Map(rows.map((row)=>[
+            row.noteId,
+            row
+        ]));
+    return uniqueNoteIds.map((noteId)=>rowMap.get(noteId)).filter((row)=>Boolean(row)).map((row)=>({
+            id: row.id,
+            noteId: row.noteId,
+            title: row.title,
+            likeCount: row.likeCount,
+            url: row.url
+        }));
+}
+export async function findPersistedPostUrlByRecordId(instanceName, id) {
+    const dataSource = await initializeRednoteDatabase();
+    const repository = dataSource.getRepository(rednotePostSchema);
+    const row = await repository.findOne({
+        where: {
+            instanceName,
+            id
+        }
+    });
+    return row?.url ?? null;
+}

package/dist/rednote/search.js

@@ -1,8 +1,10 @@
 #!/usr/bin/env node
 import { runCli } from '../utils/browser-cli.js';
+import { simulateMousePresence } from '../utils/mouse-helper.js';
 import { resolveStatusTarget } from './status.js';
 import { createRednoteSession, disconnectRednoteSession, ensureRednoteLoggedIn } from './checkLogin.js';
-import { ensureJsonSavePath, parseOutputCliArgs, renderJsonSaveSummary, renderPostsMarkdown, resolveJsonSavePath, resolveSavePath, writeJsonFile, writePostsJsonl } from './output-format.js';
+import { initializeRednoteDatabase, listPersistedPostSummaries, persistSearchPosts } from './persistence.js';
+import { ensureJsonSavePath, parseOutputCliArgs, renderPostSummaryList, resolveJsonSavePath, resolveSavePath, writeJsonFile, writePostsJsonl } from './output-format.js';
 export function parseSearchCliArgs(argv) {
     return parseOutputCliArgs(argv, {
         includeKeyword: true
@@ -32,11 +34,12 @@ function normalizeSearchPost(item) {
     const imageList = Array.isArray(noteCard.image_list) ? noteCard.image_list : [];
     const cornerTagInfo = Array.isArray(noteCard.corner_tag_info) ? noteCard.corner_tag_info : [];
     const xsecToken = item.xsec_token ?? null;
+    const url = xsecToken ? `https://www.xiaohongshu.com/explore/${item.id}?xsec_token=${xsecToken}` : `https://www.xiaohongshu.com/explore/${item.id}`;
     return {
         id: item.id,
         modelType: item.model_type,
         xsecToken,
-        url: xsecToken ? `https://www.xiaohongshu.com/explore/${item.id}?xsec_token=${xsecToken}` : `https://www.xiaohongshu.com/explore/${item.id}`,
+        url,
         noteCard: {
             type: noteCard.type ?? null,
             displayTitle: noteCard.display_title ?? null,
@@ -84,6 +87,20 @@ function normalizeSearchPost(item) {
         }
     };
 }
+function buildPostSummaryList(posts, persistedRows = []) {
+    const persistedMap = new Map(persistedRows.map((row)=>[
+            row.noteId,
+            row
+        ]));
+    return posts.map((post)=>{
+        const persisted = persistedMap.get(post.id);
+        return {
+            id: persisted?.id ?? post.id,
+            title: persisted?.title ?? post.noteCard.displayTitle ?? '',
+            like: persisted?.likeCount ?? post.noteCard.interactInfo.likedCount ?? ''
+        };
+    });
+}
 function toSearchJsonPost(post) {
     return {
         id: post.id,
@@ -155,16 +172,29 @@ async function collectSearchItems(page, keyword) {
         });
     }
     const searchInput = page.locator('#search-input');
+    await simulateMousePresence(page, {
+        locator: searchInput
+    });
     await searchInput.focus();
     await searchInput.fill(keyword);
     await page.keyboard.press('Enter');
     await page.waitForTimeout(500);
-    return await searchPromise;
+    const searchItems = await searchPromise;
+    await simulateMousePresence(page);
+    return searchItems;
 }
-export async function searchRednotePosts(session, keyword) {
+export async function searchRednotePosts(session, keyword, instanceName) {
     const page = await getOrCreateXiaohongshuPage(session);
     const items = await collectSearchItems(page, keyword);
     const posts = items.map(normalizeSearchPost);
+    let summaries = buildPostSummaryList(posts);
+    if (instanceName) {
+        await persistSearchPosts(instanceName, posts.map((post, index)=>({
+                post,
+                raw: items[index] ?? post
+            })));
+        summaries = buildPostSummaryList(posts, await listPersistedPostSummaries(instanceName, posts.map((post)=>post.id)));
+    }
     return {
         ok: true,
         search: {
@@ -172,7 +202,8 @@ export async function searchRednotePosts(session, keyword) {
             pageUrl: page.url(),
             fetchedAt: new Date().toISOString(),
             total: posts.length,
-            posts
+            posts,
+            summaries
         }
     };
 }
@@ -181,21 +212,16 @@ function writeSearchOutput(result, values) {
         const savedPath = resolveJsonSavePath(values.savePath);
         const posts = result.search.posts.map(toSearchJsonPost);
         writeJsonFile(posts, savedPath);
-        process.stdout.write(renderJsonSaveSummary(savedPath, posts));
+        process.stdout.write(renderPostSummaryList(result.search.summaries));
         return;
     }
     const posts = result.search.posts;
-    let savedPath;
     if (values.saveRequested) {
-        savedPath = resolveSavePath('search', values.savePath, result.search.keyword);
+        const savedPath = resolveSavePath('search', values.savePath, result.search.keyword);
         writePostsJsonl(posts, savedPath);
         result.search.savedPath = savedPath;
     }
-    let markdown = renderPostsMarkdown(posts);
-    if (savedPath) {
-        markdown = `Saved JSONL: ${savedPath}\n\n${markdown}`;
-    }
-    process.stdout.write(markdown);
+    process.stdout.write(renderPostSummaryList(result.search.summaries));
 }
 export async function runSearchCommand(values = {
     format: 'md',
@@ -210,11 +236,12 @@ export async function runSearchCommand(values = {
     if (!keyword) {
         throw new Error('Missing required option: --keyword');
     }
+    await initializeRednoteDatabase();
     const target = resolveStatusTarget(values.instance);
     const session = await createRednoteSession(target);
     try {
         await ensureRednoteLoggedIn(target, 'search', session);
-        const result = await searchRednotePosts(session, keyword);
+        const result = await searchRednotePosts(session, keyword, target.instanceName);
         writeSearchOutput(result, values);
     } finally{
         await disconnectRednoteSession(session);