bilisum 1.13.2 → 1.13.3

package/runtime/apps/service/src/video_sum_service/schemas.py

@@ -0,0 +1,418 @@
+from datetime import datetime, timezone
+from typing import Literal
+from uuid import uuid4
+
+from pydantic import BaseModel, Field
+
+from video_sum_core.models.tasks import MindMapNode, TaskInput, TaskMindMap, TaskResult, TaskStatus
+from video_sum_core.utils import extract_bilibili_page
+
+
+class TaskCreateRequest(TaskInput):
+    video_id: str | None = None
+
+
+class ResummaryRequest(BaseModel):
+    task_id: str | None = None
+    page_number: int | None = None
+
+
+class VideoTaskCreateRequest(BaseModel):
+    page_number: int | None = None
+
+
+class VideoTaskBatchRequest(BaseModel):
+    page_numbers: list[int] = Field(default_factory=list)
+    confirm: bool = False
+
+
+class AggregateSummaryRequest(BaseModel):
+    page_numbers: list[int] | None = None
+
+
+class TaskMarkdownExportRequest(BaseModel):
+    target: Literal["markdown", "obsidian"] = "obsidian"
+
+
+class VideoTaskBatchPageResponse(BaseModel):
+    page_number: int
+    page_title: str | None = None
+    action: str
+    reason: str | None = None
+    existing_task_id: str | None = None
+    existing_status: TaskStatus | None = None
+    has_existing_result: bool = False
+    task: "TaskDetailResponse | None" = None
+
+
+class VideoTaskBatchResponse(BaseModel):
+    operation: str
+    requested_page_numbers: list[int] = Field(default_factory=list)
+    requires_confirmation: bool = False
+    created_tasks: list["TaskDetailResponse"] = Field(default_factory=list)
+    skipped_pages: list[VideoTaskBatchPageResponse] = Field(default_factory=list)
+    conflict_pages: list[VideoTaskBatchPageResponse] = Field(default_factory=list)
+
+
+class VideoProbeRequest(BaseModel):
+    url: str
+    force_refresh: bool = False
+
+
+class VideoPageOptionResponse(BaseModel):
+    page: int
+    title: str
+    source_url: str
+    cover_url: str = ""
+    duration: float | None = None
+
+
+class VideoAssetSummaryResponse(BaseModel):
+    video_id: str
+    canonical_id: str
+    platform: str
+    title: str
+    source_url: str
+    cover_url: str = ""
+    duration: float | None = None
+    latest_task_id: str | None = None
+    latest_status: TaskStatus | None = None
+    latest_stage: str | None = None
+    has_result: bool = False
+    is_favorite: bool = False
+    favorite_updated_at: datetime | None = None
+    pages: list[VideoPageOptionResponse] = Field(default_factory=list)
+    created_at: datetime
+    updated_at: datetime
+
+
+class VideoAssetDetailResponse(VideoAssetSummaryResponse):
+    latest_result: TaskResult | None = None
+    latest_error_message: str | None = None
+
+
+class VideoAssetRecord(BaseModel):
+    video_id: str = Field(default_factory=lambda: uuid4().hex)
+    canonical_id: str
+    platform: str = "bilibili"
+    title: str
+    source_url: str
+    cover_url: str = ""
+    duration: float | None = None
+    latest_task_id: str | None = None
+    latest_status: TaskStatus | None = None
+    latest_stage: str | None = None
+    latest_result: TaskResult | None = None
+    latest_error_message: str | None = None
+    is_favorite: bool = False
+    favorite_updated_at: datetime | None = None
+    pages: list[VideoPageOptionResponse] = Field(default_factory=list)
+    created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+
+    def to_summary(self) -> VideoAssetSummaryResponse:
+        return VideoAssetSummaryResponse(
+            video_id=self.video_id,
+            canonical_id=self.canonical_id,
+            platform=self.platform,
+            title=self.title,
+            source_url=self.source_url,
+            cover_url=self.cover_url,
+            duration=self.duration,
+            latest_task_id=self.latest_task_id,
+            latest_status=self.latest_status,
+            latest_stage=self.latest_stage,
+            has_result=self.latest_result is not None,
+            is_favorite=self.is_favorite,
+            favorite_updated_at=self.favorite_updated_at,
+            pages=self.pages,
+            created_at=self.created_at,
+            updated_at=self.updated_at,
+        )
+
+    def to_detail(self) -> VideoAssetDetailResponse:
+        return VideoAssetDetailResponse(
+            **self.to_summary().model_dump(),
+            latest_result=self.latest_result,
+            latest_error_message=self.latest_error_message,
+        )
+
+
+class VideoProbeResponse(BaseModel):
+    video: VideoAssetSummaryResponse
+    cached: bool = False
+    requires_selection: bool = False
+    pages: list[VideoPageOptionResponse] = Field(default_factory=list)
+
+
+class TaskSummaryResponse(BaseModel):
+    task_id: str
+    video_id: str | None = None
+    status: TaskStatus
+    input_type: str
+    source: str
+    title: str | None = None
+    page_number: int | None = None
+    page_title: str | None = None
+    created_at: datetime
+    updated_at: datetime
+    llm_total_tokens: int | None = None
+    task_duration_seconds: float | None = None
+
+
+class TaskDetailResponse(TaskSummaryResponse):
+    result: TaskResult | None = None
+    error_code: str | None = None
+    error_message: str | None = None
+
+
+class TaskMindMapResponse(BaseModel):
+    task_id: str
+    status: str = "idle"
+    error_message: str | None = None
+    updated_at: datetime | None = None
+    mindmap: TaskMindMap | None = None
+
+
+class TaskMarkdownExportResponse(BaseModel):
+    task_id: str
+    target_format: Literal["markdown", "obsidian"]
+    path: str
+    directory: str
+    file_name: str
+    overwritten: bool = False
+    artifact_key: str
+
+
+class TaskEventResponse(BaseModel):
+    event_id: str
+    task_id: str
+    stage: str
+    progress: int
+    message: str
+    created_at: datetime
+    payload: dict[str, object] = Field(default_factory=dict)
+
+
+class TaskProgressResponse(BaseModel):
+    task_id: str
+    status: TaskStatus
+    progress: int = 0
+    latest_stage: str | None = None
+    latest_message: str | None = None
+    updated_at: datetime
+
+
+class TaskRecord(BaseModel):
+    task_id: str = Field(default_factory=lambda: uuid4().hex)
+    video_id: str | None = None
+    status: TaskStatus = TaskStatus.QUEUED
+    task_input: TaskInput
+    page_number: int | None = None
+    page_title: str | None = None
+    created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    result: TaskResult | None = None
+    error_code: str | None = None
+    error_message: str | None = None
+
+    @property
+    def task_duration_seconds(self) -> float | None:
+        if not self.created_at or not self.updated_at:
+            return None
+        return max(0.0, (self.updated_at - self.created_at).total_seconds())
+
+    def to_summary(self) -> TaskSummaryResponse:
+        page_number = self.page_number if self.page_number is not None else extract_bilibili_page(self.task_input.source)
+        return TaskSummaryResponse(
+            task_id=self.task_id,
+            video_id=self.video_id,
+            status=self.status,
+            input_type=self.task_input.input_type.value,
+            source=self.task_input.source,
+            title=self.task_input.title,
+            page_number=page_number,
+            page_title=self.page_title or self.task_input.title,
+            created_at=self.created_at,
+            updated_at=self.updated_at,
+            llm_total_tokens=self.result.llm_total_tokens if self.result else None,
+            task_duration_seconds=self.task_duration_seconds,
+        )
+
+    def to_detail(self) -> TaskDetailResponse:
+        return TaskDetailResponse(
+            **self.to_summary().model_dump(),
+            result=self.result,
+            error_code=self.error_code,
+            error_message=self.error_message,
+        )
+
+
+class TaskEventRecord(BaseModel):
+    event_id: str = Field(default_factory=lambda: uuid4().hex)
+    task_id: str
+    stage: str
+    progress: int = 0
+    message: str
+    created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    payload: dict[str, object] = Field(default_factory=dict)
+
+    def to_response(self) -> TaskEventResponse:
+        return TaskEventResponse(
+            event_id=self.event_id,
+            task_id=self.task_id,
+            stage=self.stage,
+            progress=self.progress,
+            message=self.message,
+            created_at=self.created_at,
+            payload=self.payload,
+        )
+
+
+class VideoTagRecord(BaseModel):
+    video_id: str
+    tag: str
+    source: str = "manual"
+    confidence: float = 1.0
+    created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+
+
+class KnowledgeIndexChunkRecord(BaseModel):
+    chunk_id: str
+    video_id: str
+    embedding_json: str
+    indexed_content: str
+    index_type: str = "video_summary"
+    segment_order: int | None = None
+    anchor_label: str | None = None
+    anchor_seconds: float | None = None
+    created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+
+
+class TagItem(BaseModel):
+    tag: str
+    count: int
+    videos: list[str] = Field(default_factory=list)
+
+
+class TagListResponse(BaseModel):
+    items: list[TagItem] = Field(default_factory=list)
+
+
+class VideoTagListResponse(BaseModel):
+    video_id: str
+    items: list[VideoTagRecord] = Field(default_factory=list)
+
+
+class KnowledgeTagCreateRequest(BaseModel):
+    video_id: str
+    tag: str
+
+
+class KnowledgeAutoTagRequest(BaseModel):
+    video_ids: list[str] | None = None
+
+
+class KnowledgeAutoTagVideoResult(BaseModel):
+    video_id: str
+    tags: list[str] = Field(default_factory=list)
+
+
+class KnowledgeAutoTagResponse(BaseModel):
+    items: list[KnowledgeAutoTagVideoResult] = Field(default_factory=list)
+
+
+class KnowledgeNetworkNode(BaseModel):
+    id: str
+    label: str
+    type: str
+    count: int | None = None
+    tags: list[str] = Field(default_factory=list)
+    degree: int | None = None
+    focus: bool = False
+    hidden_count: int = 0
+    video_count: int = 0
+
+
+class KnowledgeNetworkLink(BaseModel):
+    source: str
+    target: str
+    weight: float = 1.0
+    kind: str = "cooccurrence"
+
+
+class KnowledgeNetworkResponse(BaseModel):
+    nodes: list[KnowledgeNetworkNode] = Field(default_factory=list)
+    links: list[KnowledgeNetworkLink] = Field(default_factory=list)
+    mode: str = "overview"
+    hidden_tag_count: int = 0
+    selected_tags: list[str] = Field(default_factory=list)
+
+
+class KnowledgeSearchFilters(BaseModel):
+    tags: list[str] = Field(default_factory=list)
+
+
+class KnowledgeSearchRequest(BaseModel):
+    query: str
+    limit: int = 10
+    filters: KnowledgeSearchFilters | None = None
+
+
+class KnowledgeSearchResult(BaseModel):
+    video_id: str
+    title: str
+    relevance_score: float
+    snippet: str
+    tags: list[str] = Field(default_factory=list)
+    cover_url: str = ""
+    timestamp: str | None = None
+    video_title: str | None = None
+    page_title: str | None = None
+    page_number: int | None = None
+
+
+class KnowledgeSearchResponse(BaseModel):
+    query: str
+    results: list[KnowledgeSearchResult] = Field(default_factory=list)
+    total: int = 0
+
+
+class KnowledgeChatHistoryItem(BaseModel):
+    role: str
+    content: str
+
+
+class KnowledgeAskRequest(BaseModel):
+    query: str
+    context_limit: int = 5
+    history: list[KnowledgeChatHistoryItem] = Field(default_factory=list)
+
+
+class KnowledgeSourceRef(BaseModel):
+    video_id: str
+    title: str
+    relevance_score: float
+    timestamp: str | None = None
+    video_title: str | None = None
+    page_title: str | None = None
+    page_number: int | None = None
+
+
+class KnowledgeAskResponse(BaseModel):
+    query: str
+    answer: str
+    sources: list[KnowledgeSourceRef] = Field(default_factory=list)
+
+
+class KnowledgeStatsResponse(BaseModel):
+    video_count: int
+    indexed_chunk_count: int
+    tag_count: int
+    untagged_video_count: int
+    knowledge_llm_available: bool
+
+
+class KnowledgeRebuildResponse(BaseModel):
+    indexed_videos: int

package/runtime/apps/service/src/video_sum_service/settings_manager.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from pydantic import BaseModel
+
+from video_sum_infra.config import (
+    DEFAULT_SUMMARY_SYSTEM_PROMPT,
+    DEFAULT_SUMMARY_USER_PROMPT_TEMPLATE,
+    LEGACY_SUMMARY_SYSTEM_PROMPT,
+    LEGACY_SUMMARY_USER_PROMPT_TEMPLATE,
+    PREVIOUS_DEFAULT_SUMMARY_SYSTEM_PROMPT,
+    PREVIOUS_DEFAULT_SUMMARY_USER_PROMPT_TEMPLATE,
+    ServiceSettings,
+    recommend_mindmap_concurrency,
+    recommend_task_concurrency,
+)
+
+
+class SettingsUpdatePayload(BaseModel):
+    host: str | None = None
+    port: int | None = None
+    data_dir: str | None = None
+    cache_dir: str | None = None
+    tasks_dir: str | None = None
+    database_url: str | None = None
+    whisper_model: str | None = None
+    whisper_device: str | None = None
+    whisper_compute_type: str | None = None
+    transcription_provider: str | None = None
+    device_preference: str | None = None
+    compute_type: str | None = None
+    model_mode: str | None = None
+    fixed_model: str | None = None
+    siliconflow_asr_base_url: str | None = None
+    siliconflow_asr_model: str | None = None
+    siliconflow_asr_api_key: str | None = None
+    cuda_variant: str | None = None
+    runtime_channel: str | None = None
+    output_dir: str | None = None
+    preserve_temp_audio: bool | None = None
+    enable_cache: bool | None = None
+    language: str | None = None
+    summary_mode: str | None = None
+    llm_enabled: bool | None = None
+    auto_generate_mindmap: bool | None = None
+    llm_provider: str | None = None
+    llm_base_url: str | None = None
+    llm_model: str | None = None
+    llm_api_key: str | None = None
+    knowledge_llm_mode: str | None = None
+    knowledge_llm_enabled: bool | None = None
+    knowledge_llm_base_url: str | None = None
+    knowledge_llm_model: str | None = None
+    knowledge_llm_api_key: str | None = None
+    knowledge_index_auto_rebuild: str | None = None
+    knowledge_enabled: bool | None = None
+    summary_system_prompt: str | None = None
+    summary_user_prompt_template: str | None = None
+    summary_chunk_target_chars: int | None = None
+    summary_chunk_overlap_segments: int | None = None
+    task_concurrency: int | None = None
+    mindmap_concurrency: int | None = None
+    summary_chunk_concurrency: int | None = None
+    summary_chunk_retry_count: int | None = None
+    ytdlp_cookies_file: str | None = None
+    ytdlp_cookies_browser: str | None = None
+
+
+class SettingsManager:
+    def __init__(self, base_settings: ServiceSettings) -> None:
+        self._base_settings = base_settings
+        self._settings = base_settings
+        self._settings_path = Path(base_settings.data_dir) / "settings.json"
+
+    @property
+    def current(self) -> ServiceSettings:
+        return self._settings
+
+    @property
+    def has_persisted_settings(self) -> bool:
+        return self._settings_path.exists()
+
+    def load(self) -> ServiceSettings:
+        if self._settings_path.exists():
+            stored = json.loads(self._settings_path.read_text(encoding="utf-8"))
+            if stored.get("summary_system_prompt") == LEGACY_SUMMARY_SYSTEM_PROMPT:
+                stored["summary_system_prompt"] = DEFAULT_SUMMARY_SYSTEM_PROMPT
+            if stored.get("summary_system_prompt") == PREVIOUS_DEFAULT_SUMMARY_SYSTEM_PROMPT:
+                stored["summary_system_prompt"] = DEFAULT_SUMMARY_SYSTEM_PROMPT
+            if stored.get("summary_user_prompt_template") == LEGACY_SUMMARY_USER_PROMPT_TEMPLATE:
+                stored["summary_user_prompt_template"] = DEFAULT_SUMMARY_USER_PROMPT_TEMPLATE
+            if stored.get("summary_user_prompt_template") == PREVIOUS_DEFAULT_SUMMARY_USER_PROMPT_TEMPLATE:
+                stored["summary_user_prompt_template"] = DEFAULT_SUMMARY_USER_PROMPT_TEMPLATE
+            migrated = False
+            candidate = ServiceSettings.model_validate({**self._base_settings.model_dump(), **stored})
+            if "task_concurrency" not in stored:
+                stored["task_concurrency"] = recommend_task_concurrency(candidate)
+                migrated = True
+            if "mindmap_concurrency" not in stored:
+                stored["mindmap_concurrency"] = recommend_mindmap_concurrency()
+                migrated = True
+            self._settings = ServiceSettings.model_validate({**self._base_settings.model_dump(), **stored})
+            if migrated:
+                self._settings_path.parent.mkdir(parents=True, exist_ok=True)
+                self._settings_path.write_text(
+                    json.dumps(self._settings.model_dump(mode="json"), ensure_ascii=False, indent=2),
+                    encoding="utf-8",
+                )
+        else:
+            self._settings = self._base_settings.model_copy(
+                update={
+                    "task_concurrency": recommend_task_concurrency(self._base_settings),
+                    "mindmap_concurrency": recommend_mindmap_concurrency(),
+                }
+            )
+        return self._settings
+
+    def save(self, payload: SettingsUpdatePayload) -> ServiceSettings:
+        current_dump = self._settings.model_dump(mode="json")
+        updates = payload.model_dump(exclude_none=True)
+        next_settings = ServiceSettings.model_validate({**current_dump, **updates})
+        self._settings_path.parent.mkdir(parents=True, exist_ok=True)
+        self._settings_path.write_text(
+            json.dumps(next_settings.model_dump(mode="json"), ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+        self._settings = next_settings
+        return self._settings

package/runtime/apps/service/src/video_sum_service/task_artifacts.py

@@ -0,0 +1,102 @@
+import json
+import logging
+import shutil
+from pathlib import Path
+
+from fastapi import HTTPException
+
+from video_sum_core.models.tasks import TaskMindMap
+from video_sum_infra.config import ServiceSettings
+
+from video_sum_service.schemas import TaskRecord, VideoAssetRecord
+
+logger = logging.getLogger("video_sum_service.app")
+
+
+def task_artifact_directories(tasks: list[TaskRecord], tasks_dir: Path) -> list[Path]:
+    directories: set[Path] = set()
+    for task in tasks:
+        directories.add(tasks_dir / task.task_id)
+        if task.result is None:
+            continue
+        for artifact_path in task.result.artifacts.values():
+            try:
+                path = Path(str(artifact_path))
+            except (TypeError, ValueError):
+                continue
+            directories.add(path.parent)
+    return sorted(directories, key=lambda item: len(item.parts), reverse=True)
+
+
+def video_cover_paths(video: VideoAssetRecord, cache_dir: Path) -> list[Path]:
+    cover_paths: set[Path] = set()
+    cover_url = str(video.cover_url or "")
+    if cover_url.startswith("/media/covers/"):
+        cover_paths.add(cache_dir / "covers" / Path(cover_url).name)
+    return sorted(cover_paths)
+
+
+def video_source_paths(video: VideoAssetRecord, cache_dir: Path) -> list[Path]:
+    source_paths: set[Path] = set()
+    if str(video.platform or "").lower() != "local":
+        return []
+    try:
+        source_path = Path(str(video.source_url or "")).resolve()
+        uploads_dir = (cache_dir / "uploads").resolve()
+        if uploads_dir in source_path.parents:
+            source_paths.add(source_path)
+    except (OSError, RuntimeError, ValueError):
+        return []
+    return sorted(source_paths)
+
+
+def cleanup_video_files(video: VideoAssetRecord, tasks: list[TaskRecord], current_settings: ServiceSettings) -> None:
+    for directory in task_artifact_directories(tasks, current_settings.tasks_dir):
+        try:
+            if directory.exists():
+                shutil.rmtree(directory, ignore_errors=False)
+        except OSError:
+            logger.warning("failed to remove task directory: %s", directory, exc_info=True)
+
+    for cover_path in video_cover_paths(video, current_settings.cache_dir):
+        try:
+            if cover_path.exists():
+                cover_path.unlink()
+        except OSError:
+            logger.warning("failed to remove cached cover: %s", cover_path, exc_info=True)
+
+    for source_path in video_source_paths(video, current_settings.cache_dir):
+        try:
+            if source_path.exists():
+                source_path.unlink()
+        except OSError:
+            logger.warning("failed to remove cached local media source: %s", source_path, exc_info=True)
+
+
+def cleanup_task_files(task: TaskRecord, current_settings: ServiceSettings) -> None:
+    for directory in task_artifact_directories([task], current_settings.tasks_dir):
+        try:
+            if directory.exists():
+                shutil.rmtree(directory, ignore_errors=False)
+        except OSError:
+            logger.warning("failed to remove task directory: %s", directory, exc_info=True)
+
+
+def load_task_segments(summary_path: str) -> list[dict[str, object]]:
+    try:
+        payload = json.loads(Path(summary_path).read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise HTTPException(status_code=500, detail="无法读取当前任务的分段结果。") from exc
+
+    segments = payload.get("segments")
+    if not isinstance(segments, list) or not segments:
+        raise HTTPException(status_code=400, detail="当前任务缺少可复用的分段数据。")
+    return segments
+
+
+def load_task_mindmap(mindmap_path: str) -> TaskMindMap:
+    try:
+        payload = json.loads(Path(mindmap_path).read_text(encoding="utf-8"))
+        return TaskMindMap.model_validate(payload)
+    except (OSError, json.JSONDecodeError, ValueError) as exc:
+        raise HTTPException(status_code=500, detail="无法读取当前任务的思维导图结果。") from exc