bilisum 1.20.0-beta.1 → 1.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bilisum",
3
- "version": "1.20.0-beta.1",
3
+ "version": "1.20.0",
4
4
  "description": "BiliSum CLI: control the installed BiliSum app, or run standalone with its own runtime.",
5
5
  "bin": {
6
6
  "bilisum": "bin/bilisum.js"
package/runtime/VERSION CHANGED
@@ -1 +1 @@
1
- 1.20.0-beta.1
1
+ 1.20.0
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "video-sum-service"
7
- version = "1.20.0-beta.1"
7
+ version = "1.20.0"
8
8
  description = "Local backend service for the video summarizer."
9
9
  requires-python = ">=3.12"
10
10
  dependencies = [
@@ -27,6 +27,7 @@ from video_sum_service import cli_idle
27
27
  from video_sum_service.context import CACHE_STATIC_DIR, WEB_STATIC_DIR, access_token_manager, app_info, logger, settings_manager
28
28
  from video_sum_service.integrations import probe_asr_connection, probe_llm_connection
29
29
  from video_sum_service.repository import SqliteTaskRepository
30
+ from video_sum_service.knowledge.job_coordinator import KnowledgeJobCoordinator
30
31
  from video_sum_service.routers.system import router as system_router
31
32
  from video_sum_service.routers.knowledge import router as knowledge_router
32
33
  from video_sum_service.routers.tasks import router as tasks_router
@@ -113,6 +114,9 @@ async def lifespan(app: FastAPI):
113
114
  app.state.task_repository = repository
114
115
  app.state.db_connection = connection
115
116
  app.state.settings_manager = settings_manager
117
+ knowledge_job_coordinator = KnowledgeJobCoordinator(repository, current_settings)
118
+ knowledge_job_coordinator.start()
119
+ app.state.knowledge_job_coordinator = knowledge_job_coordinator
116
120
  initialize_runtime_startup_state(app.state, current_settings)
117
121
  cli_idle.start_watchdog(repository)
118
122
  start_runtime_startup(app.state, repository, current_settings, recover_incomplete_tasks)
@@ -121,6 +125,7 @@ async def lifespan(app: FastAPI):
121
125
  yield
122
126
  finally:
123
127
  request_runtime_startup_shutdown(app.state)
128
+ knowledge_job_coordinator.stop()
124
129
  startup_thread = getattr(app.state, "runtime_startup_thread", None)
125
130
  if startup_thread is not None and startup_thread.is_alive():
126
131
  startup_thread.join(timeout=10)
@@ -0,0 +1,408 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import logging
5
+ import re
6
+ from collections.abc import Callable, Iterator
7
+
8
+ from fastapi import HTTPException
9
+
10
+ from video_sum_core.models.tasks import InputType, TaskInput, TaskStatus
11
+ from video_sum_core.utils import normalize_video_url
12
+ from video_sum_infra.config import ServiceSettings
13
+ from video_sum_service.knowledge.local_llm import chat_knowledge_llm, knowledge_llm_available, parse_json_payload
14
+ from video_sum_service.knowledge.rag_service import RagService
15
+ from video_sum_service.repository import SqliteTaskRepository
16
+ from video_sum_service.runtime_startup import submit_task_or_queue
17
+ from video_sum_service.schemas import (
18
+ KnowledgeConversationMessagesResponse,
19
+ KnowledgeChatHistoryItem,
20
+ KnowledgeConversationResponse,
21
+ KnowledgeJobItemResponse,
22
+ KnowledgeJobResponse,
23
+ KnowledgeMessageResponse,
24
+ KnowledgeReference,
25
+ KnowledgeReferencesResponse,
26
+ KnowledgeSuggestion,
27
+ KnowledgeSuggestionsResponse,
28
+ )
29
+ from video_sum_service.video_assets import probe_video_asset
30
+
31
+ _VIDEO_URL_RE = re.compile(r"https?://[^\s<>\u3001\u3002\uff0c\uff1b\uff09]+", re.IGNORECASE)
32
+ _MAX_TASK_LINKS = 20
33
+ _MAX_REASONING = 4000
34
+ logger = logging.getLogger("video_sum_service.knowledge.conversation")
35
+
36
+
37
+ def _model_or_404(repository: SqliteTaskRepository, conversation_id: str) -> KnowledgeConversationResponse:
38
+ payload = repository.get_knowledge_conversation(conversation_id)
39
+ if payload is None:
40
+ raise HTTPException(status_code=404, detail="会话不存在。")
41
+ return KnowledgeConversationResponse.model_validate(payload)
42
+
43
+
44
+ def list_conversations(repository: SqliteTaskRepository, limit: int = 50) -> list[KnowledgeConversationResponse]:
45
+ return [KnowledgeConversationResponse.model_validate(item) for item in repository.list_knowledge_conversations(limit)]
46
+
47
+
48
+ def get_conversation(repository: SqliteTaskRepository, conversation_id: str) -> KnowledgeConversationResponse:
49
+ return _model_or_404(repository, conversation_id)
50
+
51
+
52
+ def get_messages(repository: SqliteTaskRepository, conversation_id: str) -> KnowledgeConversationMessagesResponse:
53
+ _model_or_404(repository, conversation_id)
54
+ return KnowledgeConversationMessagesResponse(
55
+ conversation_id=conversation_id,
56
+ messages=[KnowledgeMessageResponse.model_validate(item) for item in repository.list_knowledge_messages(conversation_id)],
57
+ )
58
+
59
+
60
+ def _job_item(item: dict[str, object]) -> KnowledgeJobItemResponse:
61
+ raw_status = str(item.get("status") or TaskStatus.FAILED.value)
62
+ try:
63
+ status = TaskStatus(raw_status)
64
+ except ValueError:
65
+ status = TaskStatus.FAILED
66
+ title = ""
67
+ try:
68
+ task_input = json.loads(str(item.get("task_input_json") or "{}"))
69
+ title = str(task_input.get("title") or task_input.get("source") or "")
70
+ except (TypeError, ValueError, AttributeError):
71
+ title = str(item.get("task_id") or "")
72
+ error_message = item.get("error_message") or item.get("task_error_message")
73
+ progress = max(0, min(100, int(item.get("progress") or 0)))
74
+ if status in {TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED}:
75
+ progress = 100
76
+ return KnowledgeJobItemResponse(
77
+ task_id=str(item.get("task_id") or ""),
78
+ video_id=str(item["video_id"]) if item.get("video_id") else None,
79
+ title=title,
80
+ status=status,
81
+ progress=progress,
82
+ message=str(item.get("task_message") or ""),
83
+ error_message=str(error_message) if error_message else None,
84
+ )
85
+
86
+
87
+ def get_job(repository: SqliteTaskRepository, job_id: str) -> KnowledgeJobResponse:
88
+ payload = repository.get_knowledge_job(job_id)
89
+ if payload is None:
90
+ raise HTTPException(status_code=404, detail="任务不存在。")
91
+ items = [_job_item(item) for item in payload.get("items", []) if isinstance(item, dict)]
92
+ completed = sum(1 for item in items if item.status in {TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED})
93
+ progress = round(completed / len(items) * 100) if items else 0
94
+ try:
95
+ agent_state = json.loads(str(payload.get("agent_state_json") or "{}"))
96
+ except (TypeError, ValueError):
97
+ agent_state = {}
98
+ return KnowledgeJobResponse(
99
+ job_id=str(payload["job_id"]),
100
+ conversation_id=str(payload["conversation_id"]),
101
+ assistant_message_id=str(payload["assistant_message_id"]),
102
+ kind=str(payload["kind"]),
103
+ status=str(payload["status"]),
104
+ query=str(payload["query"]),
105
+ progress=progress,
106
+ error_message=str(payload["error_message"]) if payload.get("error_message") else None,
107
+ items=items,
108
+ agent_round=max(1, int(payload.get("agent_round") or 1)),
109
+ agent_phase=str(agent_state.get("phase") or "waiting_tasks"),
110
+ created_at=payload["created_at"],
111
+ updated_at=payload["updated_at"],
112
+ completed_at=payload["completed_at"],
113
+ )
114
+
115
+
116
+ def search_references(repository: SqliteTaskRepository, query: str = "", limit: int = 12) -> KnowledgeReferencesResponse:
117
+ cleaned = str(query or "").strip().lower()
118
+ items = []
119
+ for video in repository.list_video_assets():
120
+ if cleaned and cleaned not in video.title.lower():
121
+ continue
122
+ items.append({
123
+ "kind": "video",
124
+ "id": video.video_id,
125
+ "label": video.title,
126
+ "subtitle": video.platform,
127
+ "cover_url": video.cover_url,
128
+ "updated_at": video.last_summary_at or video.updated_at,
129
+ })
130
+ for folder in repository.list_video_folders():
131
+ if cleaned and cleaned not in folder.name.lower():
132
+ continue
133
+ items.append({
134
+ "kind": "folder",
135
+ "id": folder.folder_id,
136
+ "label": folder.name,
137
+ "subtitle": f"{len(repository.list_video_ids_in_folder(folder.folder_id))} 个视频",
138
+ "updated_at": folder.updated_at,
139
+ })
140
+ items.sort(key=lambda item: item["updated_at"], reverse=True)
141
+ from video_sum_service.schemas import KnowledgeReferenceItem
142
+
143
+ return KnowledgeReferencesResponse(items=[KnowledgeReferenceItem.model_validate(item) for item in items[: max(1, min(limit, 50))]])
144
+
145
+
146
+ def _extract_urls(query: str) -> list[str]:
147
+ seen: set[str] = set()
148
+ urls: list[str] = []
149
+ for raw in _VIDEO_URL_RE.findall(query):
150
+ url = raw.rstrip(".,;!?,。;!?)")
151
+ if url and url not in seen:
152
+ seen.add(url)
153
+ urls.append(url)
154
+ return urls[:_MAX_TASK_LINKS]
155
+
156
+
157
+ def _is_task_request(query: str, references: list[KnowledgeReference]) -> bool:
158
+ markers = ("总结", "汇总", "整理成文档", "整理为文档", "做成文档", "生成文档")
159
+ return bool(_extract_urls(query) or references) and any(marker in query for marker in markers)
160
+
161
+
162
+ def _history(repository: SqliteTaskRepository, conversation_id: str) -> list[dict[str, str]]:
163
+ history = []
164
+ for item in repository.list_knowledge_messages(conversation_id)[-8:]:
165
+ if item["role"] in {"user", "assistant"} and item["content"] and item["status"] not in {"streaming", "interrupted"}:
166
+ history.append({"role": str(item["role"]), "content": str(item["content"])[:1200]})
167
+ return history
168
+
169
+
170
+ def _create_video_tasks(
171
+ app_state: object,
172
+ repository: SqliteTaskRepository,
173
+ query: str,
174
+ references: list[KnowledgeReference],
175
+ ) -> tuple[list[tuple[str, str | None, int]], list[str]]:
176
+ urls = _extract_urls(query)
177
+ for reference in references:
178
+ if reference.kind == "video":
179
+ video = repository.get_video_asset(reference.id)
180
+ if video is not None and video.source_url not in urls:
181
+ urls.append(video.source_url)
182
+ elif reference.kind == "folder":
183
+ for video_id in repository.list_video_ids_in_folder(reference.id):
184
+ video = repository.get_video_asset(video_id)
185
+ if video is not None and video.source_url not in urls:
186
+ urls.append(video.source_url)
187
+ task_items: list[tuple[str, str | None, int]] = []
188
+ errors: list[str] = []
189
+ for position, url in enumerate(urls[:_MAX_TASK_LINKS]):
190
+ try:
191
+ probed, _pages, _requires_selection = probe_video_asset(url)
192
+ asset = repository.upsert_video_asset(probed)
193
+ normalized = normalize_video_url(url)
194
+ record = repository.create_task(
195
+ TaskInput(input_type=InputType.URL, source=normalized.normalized_url, title=asset.title, platform_hint=asset.platform),
196
+ video_id=asset.video_id,
197
+ page_number=normalized.page_number if normalized.platform == "bilibili" else None,
198
+ page_title=asset.title,
199
+ )
200
+ submit_task_or_queue(app_state, repository, record)
201
+ task_items.append((record.task_id, asset.video_id, position))
202
+ except Exception as exc: # keep one bad URL from blocking the other items
203
+ errors.append(f"{url}:{exc}")
204
+ return task_items, errors
205
+
206
+
207
+ def _suggestion_facts(repository: SqliteTaskRepository) -> tuple[str, list[str]]:
208
+ videos = repository.list_video_assets()
209
+ recent = sorted(
210
+ [video for video in videos if video.last_summary_at is not None],
211
+ key=lambda video: video.last_summary_at or video.updated_at,
212
+ reverse=True,
213
+ )[:6]
214
+ folders = repository.list_video_folders()[:8]
215
+ tags = [str(tag.get("tag") or "").strip() for tag in repository.list_all_tags()[:12]]
216
+ tags = [tag for tag in tags if tag]
217
+ facts = "\n".join(
218
+ [
219
+ "最近总结视频:" + "、".join(video.title for video in recent),
220
+ "收藏视频:" + "、".join(video.title for video in videos if video.is_favorite)[:600],
221
+ "收藏夹:" + "、".join(f"{folder.name}({len(repository.list_video_ids_in_folder(folder.folder_id))})" for folder in folders),
222
+ "标签:" + "、".join(tags),
223
+ ]
224
+ )
225
+ based_on = [video.title for video in recent[:3]] + [folder.name for folder in folders[:2]]
226
+ return facts, based_on
227
+
228
+
229
+ def get_suggestions(repository: SqliteTaskRepository, settings: ServiceSettings) -> KnowledgeSuggestionsResponse:
230
+ try:
231
+ facts, based_on = _suggestion_facts(repository)
232
+ except Exception:
233
+ logger.exception("knowledge suggestion facts collection failed; using local fallback")
234
+ facts, based_on = "当前没有可读取的个性化推荐上下文。", []
235
+ fallback_titles = based_on[:3]
236
+ fallback = [
237
+ KnowledgeSuggestion(text="我最近在学什么主题?", reason="根据最近完成的总结"),
238
+ KnowledgeSuggestion(text="把最近的总结串成一份复习提纲。", reason="根据最近总结视频"),
239
+ KnowledgeSuggestion(text=f"总结一下收藏夹里的重点。", reason="根据收藏夹内容"),
240
+ KnowledgeSuggestion(text=(f"比较「{fallback_titles[0]}」和最近内容的共同主题。" if fallback_titles else "我的知识库里有哪些值得继续学习的主题?"), reason="根据已有知识库内容"),
241
+ ]
242
+ if not knowledge_llm_available(settings):
243
+ return KnowledgeSuggestionsResponse(suggestions=fallback[:4], based_on=based_on)
244
+ try:
245
+ text, _body = chat_knowledge_llm(
246
+ settings,
247
+ system_prompt="你是知识库首页推荐问题生成器。只返回 JSON:{\"suggestions\":[{\"text\":\"...\",\"reason\":\"...\"}]}。生成 3 到 6 个自然、具体、可回答的问题,每条不超过 60 个汉字。",
248
+ user_prompt=facts,
249
+ max_tokens=350,
250
+ temperature=0.35,
251
+ require_json=True,
252
+ )
253
+ payload = parse_json_payload(text)
254
+ raw = payload.get("suggestions")
255
+ suggestions = []
256
+ if isinstance(raw, list):
257
+ for item in raw:
258
+ if not isinstance(item, dict):
259
+ continue
260
+ label = str(item.get("text") or "").strip()[:80]
261
+ if label:
262
+ suggestions.append(KnowledgeSuggestion(text=label, reason=str(item.get("reason") or "")[:80]))
263
+ if 3 <= len(suggestions) <= 6:
264
+ return KnowledgeSuggestionsResponse(suggestions=suggestions, based_on=based_on)
265
+ except Exception as exc:
266
+ logger.warning("knowledge suggestion generation failed; using local fallback error=%s", exc)
267
+ return KnowledgeSuggestionsResponse(suggestions=fallback[:4], based_on=based_on)
268
+
269
+
270
+ def stream_conversation(
271
+ app_state: object,
272
+ repository: SqliteTaskRepository,
273
+ rag_service: RagService,
274
+ conversation_id: str,
275
+ query: str,
276
+ context_limit: int,
277
+ references: list[KnowledgeReference],
278
+ should_cancel: Callable[[], bool],
279
+ coordinator_submit: Callable[[str], None],
280
+ ) -> Iterator[tuple[str, dict[str, object]]]:
281
+ conversation = _model_or_404(repository, conversation_id)
282
+ clean_query = str(query or "").strip()
283
+ if not clean_query:
284
+ raise HTTPException(status_code=400, detail="问题不能为空。")
285
+ if conversation.title == "新会话":
286
+ repository.update_knowledge_conversation_title(conversation_id, clean_query[:36])
287
+ prior_history = _history(repository, conversation_id)
288
+ user = repository.create_knowledge_message(
289
+ conversation_id,
290
+ "user",
291
+ clean_query,
292
+ status="completed",
293
+ references=[reference.model_dump(mode="json") for reference in references],
294
+ )
295
+ assistant = repository.create_knowledge_message(
296
+ conversation_id,
297
+ "assistant",
298
+ "",
299
+ status="streaming",
300
+ references=[reference.model_dump(mode="json") for reference in references],
301
+ )
302
+ if user is None or assistant is None:
303
+ raise HTTPException(status_code=404, detail="会话不存在。")
304
+ assistant_id = str(assistant["message_id"])
305
+ yield ("message", {"user_message_id": user["message_id"], "assistant_message_id": assistant_id})
306
+ if _is_task_request(clean_query, references):
307
+ yield ("tool", {"id": "task_router", "label": "任务路由", "status": "running", "detail": "已识别为多视频总结任务,正在创建子任务。"})
308
+ items, errors = _create_video_tasks(app_state, repository, clean_query, references)
309
+ if not items:
310
+ detail = ";".join(errors) or "没有识别到可执行的视频链接。"
311
+ repository.update_knowledge_message(assistant_id, content=f"视频任务创建失败:{detail}", status="error", tools=[{"id": "task_router", "label": "任务路由", "status": "error", "detail": detail}])
312
+ yield ("error", {"message": detail, "status_code": 400})
313
+ return
314
+ job = repository.create_knowledge_job(
315
+ conversation_id,
316
+ assistant_id,
317
+ "multi_video_summary",
318
+ "queued",
319
+ clean_query,
320
+ agent_round=1,
321
+ agent_state={
322
+ "phase": "waiting_tasks",
323
+ "review": None,
324
+ "rounds": [{"round": 1, "name": "task_planning", "status": "completed"}],
325
+ },
326
+ )
327
+ if job is None:
328
+ raise HTTPException(status_code=500, detail="无法创建知识库任务。")
329
+ repository.add_knowledge_job_items(str(job["job_id"]), items)
330
+ repository.update_knowledge_message(
331
+ assistant_id,
332
+ content="任务已创建。所有视频完成后,我会继续生成聚合 Markdown 文档。",
333
+ status="waiting_tasks",
334
+ job_id=str(job["job_id"]),
335
+ tools=[{"id": "task_router", "label": "任务路由", "status": "completed", "detail": f"已创建 {len(items)} 个视频总结子任务。"}],
336
+ )
337
+ coordinator_submit(str(job["job_id"]))
338
+ if errors:
339
+ repository.update_knowledge_job(str(job["job_id"]), status="running", error_message=";".join(errors))
340
+ yield ("tool", {"id": "task_router", "label": "任务路由", "status": "completed", "detail": f"已创建 {len(items)} 个视频总结子任务。"})
341
+ yield ("job", {"job": get_job(repository, str(job["job_id"])).model_dump(mode="json")})
342
+ yield ("done", {"query": clean_query, "answer": "任务已创建。所有视频完成后,我会继续生成聚合 Markdown 文档。", "sources": [], "job_id": job["job_id"]})
343
+ return
344
+
345
+ content_parts: list[str] = []
346
+ reasoning_parts: list[str] = []
347
+ tools: list[dict[str, object]] = []
348
+ sources: list[dict[str, object]] = []
349
+ error_message = ""
350
+ finished = False
351
+ try:
352
+ for event_name, payload in rag_service.ask_stream(
353
+ clean_query,
354
+ context_limit=context_limit,
355
+ history=[KnowledgeChatHistoryItem.model_validate(item) for item in prior_history],
356
+ references=references,
357
+ should_cancel=should_cancel,
358
+ ):
359
+ if event_name == "text_delta":
360
+ content_parts.append(str(payload.get("delta") or ""))
361
+ elif event_name == "reasoning_delta":
362
+ if sum(len(item) for item in reasoning_parts) < _MAX_REASONING:
363
+ remaining = max(0, _MAX_REASONING - sum(len(item) for item in reasoning_parts))
364
+ reasoning_parts.append(str(payload.get("delta") or "")[:remaining])
365
+ elif event_name == "tool":
366
+ tool = dict(payload)
367
+ tool_id = str(tool.get("id") or "")
368
+ tools[:] = [item for item in tools if str(item.get("id")) != tool_id]
369
+ tools.append(tool)
370
+ elif event_name == "sources":
371
+ sources = list(payload.get("sources") or [])
372
+ elif event_name == "done":
373
+ finished = True
374
+ elif event_name == "error":
375
+ error_message = str(payload.get("message") or "流式回答失败")
376
+ yield (event_name, payload)
377
+ if finished:
378
+ content = "".join(content_parts).strip()
379
+ repository.update_knowledge_message(
380
+ assistant_id,
381
+ content=content or "这次没有生成可读取的回答正文。",
382
+ status="completed",
383
+ reasoning="".join(reasoning_parts),
384
+ sources=sources,
385
+ tools=tools,
386
+ )
387
+ elif should_cancel():
388
+ repository.update_knowledge_message(assistant_id, content="本轮回答已中断,可点击重新回答。", status="interrupted", reasoning="".join(reasoning_parts), sources=sources, tools=tools)
389
+ elif error_message:
390
+ repository.update_knowledge_message(assistant_id, content=f"知识库助手暂时没有完成回答。\n\n{error_message}", status="error", reasoning="".join(reasoning_parts), sources=sources, tools=tools)
391
+ else:
392
+ repository.update_knowledge_message(assistant_id, content="流式回答提前结束,可点击重新回答。", status="error", reasoning="".join(reasoning_parts), sources=sources, tools=tools)
393
+ except Exception as exc:
394
+ detail = str(exc.detail) if isinstance(exc, HTTPException) else str(exc)
395
+ repository.update_knowledge_message(assistant_id, content=f"知识库助手暂时没有完成回答。\n\n{detail}", status="error", reasoning="".join(reasoning_parts), sources=sources, tools=tools)
396
+ yield ("error", {"message": detail, "status_code": getattr(exc, "status_code", 500)})
397
+ finally:
398
+ if should_cancel() and not finished:
399
+ current = repository.get_knowledge_message(assistant_id)
400
+ if current is not None and current.get("status") == "streaming":
401
+ repository.update_knowledge_message(
402
+ assistant_id,
403
+ content="本轮回答已中断,可点击重新回答。",
404
+ status="interrupted",
405
+ reasoning="".join(reasoning_parts),
406
+ sources=sources,
407
+ tools=tools,
408
+ )
@@ -20,6 +20,18 @@ from video_sum_service.schemas import KnowledgeIndexChunkRecord, KnowledgeSearch
20
20
 
21
21
  logger = logging.getLogger("video_sum_service.knowledge")
22
22
 
23
+ _SILICONFLOW_EMBEDDING_BATCH_SIZE = 16
24
+ _SILICONFLOW_MODEL_CHAR_LIMITS = {
25
+ # SiliconFlow documents a 512-token limit for the classic BGE v1.5 models.
26
+ # A conservative character cap also works for Chinese text, where a single
27
+ # character is frequently close to a token, without requiring a tokenizer.
28
+ "BAAI/bge-large-zh-v1.5": 400,
29
+ "BAAI/bge-large-en-v1.5": 400,
30
+ "netease-youdao/bce-embedding-base_v1": 400,
31
+ "BAAI/bge-m3": 6000,
32
+ "Pro/BAAI/bge-m3": 6000,
33
+ }
34
+
23
35
 
24
36
  def format_anchor_seconds(seconds: float | None) -> str | None:
25
37
  if seconds is None:
@@ -211,40 +223,57 @@ class KnowledgeIndexService:
211
223
  base_url = self._settings.siliconflow_embedding_base_url.rstrip("/")
212
224
  model = self._settings.siliconflow_embedding_model or self._model_name
213
225
 
226
+ prepared_texts = self._prepare_siliconflow_texts(texts, model)
227
+ embeddings: list[list[float]] = []
214
228
  try:
215
229
  with httpx.Client(timeout=60.0) as client:
216
- response = client.post(
217
- f"{base_url}/embeddings",
218
- headers={
219
- "Authorization": f"Bearer {api_key}",
220
- "Content-Type": "application/json",
221
- },
222
- json={
223
- "model": model,
224
- "input": texts,
225
- "encoding_format": "float",
226
- },
227
- )
228
- response.raise_for_status()
229
- result = response.json()
230
- data = result.get("data", [])
231
- if not data:
232
- raise HTTPException(
233
- status_code=500,
234
- detail="硅基流动 Embedding API 返回数据为空。",
230
+ for offset in range(0, len(prepared_texts), _SILICONFLOW_EMBEDDING_BATCH_SIZE):
231
+ batch = prepared_texts[offset : offset + _SILICONFLOW_EMBEDDING_BATCH_SIZE]
232
+ response = client.post(
233
+ f"{base_url}/embeddings",
234
+ headers={
235
+ "Authorization": f"Bearer {api_key}",
236
+ "Content-Type": "application/json",
237
+ },
238
+ json={
239
+ "model": model,
240
+ "input": batch,
241
+ "encoding_format": "float",
242
+ },
243
+ )
244
+ response.raise_for_status()
245
+ result = response.json()
246
+ data = result.get("data", []) if isinstance(result, dict) else []
247
+ if len(data) != len(batch):
248
+ raise HTTPException(
249
+ status_code=502,
250
+ detail=(
251
+ "硅基流动 Embedding API 返回向量数量不匹配"
252
+ f"(请求 {len(batch)},返回 {len(data)})。"
253
+ ),
254
+ )
255
+ embeddings.extend(
256
+ [list(map(float, item["embedding"])) for item in sorted(data, key=lambda item: item["index"])]
235
257
  )
236
- embeddings = [item["embedding"] for item in sorted(data, key=lambda x: x["index"])]
237
- return embeddings
258
+ return embeddings
238
259
  except httpx.HTTPStatusError as exc:
260
+ api_detail = self._siliconflow_error_detail(exc.response)
239
261
  logger.exception(
240
- "siliconflow embedding API error status=%s model=%s",
262
+ "siliconflow embedding API error status=%s model=%s detail=%s trace_id=%s",
241
263
  exc.response.status_code,
242
264
  model,
265
+ api_detail,
266
+ exc.response.headers.get("x-siliconcloud-trace-id", ""),
243
267
  )
244
268
  raise HTTPException(
245
269
  status_code=502,
246
- detail=f"硅基流动 Embedding API 请求失败(HTTP {exc.response.status_code})。",
270
+ detail=(
271
+ f"硅基流动 Embedding API 请求失败(HTTP {exc.response.status_code})"
272
+ f":{api_detail}"
273
+ ),
247
274
  ) from exc
275
+ except HTTPException:
276
+ raise
248
277
  except Exception as exc:
249
278
  logger.exception(
250
279
  "siliconflow embedding API error model=%s",
@@ -255,6 +284,44 @@ class KnowledgeIndexService:
255
284
  detail=f"硅基流动 Embedding API 调用异常:{self._short_error(exc)}。",
256
285
  ) from exc
257
286
 
287
+ def _prepare_siliconflow_texts(self, texts: list[str], model: str) -> list[str]:
288
+ char_limit = _SILICONFLOW_MODEL_CHAR_LIMITS.get(model, 12000)
289
+ prepared: list[str] = []
290
+ truncated_count = 0
291
+ for raw_text in texts:
292
+ text = str(raw_text or "").strip() or "(空内容)"
293
+ if len(text) > char_limit:
294
+ text = text[:char_limit].rstrip()
295
+ truncated_count += 1
296
+ prepared.append(text)
297
+ if truncated_count:
298
+ logger.info(
299
+ "siliconflow embedding inputs shortened model=%s count=%s char_limit=%s",
300
+ model,
301
+ truncated_count,
302
+ char_limit,
303
+ )
304
+ return prepared
305
+
306
+ def _siliconflow_error_detail(self, response: Any) -> str:
307
+ detail = "上游服务未提供错误详情"
308
+ try:
309
+ payload = response.json()
310
+ if isinstance(payload, dict):
311
+ error = payload.get("error")
312
+ if isinstance(error, dict):
313
+ detail = str(error.get("message") or error.get("detail") or detail)
314
+ elif error:
315
+ detail = str(error)
316
+ else:
317
+ detail = str(payload.get("message") or payload.get("detail") or detail)
318
+ except Exception:
319
+ response_text = str(getattr(response, "text", "") or "").strip()
320
+ if response_text:
321
+ detail = response_text
322
+ detail = re.sub(r"\s+", " ", detail).strip()
323
+ return detail[:300] or "上游服务未提供错误详情"
324
+
258
325
  def _split_markdown_sections(self, markdown: str) -> list[tuple[str, str]]:
259
326
  content = str(markdown or "").strip()
260
327
  if not content:
@@ -425,7 +492,13 @@ class KnowledgeIndexService:
425
492
  count += 1
426
493
  return count
427
494
 
428
- def _fetch_candidate_chunks(self, query: str, limit: int = 10, tag_filter: list[str] | None = None) -> list[dict[str, object]]:
495
+ def _fetch_candidate_chunks(
496
+ self,
497
+ query: str,
498
+ limit: int = 10,
499
+ tag_filter: list[str] | None = None,
500
+ video_ids: set[str] | None = None,
501
+ ) -> list[dict[str, object]]:
429
502
  cleaned_query = str(query or "").strip()
430
503
  if not cleaned_query:
431
504
  return []
@@ -453,6 +526,9 @@ class KnowledgeIndexService:
453
526
  if item.tag in selected
454
527
  }
455
528
 
529
+ if video_ids is not None:
530
+ allowed_video_ids = set(video_ids) if allowed_video_ids is None else allowed_video_ids & set(video_ids)
531
+
456
532
  candidates: list[dict[str, object]] = []
457
533
  for index, chunk_id in enumerate(ids):
458
534
  metadata = metadatas[index] if index < len(metadatas) and isinstance(metadatas[index], dict) else {}
@@ -472,8 +548,14 @@ class KnowledgeIndexService:
472
548
  )
473
549
  return candidates
474
550
 
475
- def search(self, query: str, limit: int = 10, tag_filter: list[str] | None = None) -> list[KnowledgeSearchResult]:
476
- candidates = self._fetch_candidate_chunks(query, limit=limit, tag_filter=tag_filter)
551
+ def search(
552
+ self,
553
+ query: str,
554
+ limit: int = 10,
555
+ tag_filter: list[str] | None = None,
556
+ video_ids: set[str] | None = None,
557
+ ) -> list[KnowledgeSearchResult]:
558
+ candidates = self._fetch_candidate_chunks(query, limit=limit, tag_filter=tag_filter, video_ids=video_ids)
477
559
  if not candidates:
478
560
  return []
479
561
 
@@ -519,5 +601,11 @@ class KnowledgeIndexService:
519
601
  )
520
602
  return results
521
603
 
522
- def search_chunks(self, query: str, limit: int = 5, tag_filter: list[str] | None = None) -> list[dict[str, object]]:
523
- return self._fetch_candidate_chunks(query, limit=limit, tag_filter=tag_filter)[: max(1, limit)]
604
+ def search_chunks(
605
+ self,
606
+ query: str,
607
+ limit: int = 5,
608
+ tag_filter: list[str] | None = None,
609
+ video_ids: set[str] | None = None,
610
+ ) -> list[dict[str, object]]:
611
+ return self._fetch_candidate_chunks(query, limit=limit, tag_filter=tag_filter, video_ids=video_ids)[: max(1, limit)]