flare-chat-core 0.2.3 → 0.2.4

package/src/chat-core/pipelines/transport/send-real-stream.js

@@ -1,330 +0,0 @@
-import {
-  parseAgentStatus,
-  parseExecutionTrace,
-  parseKnowledgeSearch,
-  parseThinkingTrace,
-  parseSourcingCandidates,
-  parseKnowledgeCitation,
-} from '../../../contracts/sse-events.js';
-import { debugCoreLog, getCoreRuntimeMode } from '../../runtime/runtimeMode.js';
-import { buildRealStreamRequestBody, resolvePayloadExtra } from '../build-stream-request.js';
-import { buildRealActionRequestBody } from '../build-action-request.js';
-import { collectSourceIdsFromRefs, resolvePersistExchangeResult } from '../stream-persist-utils.js';
-import { requestSourcingSearch } from './send-sourcing-search.js';
-
-function joinUrl(baseUrl = '', path = '') {
-  if (!baseUrl) {
-    return path || '';
-  }
-  if (!path) {
-    return baseUrl;
-  }
-  return `${baseUrl.replace(/\/+$/, '')}/${path.replace(/^\/+/, '')}`;
-}
-
-export function shouldUseSourcingSearchPath(options = {}, content = '') {
-  void content;
-  const modeKey = String(options?.modeKey || options?.manualModeKey || '').trim();
-  if (modeKey !== 'intelligent_sourcing') {
-    return false;
-  }
-  const payloadExtra = resolvePayloadExtra(options);
-  if (payloadExtra.append_mode === true) {
-    return true;
-  }
-  if (payloadExtra.force_sourcing_search === true) {
-    return true;
-  }
-  return false;
-}
-
-export async function sendRealStreamMessage({
-  apiBaseUrl,
-  content,
-  handlers = {},
-  options = {},
-  scope = {},
-  persistExchange,
-  signal,
-}) {
-  const resolvedProjectId = String(scope?.projectId || '').trim();
-  const resolvedUserId = String(scope?.userId || '').trim();
-  const sessionId = String(options.sessionIdOverride || '').trim();
-  const requestBody = buildRealStreamRequestBody({
-    content,
-    options,
-    scope: {
-      projectId: resolvedProjectId,
-      userId: resolvedUserId,
-    },
-  });
-
-  const runtimeMode = getCoreRuntimeMode();
-  // Dev-only debug: show request-side mode and key stream switches.
-  debugCoreLog('stream.send.start', {
-    mode: runtimeMode.mode,
-    sessionId,
-    modeKey: String(options?.modeKey || ''),
-    manualModeKey: String(options?.manualModeKey || ''),
-    knowledgeRefsCount: Array.isArray(requestBody?.knowledge_refs) ? requestBody.knowledge_refs.length : 0,
-  });
-  const requestSourceIds = collectSourceIdsFromRefs([
-    ...(Array.isArray(requestBody?.context_refs) ? requestBody.context_refs : []),
-    ...(Array.isArray(requestBody?.knowledge_refs) ? requestBody.knowledge_refs : []),
-  ]);
-  const resolvedModeKey = String(options?.modeKey || options?.manualModeKey || '').trim() || 'auto';
-  handlers.onExecutionTrace?.({
-    session_id: sessionId || '',
-    step_id: 'context_binding',
-    agent: 'context_router',
-    stage: 'orchestration',
-    label: '上下文绑定',
-    detail: requestSourceIds.length > 0
-      ? `mode=${resolvedModeKey}; 绑定 ${requestSourceIds.length} 个来源: ${requestSourceIds.join(', ')}`
-      : `mode=${resolvedModeKey}; 未绑定来源，按常规上下文流程处理`,
-    reason: '',
-    status: 'running',
-    mode_key: resolvedModeKey,
-    sources: requestSourceIds.map((sourceId) => ({ source_id: sourceId })),
-    timestamp: Date.now(),
-  });
-
-  let finalContent = '';
-  let resolvedSessionId = sessionId;
-  let latestContextUsage = null;
-
-  if (shouldUseSourcingSearchPath(options, content)) {
-    await requestSourcingSearch({
-      apiBaseUrl,
-      content,
-      options,
-      scope: { projectId: resolvedProjectId, userId: resolvedUserId },
-      handlers,
-      persistExchange,
-      resolvedSessionId,
-    });
-    return;
-  }
-
-  const response = await fetch(joinUrl(apiBaseUrl, '/chat/stream'), {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify(requestBody),
-    signal,
-  });
-
-  if (!response.ok) {
-    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-  }
-
-  if (!response.body) {
-    throw new Error('stream response body is empty');
-  }
-
-  const reader = response.body.getReader();
-  const decoder = new TextDecoder();
-  let buffer = '';
-  let currentEvent = '';
-  let dataLines = [];
-
-  const emitEvent = () => {
-    if (!currentEvent || dataLines.length === 0) {
-      currentEvent = '';
-      dataLines = [];
-      return;
-    }
-
-    const rawData = dataLines.join('\n');
-    let payload = {};
-    try {
-      payload = JSON.parse(rawData);
-    } catch {
-      payload = {};
-    }
-    // Dev-only debug: trace the incoming SSE event types and payload outline.
-    debugCoreLog('stream.event', {
-      type: currentEvent,
-      keys: Object.keys(payload || {}),
-    });
-
-    if (currentEvent === 'content') {
-      const chunk = String(payload?.content || '');
-      finalContent += chunk;
-      handlers.onContent?.(chunk);
-    } else if (currentEvent === 'agent_status') {
-      handlers.onAgentStatus?.(parseAgentStatus(payload));
-    } else if (currentEvent === 'thinking_trace') {
-      handlers.onThinkingTrace?.(parseThinkingTrace(payload)?.trace || '');
-    } else if (currentEvent === 'execution_trace') {
-      handlers.onExecutionTrace?.(parseExecutionTrace(payload));
-    } else if (currentEvent === 'context_usage') {
-      latestContextUsage = (
-        payload
-        && typeof payload === 'object'
-        && !Array.isArray(payload)
-      ) ? { ...payload } : null;
-      const modeKey = String(payload?.mode_key || '').trim();
-      const modeState = String(payload?.mode_state || '').trim();
-      const profile = String(payload?.profile || '').trim();
-      const totalTokens = Number(payload?.token_estimate?.total_prompt_tokens_estimate || 0);
-      handlers.onExecutionTrace?.({
-        session_id: String(payload?.session_id || '').trim(),
-        step_id: 'context_usage',
-        agent: 'context_router',
-        stage: 'orchestration',
-        label: '上下文预算评估',
-        detail: `mode=${modeKey || 'default'}/${modeState || 'default'}; profile=${profile || 'default'}; tokens≈${totalTokens}`,
-        reason: '',
-        status: 'completed',
-        timestamp: Number.isFinite(Number(payload?.ts)) ? Number(payload.ts) : Date.now(),
-      });
-    } else if (currentEvent === 'knowledge_search') {
-      handlers.onKnowledgeSearch?.(parseKnowledgeSearch(payload));
-    } else if (currentEvent === 'sourcing_candidates') {
-      handlers.onSourcingCandidates?.(parseSourcingCandidates(payload));
-    } else if (currentEvent === 'knowledge_citation') {
-      handlers.onKnowledgeCitation?.(parseKnowledgeCitation(payload));
-    } else if (currentEvent === 'patch_event') {
-      const patchSessionId = String(payload?.session?.session_id || '').trim();
-      if (patchSessionId) {
-        resolvedSessionId = patchSessionId;
-      }
-      handlers.onPatchEvent?.(payload);
-    } else if (currentEvent === 'complete') {
-      let resolvedSessionTitle = '';
-      if (typeof persistExchange === 'function') {
-        const persistedResult = persistExchange(resolvedSessionId, content, finalContent, {
-          functionType: 'chat_component_debug',
-          projectId: resolvedProjectId || null,
-          userId: resolvedUserId || null,
-        });
-        const persistedSession = resolvePersistExchangeResult(persistedResult, resolvedSessionId);
-        resolvedSessionId = persistedSession.sessionId;
-        resolvedSessionTitle = persistedSession.title;
-      }
-      handlers.onPatchEvent?.({
-        session: {
-          session_id: resolvedSessionId,
-          turn_id: '',
-        },
-        payload: resolvedSessionTitle
-          ? {
-              session: {
-                title: resolvedSessionTitle,
-              },
-            }
-          : {},
-        patch_scope: ['session'],
-        event_type: 'final',
-        final: true,
-        invalid: false,
-      });
-      handlers.onComplete?.(finalContent, {
-        sessionId: resolvedSessionId,
-        ...(latestContextUsage
-          ? {
-              context_usage: latestContextUsage,
-            }
-          : {}),
-        ...(resolvedSessionTitle
-          ? {
-              payload: {
-                session: {
-                  title: resolvedSessionTitle,
-                },
-              },
-            }
-          : {}),
-      });
-      debugCoreLog('stream.send.complete', {
-        sessionId: resolvedSessionId,
-        finalContentLength: finalContent.length,
-      });
-    }
-
-    currentEvent = '';
-    dataLines = [];
-  };
-
-  while (true) {
-    const { done, value } = await reader.read();
-    if (done) {
-      break;
-    }
-
-    buffer += decoder.decode(value, { stream: true });
-    const lines = buffer.split('\n');
-    buffer = lines.pop() || '';
-
-    for (const line of lines) {
-      const normalized = line.replace(/\r$/, '');
-      if (!normalized) {
-        emitEvent();
-        continue;
-      }
-      if (normalized.startsWith('event:')) {
-        currentEvent = normalized.slice(6).trim();
-        continue;
-      }
-      if (normalized.startsWith('data:')) {
-        dataLines.push(normalized.slice(5).trim());
-      }
-    }
-  }
-
-  emitEvent();
-  handlers.onExecutionTrace?.({
-    session_id: resolvedSessionId || '',
-    step_id: 'context_binding',
-    agent: 'context_router',
-    stage: 'orchestration',
-    label: '上下文绑定',
-    detail: requestSourceIds.length > 0
-      ? `已完成来源绑定（${requestSourceIds.length}）`
-      : '已完成常规上下文绑定',
-    reason: '',
-    status: 'completed',
-    sources: requestSourceIds.map((sourceId) => ({ source_id: sourceId })),
-    timestamp: Date.now(),
-  });
-}
-
-export async function sendRealStreamAction({
-  apiBaseUrl,
-  action,
-  handlers = {},
-  options = {},
-  scope = {},
-  signal,
-}) {
-  const requestBody = buildRealActionRequestBody({
-    action,
-    options,
-    scope,
-  });
-
-  const response = await fetch(joinUrl(apiBaseUrl, '/chat/action'), {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify(requestBody),
-    signal,
-  });
-
-  if (!response.ok) {
-    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-  }
-
-  const payload = await response.json();
-  const result = (
-    payload?.result
-    && typeof payload.result === 'object'
-    && !Array.isArray(payload.result)
-  ) ? payload.result : {};
-  const content = String(result.message || payload?.message || '').trim();
-
-  if (content) {
-    handlers.onContent?.(content);
-  }
-  handlers.onComplete?.(content);
-  return payload;
-}

package/src/chat-core/pipelines/transport/send-real-stream.test.mjs

@@ -1,27 +0,0 @@
-import test from 'node:test';
-import assert from 'node:assert/strict';
-import { shouldUseSourcingSearchPath } from './send-real-stream.js';
-
-test('shouldUseSourcingSearchPath keeps sourcing mode on stream path by default', () => {
-  const shouldUse = shouldUseSourcingSearchPath({
-    modeKey: 'intelligent_sourcing',
-    payloadExtra: {},
-  }, '来源1');
-  assert.equal(shouldUse, false);
-});
-
-test('shouldUseSourcingSearchPath uses direct path in append rounds', () => {
-  const shouldUse = shouldUseSourcingSearchPath({
-    modeKey: 'intelligent_sourcing',
-    payloadExtra: { append_mode: true },
-  }, '能不能换一批');
-  assert.equal(shouldUse, true);
-});
-
-test('shouldUseSourcingSearchPath uses direct path only with explicit force flag', () => {
-  const shouldUse = shouldUseSourcingSearchPath({
-    modeKey: 'intelligent_sourcing',
-    payloadExtra: { force_sourcing_search: true },
-  }, '任意输入');
-  assert.equal(shouldUse, true);
-});

package/src/chat-core/pipelines/transport/send-sourcing-search.js

@@ -1,86 +0,0 @@
-import { parseSourcingCandidates } from '../../../contracts/sse-events.js';
-import { resolvePayloadExtra, resolveStreamRefs } from '../build-stream-request.js';
-import { resolvePersistExchangeResult } from '../stream-persist-utils.js';
-import { buildSourcingResponseText } from './sourcing-response-templates.js';
-import { requestSourcingSearchApi } from './sourcing-search-api.js';
-
-const SOURCING_REFRESH_COMMANDS = new Set(['换一批', '继续', '继续寻源']);
-
-function normalizeText(value) {
-  return String(value || '').trim();
-}
-
-export function resolveSourcingSearchQuery(content, payloadExtra = {}) {
-  const normalizedContent = normalizeText(content);
-  const payloadSourcingQuery = normalizeText(payloadExtra?.sourcing_query);
-  if (payloadSourcingQuery && SOURCING_REFRESH_COMMANDS.has(normalizedContent)) {
-    return payloadSourcingQuery;
-  }
-  return normalizedContent || payloadSourcingQuery;
-}
-
-export async function requestSourcingSearch({
-  apiBaseUrl,
-  content,
-  options,
-  scope,
-  handlers,
-  persistExchange,
-  resolvedSessionId,
-}) {
-  const payloadExtra = resolvePayloadExtra(options);
-  const resolvedQuery = resolveSourcingSearchQuery(content, payloadExtra);
-  const streamRefs = resolveStreamRefs(options);
-  const sourceIds = Array.isArray(streamRefs.knowledge_refs)
-    ? streamRefs.knowledge_refs.map((item) => String(item || '').trim()).filter(Boolean)
-    : [];
-  const appendMode = payloadExtra.append_mode === true;
-  const { payload, results, sourceBreakdown, sourcingPayload } = await requestSourcingSearchApi({
-    apiBaseUrl,
-    projectId: String(scope?.projectId || '').trim() || 'default',
-    userId: String(scope?.userId || '').trim() || 'default',
-    sessionId: String(resolvedSessionId || '').trim() || 'default',
-    query: resolvedQuery,
-    topK: Number.isFinite(Number(payloadExtra.top_k)) ? Number(payloadExtra.top_k) : 8,
-    appendMode,
-    lockExistingOrder: payloadExtra.lock_existing_order !== false,
-    baseResultIds: Array.isArray(payloadExtra.base_result_ids) ? payloadExtra.base_result_ids : [],
-    sourceScope: String(payloadExtra.source_scope || 'hybrid').trim() || 'hybrid',
-    contextRefs: Array.isArray(streamRefs.context_refs) ? streamRefs.context_refs : [],
-    knowledgeRefs: sourceIds,
-    sourceIds,
-  });
-  handlers.onSourcingCandidates?.(parseSourcingCandidates(sourcingPayload));
-
-  const sourceSummary = `来源分布：本地资料 ${Number(sourceBreakdown.local) || 0} / 外部连接 ${Number(sourceBreakdown.mcp) || 0} / 网页来源 ${Number(sourceBreakdown.web) || 0}`;
-  const citableResults = results
-    .filter((item) => String(item?.canonical_url || item?.url || '').trim())
-    .slice(0, 2);
-  const topRefs = citableResults
-    .map((item, index) => `- [来源${index + 1}] ${String(item?.source_title || item?.title || '未命名来源')}（${String(item?.canonical_url || item?.url || '')}）`)
-    .join('\n');
-
-  const finalContent = buildSourcingResponseText({
-    count: results.length,
-    sourceBreakdownText: sourceSummary,
-    topRefsText: topRefs,
-    nextActionText: '继续补充你的目标与约束，我会基于当前上下文递进处理。',
-  });
-
-  if (appendMode) {
-    handlers.onComplete?.(finalContent, { sessionId: String(resolvedSessionId || '').trim() });
-    return;
-  }
-
-  let finalSessionId = String(resolvedSessionId || '').trim();
-  if (typeof persistExchange === 'function') {
-    const persistedResult = persistExchange(finalSessionId, content, finalContent, {
-      functionType: 'intelligent_sourcing',
-      projectId: String(scope?.projectId || '').trim() || null,
-      userId: String(scope?.userId || '').trim() || null,
-    });
-    const persistedSession = resolvePersistExchangeResult(persistedResult, finalSessionId);
-    finalSessionId = persistedSession.sessionId;
-  }
-  handlers.onComplete?.(finalContent, { sessionId: finalSessionId });
-}

package/src/chat-core/pipelines/transport/send-sourcing-search.test.mjs

@@ -1,14 +0,0 @@
-import test from 'node:test';
-import assert from 'node:assert/strict';
-import { resolveSourcingSearchQuery } from './send-sourcing-search.js';
-
-test('resolveSourcingSearchQuery uses payload sourcing_query for refresh commands', () => {
-  const query = resolveSourcingSearchQuery('换一批', { sourcing_query: '露营地 上海 亲子' });
-  assert.equal(query, '露营地 上海 亲子');
-});
-
-test('resolveSourcingSearchQuery keeps direct semantic input', () => {
-  const query = resolveSourcingSearchQuery('上海露营地推荐', { sourcing_query: '旧查询' });
-  assert.equal(query, '上海露营地推荐');
-});
-

package/src/chat-core/pipelines/transport/sourcing-response-templates.js

@@ -1,55 +0,0 @@
-const SOURCING_RESPONSE_TEMPLATES = {
-  sufficient_hits: [
-    '已完成寻源：共 {count} 条候选。',
-    '{source_breakdown}',
-    '{top_refs}',
-    '{next_action}',
-  ],
-  limited_hits: [
-    '已完成初步寻源：当前仅命中 {count} 条候选。',
-    '{source_breakdown}',
-    '{top_refs}',
-    '建议：补充更具体的地区/行业/预算关键词后，我可以继续扩展检索。',
-    '{next_action}',
-  ],
-  no_hits: [
-    '本轮未命中有效候选。',
-    '{source_breakdown}',
-    '可能原因：关键词过宽或外部通道未返回有效内容。',
-    '建议：补充地区/行业词，或检查外部网络与网关配置后重试。',
-    '{next_action}',
-  ],
-};
-
-function renderTemplate(templateLines, variables = {}) {
-  const lines = Array.isArray(templateLines) ? templateLines : [];
-  return lines
-    .map((line) => String(line || '').replace(/\{([^}]+)\}/g, (_, key) => String(variables[key] || '')))
-    .map((line) => line.trim())
-    .filter(Boolean)
-    .join('\n');
-}
-
-export function buildSourcingResponseText({
-  count = 0,
-  sourceBreakdownText = '',
-  topRefsText = '',
-  nextActionText = '你可以直接回复序号或补充信息，我来继续整理。',
-} = {}) {
-  const normalizedCount = Number.isFinite(Number(count)) ? Number(count) : 0;
-  const variables = {
-    count: normalizedCount,
-    source_breakdown: sourceBreakdownText,
-    top_refs: topRefsText || '暂无可展示引用来源。',
-    next_action: nextActionText,
-  };
-  if (normalizedCount <= 0) {
-    return renderTemplate(SOURCING_RESPONSE_TEMPLATES.no_hits, variables);
-  }
-  if (normalizedCount <= 2) {
-    return renderTemplate(SOURCING_RESPONSE_TEMPLATES.limited_hits, variables);
-  }
-  return renderTemplate(SOURCING_RESPONSE_TEMPLATES.sufficient_hits, variables);
-}
-
-export { SOURCING_RESPONSE_TEMPLATES };

package/src/chat-core/pipelines/transport/sourcing-search-api.js

@@ -1,155 +0,0 @@
-import { resolveEntities } from '../entity-extraction.js';
-
-function joinUrl(baseUrl = '', path = '') {
-  if (!baseUrl) {
-    return path || '';
-  }
-  if (!path) {
-    return baseUrl;
-  }
-  return `${baseUrl.replace(/\/+$/, '')}/${path.replace(/^\/+/, '')}`;
-}
-
-function normalizeSourceType(value) {
-  const rawType = String(value || '').trim().toLowerCase();
-  if (rawType.includes('web')) {
-    return 'web';
-  }
-  if (rawType.includes('mcp')) {
-    return 'mcp';
-  }
-  return 'local';
-}
-
-function normalizeSourceBreakdown(payload = {}) {
-  const sourceBreakdown = (
-    payload?.source_breakdown
-    && typeof payload.source_breakdown === 'object'
-    && !Array.isArray(payload.source_breakdown)
-  ) ? payload.source_breakdown : {};
-  return {
-    local: Number(sourceBreakdown.local) || 0,
-    mcp: Number(sourceBreakdown.mcp) || 0,
-    web: Number(sourceBreakdown.web) || 0,
-  };
-}
-
-function mapResultToCandidate(item = {}, runId = '', resolvedQuery = '') {
-  return {
-    id: String(item?.result_id || item?.record_id || item?.title || ''),
-    result_id: String(item?.result_id || item?.record_id || item?.title || ''),
-    title: String(item?.source_title || item?.title || ''),
-    supplier_name: String(item?.source_title || item?.title || ''),
-    evidence_kind: 'sourcing',
-    match_score: Number(item?.score || 0),
-    match_reasons: [String(item?.snippet || '')],
-    evidence_refs: [
-      {
-        url: String(item?.canonical_url || item?.url || ''),
-        snippet: String(item?.snippet || ''),
-        title: String(item?.source_title || item?.title || ''),
-      },
-    ],
-    source_type: normalizeSourceType(item?.source_type || item?.source),
-    source_title: String(item?.source_title || item?.title || ''),
-    source_domain: String(item?.source_domain || ''),
-    source_meta: (
-      item?.source_meta
-      && typeof item.source_meta === 'object'
-      && !Array.isArray(item.source_meta)
-    ) ? item.source_meta : {},
-    canonical_url: String(item?.canonical_url || item?.url || ''),
-    confidence: item?.confidence ?? '',
-    confidence_detail: (
-      item?.confidence_detail
-      && typeof item.confidence_detail === 'object'
-      && !Array.isArray(item.confidence_detail)
-    ) ? item.confidence_detail : null,
-    summary: String(item?.snippet || ''),
-    run_id: runId,
-    query: resolvedQuery,
-  };
-}
-
-export async function requestSourcingSearchApi({
-  apiBaseUrl,
-  projectId = 'default',
-  userId = 'default',
-  sessionId = 'default',
-  query,
-  topK = 8,
-  sourceScope = 'hybrid',
-  contextRefs = [],
-  knowledgeRefs = [],
-  sourceIds = [],
-  entities = [],
-  appendMode = false,
-  lockExistingOrder = true,
-  baseResultIds = [],
-}) {
-  const resolvedEntities = resolveEntities({ text: query, entities });
-  const response = await fetch(joinUrl(apiBaseUrl, '/kernel/sourcing/search'), {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify({
-      project_id: String(projectId || '').trim() || 'default',
-      user_id: String(userId || '').trim() || 'default',
-      session_id: String(sessionId || '').trim() || 'default',
-      query: String(query || '').trim(),
-      top_k: Number.isFinite(Number(topK)) ? Number(topK) : 8,
-      append_mode: appendMode === true,
-      lock_existing_order: lockExistingOrder !== false,
-      base_result_ids: Array.isArray(baseResultIds)
-        ? baseResultIds.map((item) => String(item || '').trim()).filter(Boolean)
-        : [],
-      source_scope: String(sourceScope || 'hybrid').trim() || 'hybrid',
-      sourcing_enabled: true,
-      context_refs: Array.isArray(contextRefs) ? contextRefs : [],
-      knowledge_refs: Array.isArray(knowledgeRefs) ? knowledgeRefs : [],
-      source_ids: Array.isArray(sourceIds) ? sourceIds : [],
-      entities: resolvedEntities,
-    }),
-  });
-
-  if (!response.ok) {
-    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-  }
-
-  const payload = await response.json();
-  const runId = String(payload?.retrieval_run_id || '').trim();
-  const resolvedQuery = String(payload?.query || query).trim();
-  const results = Array.isArray(payload?.results) ? payload.results : [];
-  const newResults = Array.isArray(payload?.new_results) ? payload.new_results : [];
-  const allResults = Array.isArray(payload?.all_results) ? payload.all_results : [];
-  const sourceBreakdown = normalizeSourceBreakdown(payload);
-  const sourcingPayload = {
-    run_id: runId,
-    query: resolvedQuery,
-    candidate_count: results.length,
-    requested_top_k: Number(payload?.requested_top_k) || Number(topK) || 8,
-    returned_count: Number(payload?.returned_count) || results.length,
-    has_more: payload?.has_more === true,
-    next_top_k: Number(payload?.next_top_k) || Number(topK) || 8,
-    append_mode: appendMode === true,
-    lock_existing_order: lockExistingOrder !== false,
-    candidates: results.map((item) => mapResultToCandidate(item, runId, resolvedQuery)),
-    new_results: newResults.map((item) => mapResultToCandidate(item, runId, resolvedQuery)),
-    all_results: allResults.map((item) => mapResultToCandidate(item, runId, resolvedQuery)),
-    shortlist: Array.isArray(payload?.shortlist) ? payload.shortlist : [],
-    source_meta: {
-      source_breakdown: sourceBreakdown,
-    },
-    summary: '',
-  };
-
-  return {
-    payload,
-    runId,
-    resolvedQuery,
-    results,
-    sourceBreakdown,
-    sourcingPayload,
-  };
-}
-
-export { normalizeSourceType, joinUrl };