nolo-cli 0.1.26 → 0.1.29

package/chat/messages/fetchAndCacheMessages.ts

@@ -0,0 +1,119 @@
+import { Message } from "./types";
+import { fetchMessages as fetchLocalMessages } from "./fetchMessages";
+import { fetchConvMsgs } from "./fetchConvMsgs";
+import { swallowNonAbortError } from "../../app/utils/async";
+import { createKey } from "../../database/keys";
+import { DataType } from "../../create/types";
+import { isTombstoneRecord, shouldReplaceWithNextRecord } from "../../database/tombstones";
+
+type MessageRecord = Message & { deletedAt?: string };
+
+interface FetchAndCacheOptions {
+    db: any; // Local LevelDB instance
+    dialogId: string;
+    dialogKey?: string;
+    limit?: number;
+    beforeKey?: string;
+    token?: string | null;
+    remoteServers?: string[];
+    signal?: AbortSignal;
+}
+
+export const fetchAndCacheMessages = async ({
+    db,
+    dialogId,
+    dialogKey,
+    limit = 50,
+    beforeKey,
+    token,
+    remoteServers = [],
+    signal,
+}: FetchAndCacheOptions): Promise<Message[]> => {
+    // 1. 本地查询
+    const localPromise = fetchLocalMessages(db, dialogId, {
+        limit,
+        beforeKey,
+        throwOnError: false,
+        includeDeleted: true,
+    }).catch(() => [] as MessageRecord[]);
+
+    // 2. 远程查询
+    const remotePromise = (async () => {
+        if (!token || remoteServers.length === 0) return [] as MessageRecord[];
+
+        const results = await Promise.all(
+            remoteServers.map((server) =>
+                swallowNonAbortError(
+                    fetchConvMsgs(
+                        server,
+                        token,
+                        { dialogId, dialogKey, limit, beforeKey },
+                        { signal }
+                    ),
+                    [] as MessageRecord[],
+                    undefined
+                )
+            )
+        );
+        return results.flat() as MessageRecord[];
+    })();
+
+    const [localMsgs, remoteMsgs] = await Promise.all([
+        localPromise,
+        remotePromise,
+    ]);
+
+    const uniqueMap = new Map<string, MessageRecord>();
+
+    // 先放入本地消息
+    localMsgs.forEach((m) => uniqueMap.set(m.id, m));
+
+    const changedMessagesToCache = new Map<string, MessageRecord>();
+
+    // 4. 合并远程消息，并标记需要缓存的新消息
+    remoteMsgs.forEach((m) => {
+        if (!m || !m.id) return;
+
+        const existing = uniqueMap.get(m.id);
+        if (!existing || shouldReplaceWithNextRecord(m, existing)) {
+            uniqueMap.set(m.id, m);
+            changedMessagesToCache.set(m.id, m);
+        }
+    });
+
+    // 5. 将缺失的消息回写到本地 DB
+    if (changedMessagesToCache.size > 0) {
+        try {
+            const ops = Array.from(changedMessagesToCache.values()).map((msg) => {
+                let key = msg.dbKey || (msg as any)._key;
+
+                if (!key) {
+                    key = createKey(DataType.DIALOG, dialogId, "msg", msg.id);
+                }
+
+                return {
+                    type: "put",
+                    key,
+                    value: {
+                        ...msg,
+                        dbKey: key,
+                        type: DataType.MSG,
+                    },
+                };
+            });
+
+            await db.batch(ops);
+        } catch (_error) {
+            // Silently ignore cache write errors
+        }
+    }
+
+    // 6. 排序并返回
+    return Array.from(uniqueMap.values())
+        .filter((message) => !isTombstoneRecord(message))
+        .sort((a, b) => {
+            const tA = new Date((a as any).createdAt || 0).getTime();
+            const tB = new Date((b as any).createdAt || 0).getTime();
+            return tB - tA; // 降序 (Newest First)
+        });
+};

package/chat/messages/fetchConvMsgs.ts

@@ -0,0 +1,51 @@
+const FETCH_TIMEOUT = 5000;
+
+export const fetchConvMsgs = async (
+  server,
+  token,
+  { dialogId, dialogKey, limit, beforeKey },
+  options = {}
+) => {
+  const { signal: externalSignal } = options;
+
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), FETCH_TIMEOUT);
+  const onExternalAbort = () => controller.abort();
+  externalSignal?.addEventListener("abort", onExternalAbort);
+
+  try {
+    const response = await fetch(`${server}/rpc/getConvMsgs`, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${token}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        dialogId,
+        ...(dialogKey && { dialogKey }),
+        limit,
+        ...(beforeKey && { beforeKey }),
+      }),
+      signal: controller.signal,
+    });
+    clearTimeout(timeoutId);
+    externalSignal?.removeEventListener("abort", onExternalAbort);
+
+    if (!response.ok) {
+      console.error(`fetchConvMsgs: Failed ${response.status} from ${server}`);
+      return [];
+    }
+
+    const data = await response.json();
+    return Array.isArray(data) ? data : [];
+  } catch (error) {
+    clearTimeout(timeoutId);
+    externalSignal?.removeEventListener("abort", onExternalAbort);
+    // Only re-throw when external signal was aborted (user navigated away)
+    if (externalSignal?.aborted) {
+      throw error;
+    }
+    console.error(`fetchConvMsgs: Error fetching from ${server}:`, error);
+    return [];
+  }
+};

package/chat/messages/fetchMessages.ts

@@ -0,0 +1,84 @@
+// chat/messages/fetchMessages.ts
+import type { Message } from "./types"; // 确保引入你的 Message 类型定义
+import { isTombstoneRecord } from "../../database/tombstones";
+
+// 定义返回类型，包含数据库的 key
+export type MessageWithKey = Message & { _key: string };
+
+/**
+ * 获取指定对话的消息，支持分页加载。
+ * @param db 数据库实例。
+ * @param dialogId 对话 ID。
+ * @param options 配置选项 (limit?, beforeKey?, throwOnError?)。
+ * @returns Promise<MessageWithKey[]> 消息数组 (带 key, 默认最新在前)。
+ */
+export const fetchMessages = async (
+  db: any, // TODO: 替换为具体的数据库实例类型
+  dialogId: string,
+  options: {
+    limit?: number;
+    beforeKey?: string | null;
+    throwOnError?: boolean;
+    includeDeleted?: boolean;
+  } = {}
+): Promise<MessageWithKey[]> => {
+  const { limit = 30, beforeKey = null, throwOnError = false, includeDeleted = false } = options; // 分页默认 limit 30
+
+  // --- 输入验证 ---
+  if (!dialogId || typeof dialogId !== "string") {
+    const errorMsg = "fetchMessages: dialogId 必须是一个非空字符串";
+    if (throwOnError) throw new Error(errorMsg);
+    console.error(errorMsg);
+    return [];
+  }
+  // 其他验证 (limit, beforeKey) 可以根据需要添加
+
+  // --- 准备查询 ---
+  const messages: MessageWithKey[] = [];
+  const prefix = `dialog-${dialogId}-msg-`;
+
+  const iteratorOptions: any = {
+    // TODO: 替换为具体的迭代器选项类型
+    gte: prefix,
+    reverse: true, // 总是从新到旧获取
+    limit: limit,
+  };
+
+  if (beforeKey) {
+    // 加载更早的消息: Key < beforeKey
+    iteratorOptions.lt = beforeKey;
+  } else {
+    // 加载最新的消息: Key <= prefix + '\uffff'
+    iteratorOptions.lte = prefix + "\uffff";
+  }
+
+  // --- 执行查询 ---
+  try {
+    let iterator = db.iterator(iteratorOptions);
+    // 处理 react-native-leveldb 可能返回 Promise 的情况
+    if (iterator && typeof iterator.then === 'function') {
+      iterator = await iterator;
+    }
+
+
+    let count = 0;
+    for await (const [key, value] of iterator) {
+      count++;
+      if (!includeDeleted && isTombstoneRecord(value)) {
+        continue;
+      }
+      // 基本数据校验
+      if (value && typeof value === "object" && value.id && value.createdAt) {
+        // **关键: 将数据库 key 作为 _key 附加到消息对象上**
+        messages.push({ ...(value as Message), _key: key });
+      }
+    }
+
+
+    return messages; // 返回的消息数组是 newest first
+  } catch (error) {
+    console.error(`fetchMessages: 获取对话 ${dialogId} 消息失败:`, error);
+    if (throwOnError) throw error;
+    return [];
+  }
+};

package/chat/messages/fileUrl.ts

@@ -0,0 +1,13 @@
+import {
+  buildDatabaseFileContentUrl,
+  isLocalDatabaseFileContentUrl,
+} from "../../database/fileUrl";
+
+export const buildMessageFileContentUrl = (
+  serverOrigin: string | undefined | null,
+  fileId: string | undefined | null
+): string | null => buildDatabaseFileContentUrl(serverOrigin, fileId);
+
+export const isLocalFileContentUrl = (
+  url: string | null | undefined
+): boolean => isLocalDatabaseFileContentUrl(url);

package/chat/messages/generateMessageKey.ts

@@ -0,0 +1,21 @@
+type MessageType = "text" | "image" | "audio" | "file";
+interface Message {
+  id: string;
+  dialogId: string;
+  senderId: string;
+  content: string;
+  type: MessageType;
+  timestamp: number;
+}
+const PREFIXES = {
+  MESSAGE: "msg",
+  DIALOG: "dlg",
+  USER: "usr",
+} as const;
+const extractAutoIncrId = (messageId: string): string => {
+  const [_, autoIncrId] = messageId.split("-");
+  return autoIncrId;
+};
+
+export const generateMessageKey = (message: Message): string =>
+  `${PREFIXES.MESSAGE}-${message.dialogId}-${message.timestamp}-${extractAutoIncrId(message.id)}`;

package/chat/messages/hooks/useBase64Migration.ts

@@ -0,0 +1,115 @@
+import { useEffect, useRef } from "react";
+import { selectRuntimeCurrentServer } from "../../../app/stateViews/runtime";
+import { useAppDispatch, useAppSelector } from "../../../app/store";
+import { upload, patch } from "../../../database/dbSlice";
+import { dataURLtoFile } from "../../../app/utils/imageUtils";
+import { buildMessageFileContentUrl } from "../fileUrl";
+
+/**
+ * 自动将消息中的 base64 图片迁移为远程文件 URL
+ * 只在非只读模式下执行，每条消息最多执行一次
+ */
+export function useBase64Migration(message: any | null) {
+  const dispatch = useAppDispatch();
+  const currentServer = useAppSelector(selectRuntimeCurrentServer);
+  const migratedRef = useRef(false);
+
+  useEffect(() => {
+    if (!message) return;
+    if (migratedRef.current) return;
+    if (!message?.id || !message.dbKey || !currentServer) return;
+
+    const rawContent = message.content;
+    if (!Array.isArray(rawContent)) {
+      migratedRef.current = true;
+      return;
+    }
+
+    const base64Items = rawContent.reduce(
+      (
+        acc: { index: number; dataUrl: string }[],
+        part: any,
+        index: number
+      ) => {
+        const url = part?.image_url?.url;
+        if (
+          part?.type === "image_url" &&
+          typeof url === "string" &&
+          url.startsWith("data:")
+        ) {
+          acc.push({ index, dataUrl: url });
+        }
+        return acc;
+      },
+      []
+    );
+
+    if (!base64Items.length) {
+      migratedRef.current = true;
+      return;
+    }
+
+    let cancelled = false;
+
+    const migrate = async () => {
+      let currentContent: any[] = rawContent;
+
+      for (const { index, dataUrl } of base64Items) {
+        if (cancelled) break;
+
+        const file = dataURLtoFile(
+          dataUrl,
+          `msg-img-${message.id}-${index}.png`
+        );
+        if (!file) continue;
+
+        try {
+          const metadata = await dispatch(
+            upload({
+              file,
+              customKey: `msg-img-${message.id}-${index}`,
+            }) as any
+          ).unwrap();
+
+          const fileId = metadata?.id as string | undefined;
+          if (!fileId) continue;
+
+          const remoteUrl = buildMessageFileContentUrl(currentServer, fileId);
+          if (!remoteUrl) continue;
+
+          const newContent = currentContent.map((p, i) =>
+            i === index
+              ? {
+                ...p,
+                image_url: {
+                  ...(p.image_url || {}),
+                  url: remoteUrl,
+                },
+                original_data_url: dataUrl,
+              }
+              : p
+          );
+
+          await dispatch(
+            patch({
+              dbKey: message.dbKey,
+              changes: { content: newContent },
+            }) as any
+          ).unwrap();
+
+          currentContent = newContent;
+        } catch {
+          // 单个图片迁移失败时忽略，尝试处理后续图片
+        }
+      }
+
+      migratedRef.current = true;
+    };
+
+    void migrate();
+
+    return () => {
+      cancelled = true;
+    };
+  }, [message?.id, message?.dbKey, currentServer, dispatch]);
+}

package/chat/messages/messageAgent.ts

@@ -0,0 +1,8 @@
+import type { Message } from "./types";
+
+export function resolveMessageAgentKey(
+  message: Partial<Message> | null | undefined
+): string | undefined {
+  if (!message) return undefined;
+  return message.agentKey || message.cybotKey;
+}

package/chat/messages/messageContent.ts

@@ -0,0 +1,199 @@
+// 文件路径: packages/chat/messages/messageContent.ts
+
+/*
+ * ==================================================================
+ *  /chat/messages/messageContent.ts
+ *  - 负责处理 AI 流式内容 & 图片上传
+ * ==================================================================
+ */
+
+import type { RootState } from "../../app/store";
+import { upload } from "../../database/dbSlice";
+import { getRuntimeServerContext } from "../../database/runtimeServerContext";
+import { buildMessageFileContentUrl } from "./fileUrl";
+
+import { ContentType } from "../../app/types";
+import { addContentAction } from "../../create/space/content/addContentAction";
+
+/** 判断 URL 是否为 data:image 开头的 dataURL */
+const isDataUrlImage = (url: unknown): url is string =>
+  typeof url === "string" && url.startsWith("data:image");
+
+/**
+ * 将生成的 dataURL 图片上传为文件，并返回最终 URL。
+ * 失败时返回原始 dataURL，避免打断主流程。
+ */
+const uploadGeneratedImageDataUrl = async (
+  url: string,
+  dialogId: string,
+  messageId: string,
+  index: number,
+  dispatch: any,
+  getState: () => RootState,
+  options?: { spaceId?: string; agentName?: string }
+): Promise<string> => {
+  const fileName = `generated-image-${dialogId}-${messageId}-${index}.png`;
+  const { dataURLtoFile, waitForFileReady } = await import("../../app/utils/imageUtils");
+  const file = dataURLtoFile(url, fileName);
+
+  if (!file) {
+    console.warn(
+      "[messageContent] dataURLtoFile failed, keep dataURL for generated image"
+    );
+    return url;
+  }
+
+  const customKey = `generated-image-${dialogId}-${messageId}-${index}`;
+
+  let metadata: any;
+  try {
+    metadata = await dispatch(upload({ file, customKey }) as any).unwrap();
+  } catch (err) {
+    console.warn(
+      "[messageContent] upload generated image failed, keep dataURL:",
+      err
+    );
+    return url;
+  }
+
+  const state = getState() as RootState;
+  const { currentServer } = getRuntimeServerContext(state);
+  const fileId = (metadata?.dbKey || metadata?.id) as string | undefined;
+
+  if (!currentServer || !fileId) {
+    console.warn(
+      "[messageContent] missing currentServer or fileId, keep dataURL",
+      { currentServer, fileId }
+    );
+    return url;
+  }
+
+  // 如果提供了 spaceId，尝试保存到空间内容
+  if (options?.spaceId) {
+    try {
+      const agentPrefix = options.agentName ? `[${options.agentName}] ` : "";
+      const title = `${agentPrefix}Generated Image ${index + 1}`;
+
+      await addContentAction({
+        spaceId: options.spaceId,
+        contentKey: fileId,
+        title,
+        type: ContentType.IMAGE,
+      }, { dispatch, getState });
+
+      console.log(`[messageContent] Saved generated image to space: ${options.spaceId}`);
+    } catch (err) {
+      console.error("[messageContent] Failed to save generated image to space:", err);
+    }
+  }
+
+  const imageUrl = buildMessageFileContentUrl(currentServer, fileId);
+  if (!imageUrl) {
+    return url;
+  }
+  const ready = await waitForFileReady(imageUrl);
+
+  if (!ready) {
+    console.warn(
+      "[messageContent] uploaded generated image not ready, keep dataURL:",
+      imageUrl
+    );
+    return url;
+  }
+
+  return imageUrl;
+};
+
+/**
+ * 遍历 assistant 的 contentBuffer：
+ * - 对于 type === "image_url" 且 url 是 dataURL 的项：
+ *   - 上传为文件
+ *   - 把 url 替换成正式的文件 URL
+ * - 其他内容保持不变
+ */
+export const normalizeAssistantContentBuffer = async (
+  contentBuffer: any[],
+  dialogId: string,
+  messageId: string,
+  dispatch: any,
+  getState: () => RootState,
+  options?: { spaceId?: string; agentName?: string }
+): Promise<any[]> => {
+  if (!Array.isArray(contentBuffer) || contentBuffer.length === 0) {
+    return contentBuffer;
+  }
+
+  const updated = await Promise.all(
+    contentBuffer.map(async (part, index) => {
+      if (
+        !part ||
+        part.type !== "image_url" ||
+        !part.image_url ||
+        !isDataUrlImage(part.image_url.url)
+      ) {
+        return part;
+      }
+
+      const newUrl = await uploadGeneratedImageDataUrl(
+        part.image_url.url,
+        dialogId,
+        messageId,
+        index,
+        dispatch,
+        getState,
+        options
+      );
+
+      if (newUrl === part.image_url.url) {
+        return part;
+      }
+
+      return {
+        ...part,
+        image_url: {
+          ...part.image_url,
+          url: newUrl,
+        },
+      };
+    })
+  );
+
+  return updated;
+};
+
+/**
+ * 将 Message.content 归一化为纯文本:
+ * - string → 直接使用
+ * - OpenAI 风格数组 → 拼接 text 片段，image_url 替换为占位符
+ * - 其他 → null
+ *
+ * 共享函数：标题生成 / 摘要生成 / 上下文序列化均可复用
+ */
+export const serializeMessageContent = (
+  content: any,
+  imagePlaceholder = "[图片]"
+): string | null => {
+  if (typeof content === "string") {
+    return content.trim() || null;
+  }
+
+  if (Array.isArray(content)) {
+    const fragments: string[] = [];
+
+    for (const part of content) {
+      if (!part || typeof part !== "object") continue;
+
+      if (part.type === "text" && typeof part.text === "string") {
+        const text = part.text.trim();
+        if (text) fragments.push(text);
+      } else if (part.type === "image_url") {
+        fragments.push(imagePlaceholder);
+      }
+    }
+
+    const joined = fragments.join("\n").trim();
+    return joined || null;
+  }
+
+  return null;
+};

package/chat/messages/messageContract.ts

@@ -0,0 +1,88 @@
+import type { MessageContentPart } from "./types";
+
+type MessageLike = {
+  role?: string;
+  content?: string | MessageContentPart[] | null;
+};
+
+export function separateThinkContent(contentBuffer: MessageContentPart[]) {
+  let thinkContent = "";
+  let normalContent = "";
+
+  const combinedText = contentBuffer
+    .filter((c) => c.type === "text" && c.text)
+    .map((c) => c.text)
+    .join("");
+
+  const thinkMatches = combinedText.match(/<think\b[^>]*>(.*?)<\/think>/gis);
+
+  if (thinkMatches) {
+    thinkContent = thinkMatches
+      .map((m) => m.replace(/<think\b[^>]*>|<\/think>/gi, ""))
+      .join("\n\n");
+
+    normalContent = combinedText
+      .replace(/<think\b[^>]*>.*?<\/think>/gis, "")
+      .trim();
+  } else {
+    normalContent = combinedText;
+  }
+
+  return { thinkContent, normalContent };
+}
+
+export function countImageParts(
+  content: string | MessageContentPart[] | null | undefined,
+): number {
+  if (!Array.isArray(content)) return 0;
+  return content.filter((part) => part?.type === "image_url").length;
+}
+
+export function summarizeMessagesForDebug(
+  messages: Array<MessageLike> | null | undefined,
+) {
+  const safeMessages = Array.isArray(messages) ? messages : [];
+  return {
+    count: safeMessages.length,
+    roles: safeMessages.map((message) => message?.role ?? "unknown"),
+    imageMessageCount: safeMessages.filter(
+      (message) => countImageParts(message?.content ?? null) > 0,
+    ).length,
+  };
+}
+
+export function finalizeAssistantMessageContent(
+  normalizedContentBuffer: MessageContentPart[],
+  reasoningBuffer = "",
+) {
+  const { thinkContent: tagThink, normalContent } =
+    separateThinkContent(normalizedContentBuffer);
+
+  const thinkContent = `${tagThink}${reasoningBuffer}`.trim();
+  const hasNonTextParts = normalizedContentBuffer.some(
+    (part) => part && part.type && part.type !== "text",
+  );
+
+  return {
+    thinkContent,
+    textContent: normalContent || "",
+    visibleContent: hasNonTextParts
+      ? normalizedContentBuffer
+      : normalContent || "",
+    hasNonTextParts,
+    imagePartCount: countImageParts(normalizedContentBuffer),
+  };
+}
+
+export function appendSaveFailureToContent(
+  content: string | MessageContentPart[] | null | undefined,
+): string | MessageContentPart[] {
+  const failureText = "[Failed to save message]";
+  if (Array.isArray(content)) {
+    return [...content, { type: "text", text: failureText }];
+  }
+  if (typeof content === "string" && content.length > 0) {
+    return `${content}\n${failureText}`;
+  }
+  return failureText;
+}