@kevisual/cnb 0.0.76 → 0.0.78

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kevisual/cnb",
-  "version": "0.0.76",
+  "version": "0.0.78",
   "description": "",
   "main": "index.js",
   "basename": "/root/cnb",
@@ -48,7 +48,7 @@
     "ai": "^6.0.158",
     "commander": "^14.0.3",
     "dayjs": "^1.11.20",
-    "dotenv": "^17.4.1",
+    "dotenv": "^17.4.2",
     "fast-glob": "^3.3.3",
     "zod": "^4.3.6"
   },

package/src/cnb-core.ts

@@ -7,7 +7,8 @@ export type CNBCoreOptions<T = {}> = {
   cnb?: CNBCore;
   cors?: {
     baseUrl?: string
-  }
+  },
+  openapi?: boolean;
 } & T;
 
 export type RequestOptions = {
@@ -17,20 +18,24 @@ export type RequestOptions = {
   body?: any;
   params?: Record<string, any>;
   headers?: Record<string, any>;
+  token?: string;
   useCookie?: boolean;
   useOrigin?: boolean;
 };
-const API_BASER_URL = 'https://api.cnb.cool'
+const API_BASE_URL = 'https://api.cnb.cool'
 const API_HACK_URL = 'https://cnb.cool'
+const API_BASE_OPEN_URL = 'https://cn.cool/openapi'
 export class CNBCore {
-  baseURL = API_BASER_URL;
+  baseURL = API_BASE_URL;
   hackURL = API_HACK_URL;
   public token: string;
   public cookie?: string;
   isCors: boolean;
+  openapi?: boolean;
   constructor(options: CNBCoreOptions) {
     this.token = options.token;
     this.cookie = options.cookie;
+    this.openapi = options.openapi;
     if (options?.cnb) {
       if (!options.token) {
         this.token = options.cnb.token;
@@ -39,14 +44,20 @@ export class CNBCore {
         this.cookie = options.cnb.cookie;
       }
     }
+    if (options?.openapi) {
+      this.baseURL = API_BASE_OPEN_URL;
+    }
     if (options?.cors?.baseUrl) {
-      this.baseURL = options.cors.baseUrl + '/' + API_BASER_URL.replace('https://', '');
-      this.hackURL = options.cors.baseUrl + '/' + API_HACK_URL.replace('https://', '');
+      this.baseURL = options.cors.baseUrl + '/' + this.baseURL.replace('https://', '');
+      this.hackURL = options.cors.baseUrl + '/' + this.hackURL.replace('https://', '');
     }
     this.isCors = !!options?.cors?.baseUrl;
   }
 
-  async request({ url, method = 'GET', data, params, headers, body, useCookie, useOrigin }: RequestOptions): Promise<any> {
+  async request({ url, method = 'GET', data, params, headers, body, useCookie, useOrigin, ...rest }: RequestOptions): Promise<any> {
+    if (!url.startsWith('http')) {
+      url = this.makeUrl(url);
+    }
     const defaultHeaders: Record<string, string> = {
       'Content-Type': 'application/json',
       // 'Accept': 'application/json, application/vnd.cnb.api+json, application/vnd.cnb.web+json',
@@ -55,10 +66,13 @@ export class CNBCore {
     if (this.token) {
       defaultHeaders['Authorization'] = `Bearer ${this.token}`;
     }
+    // 如果请求参数中有 token，则覆盖默认 token
+    if (rest.token) {
+      defaultHeaders['Authorization'] = `Bearer ${rest.token}`;
+    }
     if (params) {
       const queryString = new URLSearchParams(params).toString();
       url += `?${queryString}`;
-      defaultHeaders['Accept'] = 'application/json';
     }
     const _headers = { ...defaultHeaders, ...headers };
     let _body = undefined;
@@ -142,10 +156,11 @@ export class CNBCore {
     return this.request({ url: fullUrl, method: 'PATCH', ...REST });
   }
   /**
- * 通过 PUT 请求上传文件内容
- * @param data 包含 URL、token 和文件内容
- * @returns 上传结果
- */
+   * 通过 PUT 请求上传文件内容
+   * assets上传的时候，第一次会获取到一个上传 URL 和 token，然后使用这个方法将文件内容上传到指定的 URL
+   * @param data 包含 URL、token 和文件内容
+   * @returns 上传结果
+   */
   async putFile(data: { url: string, token: string, content: string | Buffer }): Promise<any> {
     return this.request({
       url: data.url,

package/src/git/index.ts

@@ -307,7 +307,7 @@ export class Git extends CNBCore {
    * @param params 查询参数
    * @returns 文件或目录内容
    */
-  async getContent(repo: string, params?: GetContentParams): Promise<Result<Content>> {
+  async getContent(repo: string, params?: GetContentParams): Promise<Result<FileListContent>> {
     const url = `/${repo}/-/git/contents`;
     return this.get({ url, params });
   }
@@ -319,11 +319,46 @@ export class Git extends CNBCore {
    * @param params 查询参数
    * @returns 文件内容
    */
-  async getContentWithPath(repo: string, filePath: string, params?: GetContentWithPathParams): Promise<Result<Content>> {
+  async getContentWithPath(repo: string, filePath: string, params?: GetContentWithPathParams): Promise<Result<FileListContent>> {
     const url = `/${repo}/-/git/contents/${filePath}`;
     return this.get({ url, params });
   }
 
+  async getAllContentWithPath(repo: string, filePath: string = '', opts: { recursive?: boolean } & GetContentParams): Promise<FileListContent['entries']> {
+    try {
+      const { recursive, ...rest } = opts;
+      const res = await this.getContentWithPath(repo, filePath, rest);
+      if (res.code !== 200) {
+        console.log('获取文件列表失败', repo, filePath, res);
+        return [];
+      } else if (!res.data) {
+        console.log('列表为空', res);
+        return [];
+      }
+      const fileList = res.data as unknown as FileListContent;
+      const entries = fileList.entries || [];
+      console.log('文件列表', fileList);
+      const treeList = entries.filter(item => item.type === 'tree');
+      const fileListResult = entries.filter(item => item.type === 'blob');
+      if (recursive) {
+        for (const tree of treeList) {
+          let treePath = tree.path;
+          if (tree.path.includes('/')) {
+            treePath = tree.name;
+          }
+          let newPath = filePath ? `${filePath}/${treePath}` : treePath;
+          const subFileList = await this.getAllContentWithPath(`${repo}`, newPath, { recursive });
+          fileListResult.push(...subFileList);
+        }
+      }
+      return fileListResult;
+    }
+    catch (error) {
+      console.error('获取仓库文件列表失败', error);
+      return [];
+    }
+  }
+
   /**
    * 获取原始文件内容
    * @param repo 仓库名称，格式：组织名称/仓库名称
@@ -674,3 +709,20 @@ type PutTagAnnotationsData = {
   /** 注释键值对 */
   annotations: Record<string, string>;
 };
+
+
+type FileListContent = {
+  type: 'blob' | 'tree';
+  size: number;
+  path: string;
+  name: string;
+  sha: string;
+  url: string;
+  entries?: FIleEntry[];
+}
+type FIleEntry = {
+  type: 'blob' | 'tree';
+  sha: string;
+  path: string;
+  name: string;
+}

package/src/knowledge/index.ts

@@ -1,4 +1,4 @@
-import { CNBCore, CNBCoreOptions, Result } from "../cnb-core.ts";
+import { CNBCore, CNBCoreOptions, RequestOptions, Result } from "../cnb-core.ts";
 
 // https://cnb.cool/cnb/plugins/cnbcool/knowledge-base/-/blob/main/src/api.py
 export class KnowledgeBase extends CNBCore {
@@ -27,31 +27,56 @@ export class KnowledgeBase extends CNBCore {
   }
   deleteBase(repo: string): Promise<Result<any>> {
     const url = `/${repo}/-/knowledge/base`;
-    return this.request({ url, method: 'DELETE' });
+    return this.delete({ url });
   }
 
   /**
    * 未暴露
+   * 
+   * 创建知识库接口，cnb 界面操作定制模块功能依赖该接口实现
    * @param repo 
    * @param data 
    * @returns 
    */
-  createKnowledgeBase(repo: string, data: {
-    embedding_model_name: string;
+
+  createKnowledgeBase(repo: string, params: {
+    /**
+     * hunyuan | bge-m3
+     */
+    model_name: string;
     include: string;
     exclude: string;
+    chunk_size: number;
+    chunk_overlap: number;
+    text_separator: string;
     issue_sync_enabled?: boolean;
-    processing?: {
-      chunk_size: number;
-      chunk_overlap: number;
-      text_separator: string;
-    };
-    issue?: {
-      labels: string;
-      state: string;
-    };
+    issue_labels?: string;
+    issue_exclude_labels?: string;
+    issue_state?: string;
   }): Promise<Result<any>> {
     const url = `/${repo}/-/knowledge/base`;
+    const metadata: any = {
+      processing: {
+        chunk_size: params.chunk_size,
+        chunk_overlap: params.chunk_overlap,
+        text_separator: params.text_separator
+      },
+      version: "1.0"
+    };
+    if (params.issue_sync_enabled) {
+      metadata.issue = {
+        labels: params.issue_labels || "",
+        exclude_labels: params.issue_exclude_labels || "",
+        state: params.issue_state || ""
+      };
+    }
+    const data = {
+      embedding_model_name: params.model_name,
+      include: params.include,
+      exclude: params.exclude,
+      issue_sync_enabled: params.issue_sync_enabled || false,
+      metadata
+    };
     return this.post({ url, data });
   }
   /**
@@ -74,22 +99,27 @@ export class KnowledgeBase extends CNBCore {
    * @param text 
    * @returns 
    */
-  getEmbedding(repo: string, text: string): Promise<Result<{ embedding: number[] }>> {
+  getEmbedding(repo: string, text: string): Promise<Result<{ embeddings: number[] }>> {
     const url = `/${repo}/-/knowledge/embedding`;
     return this.post({ url, data: { text } });
   }
   /**
-   *  未暴露
+   * 未暴露 
+   * 只能运行在流水线，使用流水线的CNB_TOKEN去使用
+   * 否则会报错：sn not found in meta
+   * opts的token必须是流水线的CNB_TOKEN，不能是用户的token
    * @param repo 
    * @param chunksData 
    * @returns 
    */
-  addDocument(repo: string, chunksData: {
-    path: string;
-    chunks: Array<{
-      content: string;
-      hash: string;
-      offset: number;
+  addDocument(repo: string, AddDocument: AddDocument, opts?: RequestOptions): Promise<Result<null>> {
+    const url = `/${repo}/-/knowledge/documents/upsert-document-with-chunks`;
+    return this.post({ url, data: AddDocument, ...opts });
+  }
+  /**
+   * 未暴露
+   * @param repo 
+   * @param paths 
       size: number;
     }>;
   }): Promise<Result<any>> {
@@ -114,7 +144,7 @@ export class KnowledgeBase extends CNBCore {
    * @param page_size 
    * @returns 
    */
-  listDocument(repo: string, page: number = 1, page_size: number = 50): Promise<Result<any[]>> {
+  listDocuments(repo: string, page: number = 1, page_size: number = 50): Promise<Result<any[]>> {
     const url = `/${repo}/-/knowledge/documents`;
     return this.get({ url, params: { page, page_size } });
   }
@@ -143,4 +173,33 @@ type QueryRag = {
     type: string; // code, text
     url: string;
   }
+}
+
+type AddDocument = {
+  // 文档路径，必填， 例如：docs/xxx.md
+  path: string;
+  // 文件名 xxx
+  name: string;
+  // 文件扩展名，例如：md
+  extension: string;
+  // 文件大小，单位：字节
+  size: number;
+  // 文件内容的 hash,(hashlib.md5(content.encode('utf-8')).hexdigest())
+  hash: string;
+  type: "code" | "issue"; // code, pdf
+  // 类似 https://cnb.cool/kevisual/starred-auto/-/blob/62ce3d724f6e5f2cfd4b84a1df7cf4a6eaf441d6/docs/11730342.md
+  url: string;
+  chunks: Array<{
+    // 文档的分块的内容
+    "content": string,
+    "hash": string,
+    "position": number,
+    "embedding": number[]
+  }>;
+  // 处理当前批次的索引， 从0开始
+  batch_index: number;
+  // 当前批次的总数, 必须大于0
+  batch_count: number;
+  // 是否是最后一批
+  is_last_batch: boolean;
 }

package/src/labels/repo-label.ts

@@ -151,6 +151,22 @@ export class RepoLabel extends CNBCore {
     const url = `/${repo}/-/labels/${encodeURIComponent(name)}`;
     return this.delete({ url });
   }
+  async removeLabels(opts: { repo: string, labels: string[] }): Promise<Result> {
+    const { repo, labels } = opts;
+    try {
+      for (const label of labels) {
+        const res = await this.remove(repo, label);
+        await new Promise(resolve => setTimeout(resolve, 100)); // 避免请求过快导致服务器拒绝服务
+        if (res.code !== 200) {
+          console.error(`删除标签失败 ${label}:`, res.message);
+          break;
+        }
+      }
+      return { code: 200, data: labels, message: '删除标签成功' };
+    } catch (error) {
+      return { code: 500, message: '删除标签失败', data: error };
+    }
+  }
 }
 
 type ListLabelsParams = {

package/src/repo/download.ts

@@ -1,4 +1,8 @@
-type DownloadFn = (url: string, opts?: { headers?: any, hash?: string, token?: string }) => Promise<{ code: number; buffer: Buffer | null }>
+type DownloadFn = (url: string, opts?: { headers?: any, hash?: string, token?: string, stream?: boolean }) => Promise<{
+  code: number;
+  buffer: Buffer | null;
+  stream?: ReadableStream<Uint8Array> | null;
+}>;
 
 /**
  * 下载文件内容，使用CNB API，支持文件未修改时返回304状态码
@@ -17,6 +21,7 @@ export const downloadByCNBApi: DownloadFn = async (url, opts) => {
   const res = await fetch(url, {
     headers: headers
   });
+  const stream = opts?.stream || false;
   type CNBResponse = {
     type?: 'blob' | 'file' | 'dir' | 'lfs';
     size?: number;
@@ -48,6 +53,15 @@ export const downloadByCNBApi: DownloadFn = async (url, opts) => {
           return { code: 304, buffer: null };
         }
         const buffer = Buffer.from(resJson.content, 'base64');
+        if (stream) {
+          const readableStream = new ReadableStream<Uint8Array>({
+            start(controller) {
+              controller.enqueue(buffer);
+              controller.close();
+            }
+          });
+          return { code: 200, buffer: null, stream: readableStream };
+        }
         return { code: 200, buffer };
       }
     }
@@ -63,6 +77,10 @@ export const downloadByCNBApi: DownloadFn = async (url, opts) => {
             // 文件未修改，跳过下载
             return { code: 304, buffer: null };
           }
+          if (stream) {
+            const readableStream = lfsRes.body;
+            return { code: 200, buffer: null, stream: readableStream };
+          }
           const arrayBuffer = await lfsRes.arrayBuffer();
           return { code: 200, buffer: Buffer.from(arrayBuffer) };
         }
@@ -75,4 +93,73 @@ export const downloadByCNBApi: DownloadFn = async (url, opts) => {
     console.error(`下载失败 ${url}: ${res.statusText}`, resJson);
     return { code: res.status, buffer: null };
   }
+}
+
+
+export const downloadResource: DownloadFn = async (url, opts) => {
+  const headers = opts?.headers || {};
+  const res = await fetch(url, { headers });
+  const stream = opts?.stream || false;
+  const etag = res.headers.get('ETag');
+  if (opts?.hash && etag === opts.hash) {
+    // 文件未修改，跳过下载
+    return { code: 304, buffer: null };
+  }
+  if (res.ok) {
+    if (stream) {
+      const readableStream = res.body;
+      return { code: 200, buffer: null, stream: readableStream };
+    }
+    const arrayBuffer = await res.arrayBuffer();
+    return { code: 200, buffer: Buffer.from(arrayBuffer) };
+  }
+  console.error(`下载失败 ${url}: ${res.statusText}`);
+  return { code: res.status, buffer: null };
+}
+
+export const isCNBOpenApi = (url: string) => { return url.startsWith('https://api.cnb.cool') }
+
+export const download: DownloadFn = async (url, opts) => {
+  const isCNB = isCNBOpenApi(url);
+  if (isCNB) {
+    return await downloadByCNBApi(url, opts);
+  } else {
+    return await downloadResource(url, opts);
+  }
+}
+
+export const getLfsDownloadUrl = async (url: string, opts?: { headers?: any, token?: string }) => {
+  const headers = {
+    Accept: "application/vnd.cnb.api+json",
+    ...(opts?.token ? { Authorization: `${opts.token}` } : {}),
+    ...opts?.headers
+  }
+  const res = await fetch(url, {
+    headers: headers
+  });
+  type CNBResponse = {
+    type?: 'blob' | 'file' | 'dir' | 'lfs';
+    size?: number;
+    path: string;
+    name: string;
+    sha?: string;
+    encoding?: string;
+    content?: string;
+
+    lfs_oid?: string;
+    lfs_size?: number;
+    lfs_download_url?: string;
+  }
+  let resJson: CNBResponse;
+  try {
+    resJson = await res.json() as CNBResponse;
+  } catch (e) {
+    console.error(`获取LFS下载链接失败 ${url}: ${res.statusText}`, e);
+    return null;
+  }
+  if (res.ok && resJson.type === 'lfs' && resJson.lfs_download_url) {
+    return resJson.lfs_download_url;
+  }
+  console.error(`获取LFS下载链接失败 ${url}: ${res.statusText}`, resJson);
+  return null;
 }