@lobehub/chat 1.39.3 → 1.40.0

package/scripts/cdnWorkflow/index.ts

@@ -0,0 +1,217 @@
+import { consola } from 'consola';
+import { writeJSONSync } from 'fs-extra';
+import matter from 'gray-matter';
+import { createHash } from 'node:crypto';
+import { readFileSync, writeFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import pMap from 'p-map';
+
+import { uploader } from './uploader';
+import {
+  changelogIndex,
+  changelogIndexPath,
+  extractHttpsLinks,
+  fetchImageAsFile,
+  mergeAndDeduplicateArrays,
+  posts,
+  root,
+} from './utils';
+
+// 定义常量
+const GITHUB_CDN = 'https://github.com/lobehub/lobe-chat/assets/';
+const CHECK_CDN = [
+  'https://cdn.nlark.com/yuque/0/',
+  'https://s.imtccdn.com/',
+  'https://oss.home.imtc.top/',
+  'https://www.anthropic.com/_next/image',
+  'https://miro.medium.com/v2/',
+  'https://images.unsplash.com/',
+  'https://github.com/user-attachments/assets',
+];
+
+const CACHE_FILE = resolve(root, 'docs', '.cdn.cache.json');
+
+class ImageCDNUploader {
+  private cache: { [link: string]: string } = {};
+
+  constructor() {
+    this.loadCache();
+  }
+
+  // 从文件加载缓存数据
+  private loadCache() {
+    try {
+      this.cache = JSON.parse(readFileSync(CACHE_FILE, 'utf8'));
+    } catch (error) {
+      consola.error('Failed to load cache', error);
+    }
+  }
+
+  // 将缓存数据写入文件
+  private writeCache() {
+    try {
+      writeFileSync(CACHE_FILE, JSON.stringify(this.cache, null, 2));
+    } catch (error) {
+      consola.error('Failed to write cache', error);
+    }
+  }
+
+  // 收集所有的图片链接
+  private collectImageLinks(): string[] {
+    const links: string[][] = posts.map((post) => {
+      const mdx = readFileSync(post, 'utf8');
+      const { content, data } = matter(mdx);
+      let inlineLinks: string[] = extractHttpsLinks(content);
+
+      // 添加特定字段中的图片链接
+      if (data?.image) inlineLinks.push(data.image);
+      if (data?.seo?.image) inlineLinks.push(data.seo.image);
+
+      // 过滤出有效的 CDN 链接
+      return inlineLinks.filter(
+        (link) =>
+          (link.startsWith(GITHUB_CDN) || CHECK_CDN.some((cdn) => link.startsWith(cdn))) &&
+          !this.cache[link],
+      );
+    });
+
+    const communityLinks = changelogIndex.community
+      .map((post) => post.image)
+      .filter(
+        (link) =>
+          link &&
+          (link.startsWith(GITHUB_CDN) || CHECK_CDN.some((cdn) => link.startsWith(cdn))) &&
+          !this.cache[link],
+      ) as string[];
+
+    const cloudLinks = changelogIndex.cloud
+      .map((post) => post.image)
+      .filter(
+        (link) =>
+          link &&
+          (link.startsWith(GITHUB_CDN) || CHECK_CDN.some((cdn) => link.startsWith(cdn))) &&
+          !this.cache[link],
+      ) as string[];
+
+    // 合并和去重链接数组
+    return mergeAndDeduplicateArrays(links.flat().concat(communityLinks, cloudLinks));
+  }
+
+  // 上传图片到 CDN
+  private async uploadImagesToCDN(links: string[]) {
+    const cdnLinks: { [link: string]: string } = {};
+
+    await pMap(links, async (link) => {
+      consola.start('Uploading image to CDN', link);
+      const file = await fetchImageAsFile(link, 1600);
+
+      if (!file) {
+        consola.error('Failed to fetch image as file', link);
+        return;
+      }
+
+      const cdnUrl = await this.uploadFileToCDN(file, link);
+      if (cdnUrl) {
+        consola.success(link, '>>>', cdnUrl);
+        cdnLinks[link] = cdnUrl.replaceAll(process.env.DOC_S3_PUBLIC_DOMAIN || '', '');
+      }
+    });
+
+    // 更新缓存
+    this.cache = { ...this.cache, ...cdnLinks };
+    this.writeCache();
+  }
+
+  // 根据不同的 CDN 来处理文件上传
+  private async uploadFileToCDN(file: File, link: string): Promise<string | undefined> {
+    if (link.startsWith(GITHUB_CDN)) {
+      const filename = link.replaceAll(GITHUB_CDN, '');
+      return uploader(file, filename);
+    } else if (CHECK_CDN.some((cdn) => link.startsWith(cdn))) {
+      const buffer = await file.arrayBuffer();
+      const hash = createHash('md5').update(Buffer.from(buffer)).digest('hex');
+      return uploader(file, hash);
+    }
+
+    return;
+  }
+
+  // 替换文章中的图片链接
+  private replaceLinksInPosts() {
+    let count = 0;
+
+    for (const post of posts) {
+      const mdx = readFileSync(post, 'utf8');
+      let { content, data } = matter(mdx);
+      const inlineLinks = extractHttpsLinks(content);
+
+      for (const link of inlineLinks) {
+        if (this.cache[link]) {
+          content = content.replaceAll(link, this.cache[link]);
+          count++;
+        }
+      }
+
+      // 更新特定字段的图片链接
+
+      if (data['image'] && this.cache[data['image']]) {
+        data['image'] = this.cache[data['image']];
+        count++;
+      }
+
+      if (data['seo']?.['image'] && this.cache[data['seo']?.['image']]) {
+        data['seo']['image'] = this.cache[data['seo']['image']];
+        count++;
+      }
+
+      writeFileSync(post, matter.stringify(content, data));
+    }
+
+    consola.success(`${count} images have been uploaded to CDN and links have been replaced`);
+  }
+
+  private replaceLinksInChangelogIndex() {
+    let count = 0;
+    changelogIndex.community = changelogIndex.community.map((post) => {
+      if (!post.image) return post;
+      count++;
+      return {
+        ...post,
+        image: this.cache[post.image] || post.image,
+      };
+    });
+
+    changelogIndex.cloud = changelogIndex.cloud.map((post) => {
+      if (!post.image) return post;
+      count++;
+      return {
+        ...post,
+        image: this.cache[post.image] || post.image,
+      };
+    });
+
+    writeJSONSync(changelogIndexPath, changelogIndex, { spaces: 2 });
+
+    consola.success(
+      `${count} changelog index images have been uploaded to CDN and links have been replaced`,
+    );
+  }
+
+  // 运行上传过程
+  async run() {
+    const links = this.collectImageLinks();
+
+    if (links.length > 0) {
+      consola.info("Found images that haven't been uploaded to CDN:");
+      consola.info(links);
+      await this.uploadImagesToCDN(links);
+    } else {
+      consola.info('No new images to upload.');
+    }
+  }
+}
+
+// 实例化并运行
+const instance = new ImageCDNUploader();
+
+instance.run();

package/scripts/cdnWorkflow/optimized.ts

@@ -0,0 +1,21 @@
+import sharp from 'sharp';
+
+const WIDTH = 1600;
+
+export const opimized = async (
+  inputBuffer: ArrayBuffer,
+  width: number = WIDTH,
+): Promise<Buffer> => {
+  return await sharp(inputBuffer)
+    .resize({ width: width, withoutEnlargement: true })
+    .webp()
+    .toBuffer();
+};
+
+export const opimizedGif = async (inputBuffer: ArrayBuffer): Promise<Buffer> => {
+  try {
+    return await sharp(inputBuffer, { animated: true }).webp().toBuffer();
+  } catch {
+    return await sharp(inputBuffer, { animated: true, limitInputPixels: false }).webp().toBuffer();
+  }
+};

package/scripts/cdnWorkflow/s3/index.ts

@@ -0,0 +1,120 @@
+import {
+  GetObjectCommand,
+  PutObjectCommand,
+  PutObjectCommandOutput,
+  S3Client,
+  S3ClientConfig,
+} from '@aws-sdk/client-s3';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+
+import type { ImgInfo, S3UserConfig, UploadResult } from './types';
+import { extractInfo } from './utils';
+
+async function getFileURL(
+  opts: createUploadTaskOpts,
+  eTag: string,
+  versionId: string,
+): Promise<string> {
+  try {
+    const signedUrl = await getSignedUrl(
+      opts.client,
+      new GetObjectCommand({
+        Bucket: opts.bucketName,
+        IfMatch: eTag,
+        Key: opts.path,
+        VersionId: versionId,
+      }),
+      { expiresIn: 3600 },
+    );
+    const urlObject = new URL(signedUrl);
+    urlObject.search = '';
+    return urlObject.href;
+  } catch (error) {
+    // eslint-disable-next-line unicorn/no-useless-promise-resolve-reject
+    return Promise.reject(error);
+  }
+}
+
+function createS3Client(opts: S3UserConfig): S3Client {
+  const clientOptions: S3ClientConfig = {
+    credentials: {
+      accessKeyId: opts.accessKeyId,
+      secretAccessKey: opts.secretAccessKey,
+    },
+
+    endpoint: opts.endpoint || undefined,
+    forcePathStyle: opts.pathStyleAccess,
+    region: opts.region || 'auto',
+  };
+
+  const client = new S3Client(clientOptions);
+  return client;
+}
+
+interface createUploadTaskOpts {
+  acl: string;
+  bucketName: string;
+  client: S3Client;
+  item: ImgInfo;
+  path: string;
+  urlPrefix?: string;
+}
+
+async function createUploadTask(opts: createUploadTaskOpts): Promise<UploadResult> {
+  if (!opts.item.buffer) {
+    throw new Error('undefined image');
+  }
+
+  let body: Buffer;
+  let contentType: string;
+  let contentEncoding: string;
+
+  try {
+    ({ body, contentType, contentEncoding } = (await extractInfo(opts.item)) as any);
+  } catch (error) {
+    // eslint-disable-next-line unicorn/no-useless-promise-resolve-reject
+    return Promise.reject(error);
+  }
+
+  const command = new PutObjectCommand({
+    ACL: opts.acl as any,
+    Body: body,
+    Bucket: opts.bucketName,
+    ContentEncoding: contentEncoding,
+    ContentType: contentType,
+    Key: opts.path,
+  });
+
+  let output: PutObjectCommandOutput;
+  try {
+    output = await opts.client.send(command);
+  } catch (error) {
+    // eslint-disable-next-line unicorn/no-useless-promise-resolve-reject
+    return Promise.reject(error);
+  }
+
+  let url: string;
+  if (opts.urlPrefix) {
+    url = `${opts.urlPrefix}/${opts.path}`;
+  } else {
+    try {
+      url = await getFileURL(opts, output.ETag as string, output.VersionId as string);
+    } catch (error) {
+      // eslint-disable-next-line unicorn/no-useless-promise-resolve-reject
+      return Promise.reject(error);
+    }
+  }
+
+  return {
+    eTag: output.ETag,
+    imgURL: url,
+    key: opts.path,
+    url: url,
+    versionId: output.VersionId,
+  };
+}
+
+export default {
+  createS3Client,
+  createUploadTask,
+};

package/scripts/cdnWorkflow/s3/types.ts

@@ -0,0 +1,25 @@
+export interface ImgInfo {
+  [propName: string]: any;
+  buffer: Buffer;
+  extname: string;
+  fileName: string;
+}
+
+export interface S3UserConfig {
+  accessKeyId: string;
+  bucketName: string;
+  endpoint: string;
+  pathPrefix: string;
+  pathStyleAccess?: boolean;
+  region: string;
+  secretAccessKey: string;
+  uploadPath?: string;
+}
+
+export interface UploadResult {
+  eTag?: string;
+  imgURL: string;
+  key: string;
+  url: string;
+  versionId?: string;
+}

package/scripts/cdnWorkflow/s3/utils.ts

@@ -0,0 +1,106 @@
+import CryptoJS from 'crypto-js';
+import mime from 'mime';
+
+import { ImgInfo } from './types';
+
+class FileNameGenerator {
+  date: Date;
+  info: ImgInfo;
+
+  static fields = [
+    'year',
+    'month',
+    'day',
+    'fullName',
+    'fileName',
+    'extName',
+    'timestamp',
+    'timestampMS',
+    'md5',
+  ];
+
+  constructor(info: ImgInfo) {
+    this.date = new Date();
+    this.info = info;
+  }
+
+  public year(): string {
+    return `${this.date.getFullYear()}`;
+  }
+
+  public month(): string {
+    return this.date.getMonth() < 9
+      ? `0${this.date.getMonth() + 1}`
+      : `${this.date.getMonth() + 1}`;
+  }
+
+  public day(): string {
+    return this.date.getDate() < 9 ? `0${this.date.getDate()}` : `${this.date.getDate()}`;
+  }
+
+  public fullName(): string {
+    return this.info.fileName;
+  }
+
+  public fileName(): string {
+    return this.info.fileName.replace(this.info.extname, '');
+  }
+
+  public extName(): string {
+    return this.info.extname.replace('.', '');
+  }
+
+  public timestamp(): string {
+    return Math.floor(Date.now() / 1000).toString();
+  }
+
+  public timestampMS(): string {
+    return Date.now().toString();
+  }
+
+  public md5(): string {
+    const wordArray = CryptoJS.lib.WordArray.create(this.imgBuffer());
+    const md5Hash = CryptoJS.MD5(wordArray);
+    return md5Hash.toString(CryptoJS.enc.Hex);
+  }
+  private imgBuffer(): Buffer {
+    return this.info.buffer;
+  }
+}
+
+export function formatPath(info: ImgInfo, format?: string): string {
+  if (!format) {
+    return info.fileName;
+  }
+
+  const fileNameGenerator = new FileNameGenerator(info);
+
+  let formatPath: string = format;
+
+  for (const key of FileNameGenerator.fields) {
+    const re = new RegExp(`{${key}}`, 'g');
+    // @ts-ignore
+    formatPath = formatPath.replace(re, fileNameGenerator[key]());
+  }
+
+  return formatPath;
+}
+
+export async function extractInfo(info: ImgInfo): Promise<{
+  body?: Buffer;
+  contentEncoding?: string;
+  contentType?: string;
+}> {
+  const result: {
+    body?: Buffer;
+    contentEncoding?: string;
+    contentType?: string;
+  } = {};
+
+  if (info.extname) {
+    result.contentType = mime.getType(info.extname) as string;
+  }
+  result.body = info.buffer;
+
+  return result;
+}

package/scripts/cdnWorkflow/uploader.ts

@@ -0,0 +1,73 @@
+import { consola } from 'consola';
+import dotenv from 'dotenv';
+
+import s3 from './s3';
+import type { ImgInfo, S3UserConfig, UploadResult } from './s3/types';
+import { formatPath } from './s3/utils';
+
+dotenv.config();
+
+if (!process.env.DOC_S3_ACCESS_KEY_ID) {
+  consola.error('请配置 Doc S3 存储的环境变量: DOC_S3_ACCESS_KEY_ID');
+  // eslint-disable-next-line unicorn/no-process-exit
+  process.exit(1);
+}
+
+if (!process.env.DOC_S3_SECRET_ACCESS_KEY) {
+  consola.error('请配置 Doc S3 存储的环境变量: DOC_S3_SECRET_ACCESS_KEY');
+  // eslint-disable-next-line unicorn/no-process-exit
+  process.exit(1);
+}
+
+if (!process.env.DOC_S3_PUBLIC_DOMAIN) {
+  consola.error('请配置 Doc S3 存储的环境变量: DOC_S3_PUBLIC_DOMAIN');
+  // eslint-disable-next-line unicorn/no-process-exit
+  process.exit(1);
+}
+
+export const BASE_PATH = 'blog/assets';
+
+export const uploader = async (
+  file: File,
+  filename: string,
+  basePath: string = BASE_PATH,
+  uploadPath?: string,
+) => {
+  const item: ImgInfo = {
+    buffer: Buffer.from(await file.arrayBuffer()),
+    extname: file.name.split('.').pop() as string,
+    fileName: file.name,
+    mimeType: file.type,
+  };
+
+  const userConfig: S3UserConfig = {
+    accessKeyId: process.env.DOC_S3_ACCESS_KEY_ID || '',
+    bucketName: 'hub-apac-1',
+    endpoint: 'https://d35842305b91be4b48e06ff9a9ad83f5.r2.cloudflarestorage.com',
+    pathPrefix: process.env.DOC_S3_PUBLIC_DOMAIN || '',
+    pathStyleAccess: true,
+    region: 'auto',
+    secretAccessKey: process.env.DOC_S3_SECRET_ACCESS_KEY || '',
+    uploadPath: uploadPath || `${basePath}${filename}.{extName}`,
+  };
+
+  const client = s3.createS3Client(userConfig);
+
+  let results: UploadResult;
+
+  try {
+    results = await s3.createUploadTask({
+      acl: 'public-read',
+      bucketName: userConfig.bucketName,
+      client,
+      item: item,
+      path: formatPath(item, userConfig.uploadPath),
+      urlPrefix: userConfig.pathPrefix,
+    });
+
+    return results.url;
+  } catch (error) {
+    consola.error('上传到 S3 存储发生错误，请检查网络连接和配置是否正确');
+    consola.error(error);
+  }
+};

package/scripts/cdnWorkflow/utils.ts

@@ -0,0 +1,93 @@
+import { readJSONSync } from 'fs-extra';
+import { globSync } from 'glob';
+import { resolve } from 'node:path';
+
+import { opimized, opimizedGif } from './optimized';
+
+export const fixWinPath = (path: string) => path.replaceAll('\\', '/');
+
+export const root = resolve(__dirname, '../..');
+
+export const posts = globSync(fixWinPath(resolve(root, 'docs/changelog/*.mdx')));
+
+interface ChangelogItem {
+  date: string;
+  id: string;
+  image?: string;
+  versionRange: string[];
+}
+
+export const changelogIndexPath = resolve(root, 'docs/changelog/index.json');
+
+export const changelogIndex: {
+  cloud: ChangelogItem[];
+  community: ChangelogItem[];
+} = readJSONSync(changelogIndexPath);
+
+export const extractHttpsLinks = (text: string) => {
+  const regex = /https:\/\/[^\s"')>]+/g;
+  const links = text.match(regex);
+  return links || [];
+};
+
+export const mergeAndDeduplicateArrays = (...arrays: string[][]) => {
+  const combinedArray = arrays.flat();
+  const uniqueSet = new Set(combinedArray);
+  return Array.from(uniqueSet);
+};
+
+const mimeToExtensions = {
+  'image/gif': '.gif',
+  // 图片类型
+  'image/jpeg': '.jpg',
+  'image/png': '.png',
+  'image/svg+xml': '.svg',
+  'image/webp': '.webp',
+  // 视频类型
+  'video/mp4': '.mp4',
+  'video/mpeg': '.mpeg',
+  'video/ogg': '.ogv',
+  'video/quicktime': '.mov',
+  'video/webm': '.webm',
+  'video/x-flv': '.flv',
+  'video/x-matroska': '.mkv',
+  'video/x-ms-wmv': '.wmv',
+  'video/x-msvideo': '.avi',
+};
+
+// @ts-ignore
+const getExtension = (type: string) => mimeToExtensions?.[type] || '.png';
+
+export const fetchImageAsFile = async (url: string, width: number) => {
+  try {
+    // Step 1: Fetch the image
+    const response = await fetch(url);
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+
+    // Step 2: Create a blob from the response data
+    const blob = await response.blob();
+    let buffer: ArrayBuffer | Buffer = await blob.arrayBuffer();
+    let type = getExtension(blob.type);
+    if (type === '.gif') {
+      buffer = await opimizedGif(buffer);
+      type = '.webp';
+    } else if (type === '.png' || type === '.jpg') {
+      buffer = await opimized(buffer, width);
+      type = '.webp';
+    }
+
+    const filename = Date.now().toString() + type;
+
+    // Step 3: Create a file from the blob
+    const file: File = new File([buffer], filename, {
+      lastModified: Date.now(),
+      type: type === '.webp' ? 'image/webp' : blob.type,
+    });
+
+    return file;
+  } catch (error) {
+    console.error('Error fetching image as file:', error);
+  }
+};