czon 0.1.0 → 0.1.1

package/src/languages.ts

@@ -0,0 +1,37 @@
+export const LANGUAGE_NAMES: Record<string, string> = {
+  // 现有语言
+  'zh-Hans': '简体中文',
+  'en-US': 'English',
+  'ja-JP': '日本語',
+  'ko-KR': '한국어',
+  'es-ES': 'Español',
+  'fr-FR': 'Français',
+  'de-DE': 'Deutsch',
+  'ru-RU': 'Русский',
+
+  // 补充缺失的项目支持语言
+  'pt-PT': 'Português',
+
+  // 欧洲主要语言
+  'it-IT': 'Italiano',
+  'nl-NL': 'Nederlands',
+  'pl-PL': 'Polski',
+  'sv-SE': 'Svenska',
+  'fi-FI': 'Suomi',
+  'da-DK': 'Dansk',
+  'no-NO': 'Norsk',
+
+  // 亚洲主要语言
+  'zh-Hant': '繁體中文',
+  'hi-IN': 'हिन्दी',
+  'ar-SA': 'العربية',
+  'th-TH': 'ไทย',
+  'vi-VN': 'Tiếng Việt',
+  'id-ID': 'Bahasa Indonesia',
+
+  // 其他重要语言
+  'pt-BR': 'Português (Brasil)',
+  'es-MX': 'Español (México)',
+  'tr-TR': 'Türkçe',
+  'uk-UA': 'Українська',
+};

package/src/metadata.ts

@@ -0,0 +1,44 @@
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { ZEN_META_PATH } from './paths';
+import { MetaDataStore } from './types';
+
+/**
+ * 全局 MetaDataStore 单例
+ */
+export const MetaData: MetaDataStore = {
+  // 稍后覆盖
+  version: '1.0.0',
+  options: {},
+  files: [],
+};
+
+/**
+ * 从文件中读取数据，覆盖 store，但是要保持它仍然是同一个对象
+ */
+export async function loadMetaData(): Promise<void> {
+  try {
+    await fs.access(ZEN_META_PATH);
+    const content = await fs.readFile(ZEN_META_PATH, 'utf-8');
+    const newData = JSON.parse(content);
+
+    // 使用 Object.assign 保持同一个对象引用
+    Object.assign(MetaData, newData);
+  } catch (error) {
+    // 如果文件不存在，初始化默认值
+    MetaData.version = '1.0.0';
+    MetaData.files = [];
+  }
+}
+
+/**
+ * 将 MetaData 写入 store
+ */
+export async function saveMetaData(): Promise<void> {
+  // 确保 .zen 目录存在
+  const zenDir = path.dirname(ZEN_META_PATH);
+  await fs.mkdir(zenDir, { recursive: true });
+
+  // 保存文件
+  await fs.writeFile(ZEN_META_PATH, JSON.stringify(MetaData, null, 2), 'utf-8');
+}

package/src/paths.ts

@@ -0,0 +1,7 @@
+import { join } from 'path';
+
+export const INPUT_DIR = process.cwd();
+export const ZEN_DIR = join(process.cwd(), '.zen');
+export const ZEN_DIST_DIR = join(ZEN_DIR, 'dist');
+export const ZEN_SRC_DIR = join(ZEN_DIR, 'src');
+export const ZEN_META_PATH = join(ZEN_DIR, 'meta.json');

package/src/process/extractMetadataByAI.ts

@@ -0,0 +1,31 @@
+import { readFile } from 'fs/promises';
+import { extractMetadataFromMarkdown } from '../ai/extractMetadataFromMarkdown';
+import { MetaData } from '../metadata';
+
+/**
+ * 运行 AI 元数据提取
+ */
+export async function extractMetadataByAI(): Promise<void> {
+  const { files } = MetaData;
+
+  if (MetaData.options.verbose) console.log(`🤖 Running AI metadata extraction...`);
+  console.log(`🤖 Processing ${files.length} files with AI...`);
+
+  await Promise.all(
+    files.map(async file => {
+      try {
+        if (file.metadata) {
+          console.info(`ℹ️ Skipping ${file.path}, already has metadata`);
+          return;
+        }
+        const content = await readFile(file.path, 'utf-8');
+        file.metadata = await extractMetadataFromMarkdown(content);
+        console.log(`✅ Extracted AI metadata for ${file.path}`, file.metadata.tokens_used);
+      } catch (error) {
+        console.error(`⚠️ Failed to process file ${file.path}:`, error);
+      }
+    })
+  );
+
+  console.log(`✅ AI processing completed for ${files.length} files`);
+}

package/src/process/scanSourceFiles.ts

@@ -0,0 +1,73 @@
+import { readFile } from 'fs/promises';
+import path from 'path';
+import { findMarkdownEntries } from '../findEntries';
+import { MetaData } from '../metadata';
+import { INPUT_DIR } from '../paths';
+import { sha256 } from '../utils/sha256';
+
+const extractLinksFromMarkdown = (content: string): string[] => {
+  const linkRegex = /\[.*?\]\((.*?)\)/g;
+  const links: string[] = [];
+  let match;
+  while ((match = linkRegex.exec(content)) !== null) {
+    links.push(match[1]);
+  }
+  return links;
+};
+
+/**
+ * 扫描源文件
+ */
+export async function scanSourceFiles(): Promise<void> {
+  console.log(`🔍 Scanning source directory...`);
+  const markdownFiles = await findMarkdownEntries(INPUT_DIR);
+  const hashes = new Set<string>();
+
+  for (const relativePath of markdownFiles) {
+    const fullPath = path.join(INPUT_DIR, relativePath);
+
+    try {
+      // 检查文件是否存在
+
+      const content = await readFile(fullPath, 'utf-8'); // 确保文件可读
+
+      const hash = sha256(content);
+      const links = extractLinksFromMarkdown(content);
+      console.info(`  - Found file: ${relativePath} (hash: ${hash})`);
+      console.info(`    Links: ${links.join(', ') || 'None'}`);
+
+      hashes.add(hash);
+
+      const metaWithSameHash = MetaData.files.find(f => f.hash === hash);
+      if (metaWithSameHash) {
+        metaWithSameHash.path = relativePath;
+        metaWithSameHash.links = links;
+      } else {
+        // 如果没有相同哈希的元数据，则添加一个新的占位符
+        MetaData.files.push({
+          hash,
+          path: relativePath,
+          links,
+        });
+      }
+    } catch (error) {
+      console.warn(`⚠️ File not found or inaccessible: ${fullPath}`, error);
+    }
+  }
+  // 移除不再存在的文件元数据
+  MetaData.files = MetaData.files.filter(f => hashes.has(f.hash));
+  // 按路径降序排序 (通常外层目录优先)
+  MetaData.files.sort(
+    (a, b) =>
+      // 第一级按目录排序
+      path.dirname(a.path).localeCompare(path.dirname(b.path)) ||
+      // 第二级按文件名排序
+      a.path.localeCompare(b.path)
+  );
+
+  console.log(`✅ Found ${MetaData.files.length} Markdown files`);
+
+  if (MetaData.files.length === 0) {
+    console.warn(`⚠️ No Markdown files found in ${INPUT_DIR}`);
+  }
+}

package/src/process/template.ts

@@ -0,0 +1,222 @@
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { LANGUAGE_NAMES } from '../languages';
+import { MetaData } from '../metadata';
+import { ZEN_DIST_DIR, ZEN_SRC_DIR } from '../paths';
+import { MetaDataStore } from '../types';
+import { convertMarkdownToHtml } from '../utils/convertMarkdownToHtml';
+import { parseFrontmatter } from '../utils/frontmatter';
+
+/**
+ * 生成语言切换器 HTML
+ * @param currentLang 当前语言
+ * @param availableLangs 可用语言列表
+ * @returns 语言切换器 HTML 字符串
+ */
+function generateLanguageSwitcher(templateData: TemplateData): string {
+  const {
+    options: { langs = [] },
+  } = MetaData;
+
+  const items = langs
+    .map(lang => {
+      const langName = LANGUAGE_NAMES[lang] || lang;
+      const isCurrent = lang === templateData.lang;
+      const activeClass = isCurrent ? 'active' : '';
+
+      const link = path.join('..', lang, templateData.file.hash + '.html');
+
+      return `<li class="lang-item ${activeClass}">
+        <a href="${link}" class="lang-link">${langName}</a>
+      </li>`;
+    })
+    .join('');
+
+  return `<div class="language-switcher">
+    <ul class="lang-list">${items}</ul>
+  </div>`;
+}
+
+const generateTagsHtml = (tags: string[]): string => {
+  return `<ul class="tags-list">${tags
+    .map(tag => `<li class="tag-item">${tag}</li>`)
+    .join('')}</ul>`;
+};
+
+/**
+ * 生成导航 HTML
+ * @param navigation 导航树
+ * @param currentPath 当前路径（可选，用于高亮当前页面）
+ * @returns 导航 HTML 字符串
+ */
+async function generateNavigationHtml(data: TemplateData): Promise<string> {
+  const { files } = MetaData;
+
+  const navigation = await Promise.all(
+    files.map(async file => {
+      const content = await fs.readFile(
+        path.join(ZEN_SRC_DIR, data.lang, file.hash + '.md'),
+        'utf-8'
+      );
+      const { frontmatter } = parseFrontmatter(content);
+      const title = frontmatter.title || file.metadata?.title || file.path; // 优先使用提取的标题
+
+      // 使用相对链接
+      const link = file.hash + '.html';
+
+      return {
+        title,
+        link,
+        isActive: data.file.hash === file.hash,
+      };
+    })
+  );
+
+  return `<ul class="nav-list">${navigation
+    .map(item => {
+      const activeClass = item.isActive ? 'active' : '';
+
+      let html = `<li class="nav-item">`;
+      html += `<a href="${item.link}" class="nav-link ${activeClass}">${item.title}</a>`;
+
+      html += `</li>`;
+      return html;
+    })
+    .join('')}</ul>`;
+}
+
+const replaceInnerLinks = (data: TemplateData, markdownContent: string): string => {
+  let content = markdownContent;
+  for (const link of data.file.links) {
+    if (URL.canParse(link)) continue; // 跳过绝对 URL
+
+    const targetPath = path.resolve('/', path.dirname(data.file.path), link).slice(1);
+
+    const targetFile = MetaData.files.find(f => f.path === targetPath);
+
+    if (!targetFile) {
+      console.warn(`⚠️ Link target not found for ${link} in file ${data.file.path}`);
+      continue;
+    }
+    // 替换链接 (使用相对链接)
+    const targetLink = path.join(targetFile.hash + '.html');
+
+    // 全局替换链接
+    const linksRegex = new RegExp(`\\]\\(${link.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\)`, 'g');
+    content = content.replace(linksRegex, `](${targetLink})`);
+  }
+  return content;
+};
+
+interface TemplateData {
+  file: MetaDataStore['files'][0];
+  content: string;
+  lang: string;
+}
+
+/**
+ * 简单的模板变量替换
+ * @param template 模板字符串
+ * @param data 模板数据
+ * @returns 渲染后的 HTML 字符串
+ */
+async function renderTemplate(template: string, data: TemplateData): Promise<string> {
+  const {
+    options: { langs = [] },
+  } = MetaData;
+  const markdownContent = data.content;
+  const { frontmatter, body } = parseFrontmatter(markdownContent);
+
+  const htmlContent = convertMarkdownToHtml(replaceInnerLinks(data, body));
+
+  let result = template;
+
+  // 替换导航
+  const navigationHtml = await generateNavigationHtml(data);
+  result = result.replace(/{{navigation}}/g, navigationHtml);
+
+  // 替换其他变量 - 使用全局替换
+  result = result.replace(/{{title}}/g, frontmatter.title || 'Untitled');
+  result = result.replace(/{{content}}/g, htmlContent);
+
+  // 替换元数据变量
+  if (frontmatter) {
+    result = result.replace(/{{summary}}/g, frontmatter.summary || '');
+    result = result.replace(/{{tags}}/g, generateTagsHtml(frontmatter.tags || []));
+    result = result.replace(/{{inferred_date}}/g, frontmatter.inferred_date || '--');
+    result = result.replace(/{{inferred_lang}}/g, frontmatter.inferred_lang || '--');
+  }
+
+  // 替换语言相关变量
+  result = result.replace(/{{lang}}/g, data.lang || '');
+  if (langs && langs.length > 1 && data.lang) {
+    const langSwitcher = generateLanguageSwitcher(data);
+    result = result.replace(/{{language_switcher}}/g, langSwitcher);
+  }
+
+  return result;
+}
+
+const renderRedirectTemplate = async (from: string, to: string): Promise<void> => {
+  const toURL = path.relative(path.dirname(from), to);
+  const html = `<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta http-equiv="refresh" content="0; url=${toURL}">
+    <title>Redirecting...</title>
+</head>
+<body>
+    <p>Redirecting to <a href="${toURL}">${toURL}</a></p>
+</body>
+</html>`;
+  const targetPath = path.join(ZEN_DIST_DIR, from);
+  await fs.mkdir(path.dirname(targetPath), { recursive: true });
+  await fs.writeFile(targetPath, html, 'utf-8');
+};
+
+/**
+ * 渲染模板并保存文件
+ */
+export async function renderTemplates(): Promise<void> {
+  const {
+    files,
+    options: { langs, verbose },
+  } = MetaData;
+
+  if (verbose) console.log(`⚡ Processing files...`);
+  const layoutTemplate = await fs.readFile(
+    path.join(__dirname, '../../assets/templates/default/layout.html'),
+    'utf-8'
+  );
+
+  for (const file of files) {
+    for (const lang of langs || []) {
+      console.info(`📄 Preparing file for language: ${file.path} [${file.hash}] [${lang}]`);
+      const targetPath = path.join(ZEN_DIST_DIR, lang, file.hash + '.html');
+      const content = await fs.readFile(path.join(ZEN_SRC_DIR, lang, file.hash + '.md'), 'utf-8');
+      try {
+        const html = await renderTemplate(layoutTemplate, {
+          file,
+          content,
+          lang,
+        });
+        await fs.mkdir(path.dirname(targetPath), { recursive: true });
+        await fs.writeFile(targetPath, html, 'utf-8');
+        if (verbose) console.log(`✅ Rendered: ${targetPath}`);
+      } catch (error) {
+        console.error(`❌ Failed to render ${file.path}:`, error);
+      }
+    }
+  }
+
+  for (const lang of langs || []) {
+    await renderRedirectTemplate(
+      path.join(lang, 'index.html'),
+      path.join(lang, files[0].hash + '.html')
+    );
+  }
+  await renderRedirectTemplate('index.html', path.join(langs?.[0] || 'en-US', 'index.html'));
+}

package/src/services/openai.ts

@@ -0,0 +1,92 @@
+/**
+ * OpenAI 消息接口
+ */
+export interface OpenAIMessage {
+  role: 'system' | 'user' | 'assistant';
+  content: string;
+}
+
+/**
+ * OpenAI 响应接口
+ */
+export interface OpenAIResponse {
+  id: string;
+  object: string;
+  created: number;
+  model: string;
+  choices: Array<{
+    index: number;
+    message: {
+      role: string;
+      content: string;
+    };
+    finish_reason: string;
+  }>;
+  usage: {
+    prompt_tokens: number;
+    completion_tokens: number;
+    total_tokens: number;
+  };
+}
+
+/**
+ * 使用 OpenAI API 补全消息
+ * @param messages 消息数组
+ * @param options 可选配置
+ * @returns Promise<OpenAIResponse> 返回完整的OpenAI响应
+ */
+export const completeMessages = async (
+  messages: OpenAIMessage[],
+  options?: {
+    response_format?: { type: 'json_object' | 'text' };
+  }
+): Promise<OpenAIResponse> => {
+  // 从环境变量读取配置
+  const apiKey = process.env.OPENAI_API_KEY || '';
+  const baseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1';
+  const model = process.env.OPENAI_MODEL || 'gpt-3.5-turbo';
+
+  if (!apiKey) {
+    throw new Error('OPENAI_API_KEY environment variable is not set');
+  }
+
+  try {
+    const requestBody: any = {
+      model,
+      messages,
+      temperature: 0, // 总是设置为 0，提取内容不需要随机性
+      // 不设置 max_tokens，让API自动决定
+    };
+
+    // 添加可选的response_format
+    if (options?.response_format) {
+      requestBody.response_format = options.response_format;
+    }
+
+    const response = await fetch(`${baseUrl}/chat/completions`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${apiKey}`,
+      },
+      body: JSON.stringify(requestBody),
+    });
+
+    if (!response.ok) {
+      const errorText = await response.text();
+      throw new Error(`OpenAI API error (${response.status}): ${errorText}`);
+    }
+
+    const data: OpenAIResponse = await response.json();
+
+    // 验证响应
+    if (!data.choices?.[0]?.message?.content?.trim()) {
+      throw new Error('Empty response from OpenAI API');
+    }
+
+    return data;
+  } catch (error) {
+    console.error('❌ Failed to call OpenAI API:', error);
+    throw error;
+  }
+};

package/src/types.ts

@@ -0,0 +1,37 @@
+/**
+ * 单个文件的元数据缓存项
+ */
+export interface FileMetaData {
+  hash: string;
+  path: string;
+  links: string[];
+  metadata?: AIMetadata;
+}
+
+/**
+ * .zen/meta.json 文件结构
+ */
+export interface MetaDataStore {
+  version: string;
+  options: BuildOptions;
+  files: FileMetaData[];
+}
+
+export interface BuildOptions {
+  template?: string;
+  verbose?: boolean;
+  langs?: string[]; // 目标语言数组
+}
+
+export interface AIMetadata {
+  title: string; // AI 提取的标题
+  summary: string; // AI 提取的摘要，控制在 100字以内
+  tags: string[]; // AI 提取的关键字
+  inferred_date?: string; // 正文中隐含的文档创建日期，没有就留空
+  inferred_lang: string; // 文章使用的语言，例如 zh-Hans 或者 en-US
+  tokens_used?: {
+    prompt: number;
+    completion: number;
+    total: number;
+  }; // tokens 使用情况
+}

package/src/utils/convertMarkdownToHtml.ts

@@ -0,0 +1,93 @@
+import hljs from 'highlight.js';
+import { marked } from 'marked';
+import markedKatex from 'marked-katex-extension';
+
+// 辅助函数：转义 HTML 特殊字符
+function escapeHtml(unsafe: string): string {
+  return unsafe
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&#039;');
+}
+marked.use(markedKatex({ throwOnError: false, nonStandard: true } as any));
+
+/**
+ * 将 Markdown 内容转换为 HTML
+ * @param mdContent Markdown 内容字符串
+ * @returns 转换后的 HTML 字符串
+ */
+export const convertMarkdownToHtml = (mdContent: string): string => {
+  // 创建自定义渲染器
+  const renderer = new marked.Renderer();
+  const originalCodeRenderer = renderer.code;
+
+  // 重写代码块渲染器以支持 Mermaid - 使用 any 类型绕过类型检查
+  (renderer as any).code = function (code: any, language?: string, isEscaped?: boolean) {
+    // 在 marked 17+ 中，code 参数是一个对象，包含 text 和 lang 属性
+    const codeText = typeof code === 'string' ? code : code?.text || '';
+    // 语言信息在 code.lang 中，而不是 language 参数
+    const lang = code?.lang || language;
+
+    // 检测 Mermaid 代码块
+    if (lang === 'mermaid') {
+      // 生成唯一的 ID 用于图表容器
+      const chartId = 'mermaid-' + Date.now() + '-' + Math.random().toString(36).substr(2, 9);
+      return `
+<div class="mermaid-diagram" data-mermaid-id="${chartId}">
+  <pre class="mermaid">${escapeHtml(codeText)}</pre>
+</div>
+`;
+    }
+
+    // 其他代码块使用原有高亮逻辑
+    return (originalCodeRenderer as any).call(this, code, language, isEscaped);
+  };
+
+  // 使用 marked.parse 的同步版本
+  // marked 17+ 默认返回 Promise，但我们可以使用 marked.parseSync 或 marked.parse 的同步模式
+  // 这里我们使用 marked.parse 并假设它是同步的（对于简单情况）
+  try {
+    // 尝试同步解析
+    const result = marked.parse(mdContent, {
+      renderer,
+      highlight: function (code: string, lang: string) {
+        // 跳过 Mermaid 代码块的高亮
+        if (lang === 'mermaid') {
+          return code;
+        }
+
+        if (lang && hljs.getLanguage(lang)) {
+          try {
+            return hljs.highlight(code, { language: lang }).value;
+          } catch (err) {
+            console.warn(`Failed to highlight code with language ${lang}:`, err);
+          }
+        }
+        return hljs.highlightAuto(code).value;
+      },
+      pedantic: false,
+      gfm: true,
+      breaks: false,
+      sanitize: false,
+      smartLists: true,
+      smartypants: false,
+      xhtml: false,
+      async: false, // 强制同步模式
+    } as any);
+
+    // 如果结果是 Promise，等待它（虽然我们设置了 async: false）
+    if (result && typeof result.then === 'function') {
+      // 这不应该发生，但如果发生了，返回一个占位符
+      console.warn('marked.parse returned a Promise despite async: false');
+      return '<!-- Markdown conversion in progress -->';
+    }
+
+    return result as unknown as string;
+  } catch (error) {
+    console.error('Error converting Markdown to HTML:', error);
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    return `<div class="error">Error converting Markdown: ${errorMessage}</div>`;
+  }
+};

package/src/utils/frontmatter.ts

@@ -0,0 +1,18 @@
+import { parse, stringify } from 'yaml';
+
+export const parseFrontmatter = (content: string): { frontmatter: any; body: string } => {
+  const frontmatterRegex = /^---\n([\s\S]*?)\n---/;
+  const match = content.match(frontmatterRegex);
+  if (match) {
+    const frontmatterContent = match[1];
+    const body = content.slice(match[0].length).trim();
+    return { frontmatter: parse(frontmatterContent.trim()), body };
+  }
+  return { frontmatter: {}, body: content };
+};
+
+export const updateFrontmatter = (content: string, newFrontmatter: any): string => {
+  const { body } = parseFrontmatter(content);
+  const frontmatterContent = `---\n${stringify(newFrontmatter, { defaultStringType: 'QUOTE_DOUBLE' })}---\n\n`;
+  return frontmatterContent + body;
+};

package/src/utils/sha256.ts

@@ -0,0 +1,4 @@
+import { createHash } from 'crypto';
+
+export const sha256 = (content: string): string =>
+  createHash('sha256').update(content).digest('hex');