@newest-ai/notes-pipeline 0.0.0-stage → 0.1.0

package/src/markdown.mjs

@@ -0,0 +1,255 @@
+/**
+ * Markdown 渲染管线：代码高亮、KaTeX、Mermaid、wikilink、站内链接与图片重写。
+ *
+ * createMarkdown({ images, articles, base }) 返回 markdown-it 实例：
+ *   images   createImages() 产物，负责图片解析与拷贝
+ *   articles { byId, byBasename } 文章查找表
+ *   base     站点 base（如 '/'、'/agent/'），用于产出绝对图片地址
+ */
+import path from 'node:path'
+import MarkdownIt from 'markdown-it'
+import anchor from 'markdown-it-anchor'
+import taskLists from 'markdown-it-task-lists'
+import texmath from 'markdown-it-texmath'
+import katex from 'katex'
+import hljs from 'highlight.js/lib/common'
+import twig from 'highlight.js/lib/languages/twig'
+import powershell from 'highlight.js/lib/languages/powershell'
+
+hljs.registerLanguage('twig', twig)
+hljs.registerLanguage('powershell', powershell)
+
+const LANG_ALIAS = {
+  shell: 'bash',
+  bash: 'bash',
+  sh: 'bash',
+  console: 'plaintext',
+  terminal: 'plaintext',
+  text: 'plaintext',
+  plaintext: 'plaintext',
+  url: 'plaintext',
+  ts: 'typescript',
+  typescript: 'typescript',
+  powshell: 'powershell',
+  powershell: 'powershell',
+  jinja: 'twig',
+}
+
+function headingSlug(text) {
+  let t = String(text)
+    .replace(/`([^`]*)`/g, '$1')
+    .replace(/!\[[^\]]*\]\([^)]*\)/g, '')
+    .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1')
+    .replace(/[*_~]{1,2}([^*_~]+)[*_~]{1,2}/g, '$1')
+    .replace(/<[^>]+>/g, '')
+    .trim()
+  t = t.replace(/[^\p{L}\p{N}\s\-_]/gu, '')
+  t = t.replace(/\s+/g, '-')
+  return t || 'section'
+}
+
+const stripTags = (html) =>
+  html
+    .replace(/<[^>]*>/g, '')
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .trim()
+
+export function extractHeadings(html) {
+  const headings = []
+  const re = /<h([1-6])\b[^>]*\bid="([^"]*)"[^>]*>([\s\S]*?)<\/h\1>/gi
+  let m
+  while ((m = re.exec(html))) {
+    headings.push({ level: +m[1], id: m[2], text: stripTags(m[3]) })
+  }
+  return headings
+}
+
+export function createMarkdown({ images, articles, base = '/' }) {
+  const imageBase = `${base}content/images/`
+
+  function resolveArticle(target) {
+    let t = String(target || '').trim().replace(/\.md$/i, '')
+    t = t.replace(/\\/g, '/')
+    const key = t.toLowerCase()
+    if (articles.byBasename.has(key)) return articles.byBasename.get(key)
+    const norm = path.posix.normalize(key)
+    if (articles.byId.has(norm)) return articles.byId.get(norm)
+    return null
+  }
+
+  function resolveMdLink(href, dir) {
+    let pathPart = href
+    const hashIdx = href.indexOf('#')
+    if (hashIdx >= 0) pathPart = href.slice(0, hashIdx)
+    pathPart = pathPart.trim()
+    if (!pathPart) return null
+    if (/^[a-z][a-z0-9+.-]*:/i.test(pathPart)) return null
+    if (/\.(png|jpe?g|gif|svg|webp|pdf|docx?|xlsx?|zip|bmp)$/i.test(pathPart)) return null
+    const rel = path.posix.normalize(path.posix.join(dir || '.', pathPart)).replace(/\.md$/i, '')
+    const key = rel.toLowerCase()
+    if (articles.byId.has(key)) return articles.byId.get(key)
+    const basename = path.posix.basename(rel).toLowerCase()
+    if (articles.byBasename.has(basename)) return articles.byBasename.get(basename)
+    return null
+  }
+
+  const md = new MarkdownIt({ html: true, linkify: true, breaks: false })
+
+  md.use(anchor, { slugify: headingSlug, level: [1, 2, 3, 4, 5, 6] })
+  md.use(taskLists, { enabled: true, label: true })
+  md.use(texmath, {
+    engine: katex,
+    delimiters: ['dollars', 'brackets', 'beg_end'],
+    katexOptions: { throwOnError: false, strict: false, errorColor: '#cc0000', trust: true },
+  })
+
+  function highlight(code, lang) {
+    const l = (LANG_ALIAS[lang] || lang || 'plaintext').toLowerCase()
+    if (l === 'plaintext') return md.utils.escapeHtml(code)
+    if (hljs.getLanguage(l)) {
+      try {
+        return hljs.highlight(code, { language: l }).value
+      } catch {
+        return md.utils.escapeHtml(code)
+      }
+    }
+    return md.utils.escapeHtml(code)
+  }
+
+  md.renderer.rules.fence = (tokens, idx) => {
+    const token = tokens[idx]
+    const info = (token.info || '').trim()
+    const lang = info.split(/\s+/)[0].toLowerCase()
+    const code = token.content
+    if (lang === 'mermaid' || lang === 'mermaid.js') {
+      return `<pre class="mermaid">${md.utils.escapeHtml(code)}</pre>\n`
+    }
+    const highlighted = highlight(code, lang)
+    const cls = `hljs language-${md.utils.escapeHtml(lang || 'plaintext')}`
+    return `<pre class="code-block"><code class="${cls}">${highlighted}</code></pre>\n`
+  }
+
+  // wikilink：[[目标]] / [[目标|别名]] / [[目标#锚点]]
+  function wikilinkRule(state, silent) {
+    const src = state.src
+    const start = state.pos
+    if (src.charCodeAt(start) !== 0x5b || src.charCodeAt(start + 1) !== 0x5b) return false
+    const end = src.indexOf(']]', start + 2)
+    if (end < 0) return false
+    const inner = src.slice(start + 2, end)
+    if (inner.includes('\n') || inner.includes('[')) return false
+
+    if (!silent) {
+      let target = inner
+      let alias = null
+      let anchorText = ''
+      const hashIdx = target.indexOf('#')
+      if (hashIdx >= 0) {
+        anchorText = target.slice(hashIdx + 1)
+        target = target.slice(0, hashIdx)
+      }
+      const pipeIdx = target.indexOf('|')
+      if (pipeIdx >= 0) {
+        alias = target.slice(pipeIdx + 1)
+        target = target.slice(0, pipeIdx)
+      }
+      target = target.trim()
+      const text = alias || target || inner
+
+      const artId = resolveArticle(target)
+      if (artId) {
+        const open = state.push('link_open', 'a', 1)
+        open.attrs = [['href', `#/article/${encodeURIComponent(artId)}`]]
+        if (anchorText) open.attrs.push(['data-anchor', anchorText])
+        const t = state.push('text', '', 0)
+        t.content = text
+        state.push('link_close', 'a', -1)
+      } else {
+        const imgRel = images.resolve(target, '')
+        if (imgRel) {
+          const outName = images.copy(imgRel)
+          const open = state.push('link_open', 'a', 1)
+          open.attrs = [['href', imageBase + outName], ['target', '_blank']]
+          const t = state.push('text', '', 0)
+          t.content = text
+          state.push('link_close', 'a', -1)
+        } else {
+          const t = state.push('text', '', 0)
+          t.content = text
+        }
+      }
+    }
+    state.pos = end + 2
+    return true
+  }
+
+  // 图片嵌入：![[image.png]]
+  function wikilinkEmbedRule(state, silent) {
+    const src = state.src
+    const start = state.pos
+    if (src.charCodeAt(start) !== 0x21) return false
+    if (src.charCodeAt(start + 1) !== 0x5b || src.charCodeAt(start + 2) !== 0x5b) return false
+    const end = src.indexOf(']]', start + 3)
+    if (end < 0) return false
+    const inner = src.slice(start + 3, end)
+    if (inner.includes('\n')) return false
+    const imgRel = images.resolve(inner, '')
+    if (!imgRel) return false
+    if (!silent) {
+      const tok = state.push('image', 'img', 0)
+      tok.attrs = [['src', `img::${imgRel}`], ['alt', inner]]
+      tok.content = inner
+    }
+    state.pos = end + 2
+    return true
+  }
+
+  md.inline.ruler.before('link', 'wikilink', wikilinkRule)
+  md.inline.ruler.before('image', 'wikilink_embed', wikilinkEmbedRule)
+
+  md.renderer.rules.image = (tokens, idx, options, env, self) => {
+    const token = tokens[idx]
+    if (token.children && token.children.length) {
+      token.attrSet('alt', self.renderInlineAsText(token.children, options, env))
+    } else if (token.content) {
+      token.attrSet('alt', token.content)
+    }
+    const src = token.attrGet('src') || ''
+    let outName = null
+    if (src.startsWith('img::')) {
+      outName = images.copy(src.slice(5))
+    } else {
+      const rel = images.resolve(src, env.dir || '')
+      if (rel) outName = images.copy(rel)
+    }
+    if (outName) {
+      token.attrSet('src', imageBase + outName)
+    } else {
+      token.attrSet('data-broken', 'true')
+    }
+    token.attrSet('loading', 'lazy')
+    return self.renderToken(tokens, idx, options)
+  }
+
+  md.renderer.rules.link_open = (tokens, idx, options, env, self) => {
+    const token = tokens[idx]
+    const href = token.attrGet('href') || ''
+    if (href.startsWith('#/article/')) return self.renderToken(tokens, idx, options)
+    if (href.startsWith('#')) return self.renderToken(tokens, idx, options)
+    if (/^[a-z][a-z0-9+.-]*:/i.test(href)) {
+      if (/^https?:/i.test(href)) {
+        token.attrSet('target', '_blank')
+        token.attrSet('rel', 'noopener noreferrer')
+      }
+      return self.renderToken(tokens, idx, options)
+    }
+    const artId = resolveMdLink(href, env.dir || '')
+    if (artId) token.attrSet('href', `#/article/${encodeURIComponent(artId)}`)
+    return self.renderToken(tokens, idx, options)
+  }
+
+  return md
+}

package/src/page.mjs

@@ -0,0 +1,48 @@
+/**
+ * 页面装配：生成站点运行时配置，并把预构建 UI 的 index.html 改写为站点入口。
+ */
+const esc = (s) =>
+  String(s ?? '')
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+
+/** 注入到 index.html、供前端读取的站点配置 */
+export function buildSitePayload({ config, total }) {
+  return {
+    title: config.site.title,
+    brand: config.site.brand,
+    brandAccent: config.site.brandAccent,
+    description: config.site.description,
+    lang: config.site.lang,
+    base: config.site.base,
+    heroText: config.site.heroText,
+    menuType: config.menu.type,
+    total,
+    footer: config.site.footer,
+    generatedAt: new Date().toISOString(),
+  }
+}
+
+/**
+ * @param {string} template ui-dist/index.html 内容
+ * @param {{ payload: object, base: string }} options
+ */
+export function renderIndexHtml(template, { payload, base }) {
+  const inline = JSON.stringify(payload).replace(/</g, '\\u003c')
+  return template
+    .replace(/<html lang="[^"]*"/, `<html lang="${esc(payload.lang)}"`)
+    .replace(/<title>[\s\S]*?<\/title>/, `<title>${esc(payload.title)}</title>`)
+    .replace(
+      /<meta name="description" content="[^"]*"\s*\/?>/,
+      `<meta name="description" content="${esc(payload.description)}">`,
+    )
+    .replace(/\.\/assets\//g, `${base}assets/`)
+    .replace(/\.\/favicon\.svg/g, `${base}favicon.svg`)
+    .replace(/\.\/gongan\.png/g, `${base}gongan.png`)
+    .replace(
+      '</head>',
+      `  <script>window.__NOTES_SITE__ = ${inline}</script>\n</head>`,
+    )
+}

package/src/scan.mjs

@@ -0,0 +1,196 @@
+/**
+ * 扫描笔记目录，产出：
+ *   mdFiles / imageFiles  文件清单
+ *   roots                 导航树（category 节点；主站分组由 build 阶段再包一层 group）
+ *   flat                  阅读顺序的扁平索引
+ *
+ * 目录约定：顶层目录 = 分类，子目录任意深度；根目录散落笔记归入「概览」。
+ * 编号前缀（NN-）只影响排序，生成展示标题时会被去掉。
+ */
+import fs from 'node:fs'
+import path from 'node:path'
+import crypto from 'node:crypto'
+
+export const IMAGE_EXT = /\.(png|jpe?g|gif|svg|webp|bmp)$/i
+
+export const toPosix = (p) => p.split(path.sep).join('/')
+
+const shortHash = (s) => crypto.createHash('md5').update(s).digest('hex').slice(0, 6)
+
+export function naturalCompare(a, b) {
+  const num = (s) => {
+    const m = s.match(/^(\d+)/)
+    return m ? parseInt(m[1], 10) : null
+  }
+  const na = num(a)
+  const nb = num(b)
+  if (na !== null && nb !== null && na !== nb) return na - nb
+  return a.localeCompare(b, 'zh-CN', { numeric: true, sensitivity: 'base' })
+}
+
+/** 目录名 -> 标题：去掉 "NN-" 前缀 */
+export function dirTitle(name) {
+  return name.replace(/^\d+\s*[-_]\s*/, '').trim() || name
+}
+
+/** 文件名 -> 标题：去掉 "NN-" 前缀与扩展名 */
+export function fileTitle(rel) {
+  const base = path.posix.basename(rel, path.posix.extname(rel))
+  return base.replace(/^\d+\s*[-_]\s*/, '').trim() || base
+}
+
+/** 文章 id -> 产物文件名（ASCII 段 + 稳定短哈希，避免中文文件名问题） */
+export function articleSlug(id) {
+  const ascii = id
+    .replace(/[^\x00-\x7f]+/g, '')
+    .replace(/[^\w.-]+/g, '-')
+    .replace(/-{2,}/g, '-')
+    .replace(/^-+|-+$/g, '')
+  return `${ascii || 'note'}-${shortHash(id)}`
+}
+
+export function makeArticleNode(rel) {
+  const id = rel.replace(/\.md$/i, '')
+  return {
+    type: 'article',
+    id,
+    title: fileTitle(rel),
+    file: `content/articles/${articleSlug(id)}.json`,
+  }
+}
+
+/**
+ * @param {string} contentDir 笔记根目录
+ * @param {{ exclude?: string[], rootSkip?: string[], rootSectionTitle?: string }} options
+ */
+export function scanNotes(contentDir, options = {}) {
+  const exclude = new Set(options.exclude || [])
+  const rootSkip = new Set((options.rootSkip || []).map((s) => String(s).toLowerCase()))
+  const rootSectionTitle = options.rootSectionTitle || '概览'
+
+  const mdFiles = [] // { abs, rel }
+  const imageFiles = [] // { abs, rel, base }
+
+  function walk(absDir, relDir) {
+    const entries = fs.readdirSync(absDir, { withFileTypes: true })
+    for (const e of entries) {
+      if (e.name.startsWith('.') || exclude.has(e.name)) continue
+      const abs = path.join(absDir, e.name)
+      const rel = relDir ? `${relDir}/${e.name}` : e.name
+      if (e.isDirectory()) {
+        walk(abs, rel)
+      } else if (e.isFile()) {
+        if (e.name.toLowerCase().endsWith('.md')) {
+          mdFiles.push({ abs, rel: toPosix(rel) })
+        } else if (IMAGE_EXT.test(e.name)) {
+          imageFiles.push({ abs, rel: toPosix(rel), base: e.name })
+        }
+      }
+    }
+  }
+  walk(contentDir, '')
+  mdFiles.sort((a, b) => naturalCompare(a.rel, b.rel))
+  imageFiles.sort((a, b) => naturalCompare(a.rel, b.rel))
+
+  // 递归构建分类节点：子目录在前，文章在后
+  function buildCategory(relDir) {
+    const abs = path.join(contentDir, relDir)
+    const entries = fs.readdirSync(abs, { withFileTypes: true })
+    const dirs = []
+    const files = []
+    for (const e of entries) {
+      if (e.name.startsWith('.') || exclude.has(e.name)) continue
+      if (e.isDirectory()) {
+        if (e.name.toLowerCase() !== 'images') dirs.push(e.name)
+      } else if (e.isFile() && e.name.toLowerCase().endsWith('.md')) {
+        files.push(e.name)
+      }
+    }
+    dirs.sort(naturalCompare)
+    files.sort(naturalCompare)
+
+    const children = []
+    for (const d of dirs) {
+      const sub = buildCategory(path.posix.join(relDir, d))
+      if (sub.children.length > 0) {
+        children.push({ type: 'subcategory', id: sub.id, title: sub.title, children: sub.children })
+      }
+    }
+    for (const f of files) {
+      children.push(makeArticleNode(path.posix.join(relDir, f)))
+    }
+    return { type: 'category', id: relDir, title: dirTitle(path.posix.basename(relDir)), children }
+  }
+
+  const roots = []
+
+  // 根目录散落笔记 → 「概览」
+  const rootNotes = mdFiles.filter(
+    (f) => !f.rel.includes('/') && !rootSkip.has(path.posix.basename(f.rel).toLowerCase()),
+  )
+  if (rootNotes.length > 0) {
+    roots.push({
+      type: 'category',
+      id: `00-${rootSectionTitle}`,
+      title: rootSectionTitle,
+      children: rootNotes.map((f) => makeArticleNode(f.rel)),
+    })
+  }
+
+  // 顶层目录 → 分类
+  const topDirs = fs
+    .readdirSync(contentDir, { withFileTypes: true })
+    .filter((e) => e.isDirectory() && !e.name.startsWith('.') && !exclude.has(e.name))
+    .map((e) => e.name)
+    .sort(naturalCompare)
+
+  for (const d of topDirs) {
+    const node = buildCategory(d)
+    if (node.children.length > 0) roots.push(node)
+  }
+
+  return { mdFiles, imageFiles, roots, rootSectionTitle }
+}
+
+/** 按阅读顺序（深度优先，目录在前文章在后）摊平树 */
+export function flattenRoots(roots) {
+  const flat = []
+  const walk = (node) => {
+    if (node.type === 'article') {
+      flat.push({ id: node.id, title: node.title, file: node.file })
+    } else if (node.children) {
+      node.children.forEach(walk)
+    }
+  }
+  roots.forEach(walk)
+  return flat
+}
+
+/**
+ * 主站菜单：在分类之上再包一层 group（如 16 个目录归入 4 个类别）。
+ * 未命中分组的目录、以及「概览」保持顶层。
+ */
+export function applyMenuGroups(roots, groups) {
+  const byId = new Map(roots.map((r) => [r.id, r]))
+  const used = new Set()
+  const out = []
+
+  const overview = roots.find((r) => r.id.startsWith('00-'))
+  if (overview) {
+    out.push(overview)
+    used.add(overview.id)
+  }
+
+  groups.forEach((g, i) => {
+    const children = g.dirs.map((d) => byId.get(d)).filter(Boolean)
+    children.forEach((c) => used.add(c.id))
+    if (children.length > 0) {
+      out.push({ type: 'group', id: `group-${i}`, title: g.title, children })
+    }
+  })
+
+  for (const r of roots) {
+    if (!used.has(r.id)) out.push(r)
+  }
+  return out
+}

package/src/server.mjs

@@ -0,0 +1,64 @@
+/**
+ * 极简静态服务器（dev 预览用），无第三方依赖。
+ * 支持 SPA 兜底：无扩展名的路径回退到 index.html。
+ */
+import http from 'node:http'
+import fs from 'node:fs'
+import path from 'node:path'
+
+const MIME = {
+  '.html': 'text/html; charset=utf-8',
+  '.js': 'text/javascript; charset=utf-8',
+  '.css': 'text/css; charset=utf-8',
+  '.json': 'application/json; charset=utf-8',
+  '.svg': 'image/svg+xml',
+  '.png': 'image/png',
+  '.jpg': 'image/jpeg',
+  '.jpeg': 'image/jpeg',
+  '.gif': 'image/gif',
+  '.webp': 'image/webp',
+  '.woff': 'font/woff',
+  '.woff2': 'font/woff2',
+  '.ttf': 'font/ttf',
+  '.ico': 'image/x-icon',
+}
+
+export function startServer({ dir, port = 4321, host = '127.0.0.1' }) {
+  const root = path.resolve(dir)
+
+  const server = http.createServer((req, res) => {
+    const urlPath = decodeURIComponent((req.url || '/').split('?')[0])
+    let file = path.resolve(root, `.${path.posix.normalize(urlPath)}`)
+
+    // 防目录穿越
+    if (!file.startsWith(root)) {
+      res.writeHead(403, { 'Content-Type': 'text/plain; charset=utf-8' }).end('Forbidden')
+      return
+    }
+
+    if (fs.existsSync(file) && fs.statSync(file).isDirectory()) {
+      file = path.join(file, 'index.html')
+    }
+    if (!fs.existsSync(file)) {
+      // 无扩展名路径回退首页（hash 路由本身不需要，但避免直接访问子路径 404）
+      file = path.join(root, 'index.html')
+    }
+
+    fs.readFile(file, (err, data) => {
+      if (err) {
+        res.writeHead(404, { 'Content-Type': 'text/plain; charset=utf-8' }).end('Not Found')
+        return
+      }
+      res.writeHead(200, {
+        'Content-Type': MIME[path.extname(file).toLowerCase()] || 'application/octet-stream',
+        'Cache-Control': 'no-cache',
+      })
+      res.end(data)
+    })
+  })
+
+  server.listen(port, host, () => {
+    console.log(`\n🌐 本地预览：http://${host}:${port}/\n`)
+  })
+  return server
+}

package/src/sync.mjs

@@ -0,0 +1,45 @@
+/**
+ * 笔记源同步：local（复制本地目录）或 git（浅克隆 / fetch+reset）到 workspace。
+ */
+import fs from 'node:fs'
+import path from 'node:path'
+import { execSync } from 'node:child_process'
+
+function run(cmd, cwd) {
+  // 非交互环境下避免 git 卡在凭证提示，直接失败
+  execSync(cmd, { cwd, stdio: 'inherit', env: { ...process.env, GIT_TERMINAL_PROMPT: '0' } })
+}
+
+export function syncSource(config) {
+  const { source, root, content } = config
+  if (!source) return { skipped: true }
+
+  const workspace = path.resolve(root, source.workspace)
+
+  if (source.mode === 'local') {
+    const ignore = new Set(content.exclude || [])
+    if (!fs.existsSync(source.dir)) throw new Error(`本地笔记目录不存在：${source.dir}`)
+    fs.rmSync(workspace, { recursive: true, force: true })
+    fs.mkdirSync(workspace, { recursive: true })
+    const entries = fs.readdirSync(source.dir, { withFileTypes: true })
+    let copied = 0
+    for (const e of entries) {
+      if (ignore.has(e.name)) continue
+      fs.cpSync(path.join(source.dir, e.name), path.join(workspace, e.name), { recursive: true })
+      copied++
+    }
+    return { mode: 'local', from: source.dir, to: workspace, copied }
+  }
+
+  const { url, branch, depth } = source.git
+  const depthArg = depth ? `--depth ${depth}` : ''
+  const isRepo = fs.existsSync(path.join(workspace, '.git'))
+  if (!isRepo) {
+    fs.rmSync(workspace, { recursive: true, force: true })
+    run(`git clone ${depthArg} --branch ${branch} -- ${url} ${workspace}`, root)
+  } else {
+    run(`git fetch ${depthArg} origin ${branch}`, workspace)
+    run('git reset --hard FETCH_HEAD', workspace)
+  }
+  return { mode: 'git', from: url, to: workspace, branch }
+}