@plutocms/docs 0.0.0-stage → 0.16.0

package/modules/pluto-docs/blocks.ts

@@ -0,0 +1,217 @@
+// Find the block components of a site and read their definitions at build time.
+// A component is a block only when its file calls `definePlutoBlock(...)`. The props and slots come
+// from the TypeScript meta (`nuxt-component-meta`), the label, icon and more from the call.
+// This file has no Nuxt import, so plain Vitest tests it.
+
+import type { PlutoComponentMeta } from '../../shared/utils/blocks/component-meta'
+import type {
+  PlutoBlockPatch,
+  PlutoBlocksConfig,
+} from '../../shared/utils/blocks/types'
+import { createHash } from 'node:crypto'
+import { readdirSync, readFileSync } from 'node:fs'
+import { basename, join } from 'node:path'
+import { blockFromComponentMeta } from '../../shared/utils/blocks/component-meta'
+
+/** The name of the macro. A component file calls it in `<script setup>`. */
+export const BLOCK_MACRO = 'definePlutoBlock'
+
+/** The key of the macro argument in the output of the parser. */
+export const MACRO_KEY = 'pluto'
+
+/** The folder of the components that Nuxt Content renders, in the app and in each layer. */
+export const CONTENT_COMPONENTS_DIR = 'components/content'
+
+export interface BlockFile {
+  /** The component name in Pascal case: `PlutoHero`. */
+  name: string
+  file: string
+}
+
+/** The output of the parser, with the macro argument under `pluto`. */
+export type ParsedComponent = PlutoComponentMeta & {
+  [MACRO_KEY]?: PlutoBlockPatch
+}
+
+export type MetaReader = (file: string) => ParsedComponent
+
+export interface BlockEntry {
+  /** The hash of the file text. The entry is valid while the hash is the same. */
+  hash: string
+  block?: { name: string; definition: PlutoBlockPatch }
+  warnings: string[]
+  /** The time of the parse, in milliseconds. 0 when the entry came from the cache. */
+  ms: number
+}
+
+export interface BlockLog {
+  warn: (message: string) => void
+  debug: (message: string) => void
+}
+
+/** Whether the file text calls the macro: `definePlutoBlock(...)` or `definePlutoBlock<Props>(...)`. */
+export function hasBlockMacro(text: string): boolean {
+  return /\bdefinePlutoBlock\s*[(<]/.test(text)
+}
+
+/**
+ * The component files of the `components/content` folders. The folders come in order of priority
+ * (the app first, then its layers). When two files have the same name, the first one wins. A
+ * `Prose*` file is a prose override, an `.client` or `.server` file is a variant: none is a block.
+ */
+export function findContentComponents(dirs: string[]): BlockFile[] {
+  const found = new Map<string, BlockFile>()
+  const visit = (dir: string) => {
+    let entries
+    try {
+      entries = readdirSync(dir, { withFileTypes: true })
+    } catch {
+      return
+    }
+    for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
+      const path = join(dir, entry.name)
+      if (entry.isDirectory()) {
+        visit(path)
+        continue
+      }
+      if (!entry.name.endsWith('.vue')) {
+        continue
+      }
+      const name = basename(entry.name, '.vue')
+      if (
+        name.startsWith('Prose') ||
+        name.endsWith('.client') ||
+        name.endsWith('.server') ||
+        found.has(name)
+      ) {
+        continue
+      }
+      found.set(name, { name, file: path })
+    }
+  }
+  for (const dir of dirs) {
+    visit(dir)
+  }
+  return [...found.values()]
+}
+
+function hashOf(text: string): string {
+  return createHash('sha1').update(text).digest('hex')
+}
+
+/**
+ * Read the block of one component file. A file without the macro gives an entry with no block, and
+ * the parser does not run. A parse error gives a warning and no block. A macro that the parser
+ * cannot evaluate (for example, an argument that uses a local variable) gives a warning and no
+ * block: a half-read definition would hide the mistake.
+ */
+export function readBlockEntry(
+  { name, file }: BlockFile,
+  text: string,
+  read: MetaReader,
+  now: () => number = Date.now
+): BlockEntry {
+  const hash = hashOf(text)
+  if (!hasBlockMacro(text)) {
+    return { hash, warnings: [], ms: 0 }
+  }
+  const start = now()
+  let parsed: ParsedComponent
+  try {
+    parsed = read(file)
+  } catch (error) {
+    return {
+      hash,
+      warnings: [
+        `${BLOCK_MACRO} in ${file} was skipped. The component did not parse: ${error instanceof Error ? error.message : String(error)}`,
+      ],
+      ms: now() - start,
+    }
+  }
+  const ms = now() - start
+  const macro = parsed[MACRO_KEY]
+  if (!macro) {
+    return {
+      hash,
+      warnings: [
+        `${BLOCK_MACRO} in ${file} was skipped. Its argument must be an object literal that does not use a local variable. It can use a constant imported from a file.`,
+      ],
+      ms,
+    }
+  }
+  // The key is not a block field. The meta is only props and slots.
+  const { [MACRO_KEY]: _macro, ...meta } = parsed
+  return {
+    hash,
+    block: blockFromComponentMeta(name, pickMeta(meta), macro),
+    warnings: [],
+    ms,
+  }
+}
+
+function pickMeta(meta: ParsedComponent): PlutoComponentMeta {
+  return { props: meta.props, slots: meta.slots }
+}
+
+/**
+ * The component layer of a site: the blocks of the component files, with a cache by file. A file
+ * is parsed again only when its text changed. Call `collect` for each build and each hot update.
+ */
+export class BlockCatalog {
+  private cache = new Map<string, BlockEntry>()
+
+  constructor(
+    private dirs: () => string[],
+    private read: MetaReader,
+    private log: BlockLog,
+    private now: () => number = Date.now
+  ) {}
+
+  collect(): PlutoBlocksConfig {
+    const start = this.now()
+    const files = findContentComponents(this.dirs())
+    const result: PlutoBlocksConfig = {}
+    const seen = new Set<string>()
+    let parsed = 0
+    for (const item of files) {
+      seen.add(item.file)
+      let text: string
+      try {
+        text = readFileSync(item.file, 'utf8')
+      } catch {
+        continue
+      }
+      let entry = this.cache.get(item.file)
+      if (!entry || entry.hash !== hashOf(text)) {
+        entry = readBlockEntry(item, text, this.read, this.now)
+        this.cache.set(item.file, entry)
+        for (const warning of entry.warnings) {
+          this.log.warn(warning)
+        }
+        if (entry.block) {
+          parsed++
+          this.log.debug(`${item.name}: ${entry.ms} ms`)
+        }
+      }
+      if (entry.block) {
+        result[entry.block.name] = entry.block.definition
+      }
+    }
+    for (const file of this.cache.keys()) {
+      if (!seen.has(file)) {
+        this.cache.delete(file)
+      }
+    }
+    if (parsed > 0) {
+      this.log.debug(
+        `Read ${parsed} block component(s) in ${this.now() - start} ms.`
+      )
+    }
+    return result
+  }
+}
+
+/** The source of the Vite virtual module for a component layer. */
+export function blocksModuleSource(blocks: PlutoBlocksConfig): string {
+  return `export default ${JSON.stringify(blocks)}\n`
+}

package/modules/pluto-docs/index.ts

@@ -0,0 +1,230 @@
+import type { Nuxt } from '@nuxt/schema'
+import type { PlutoDocsOptions } from './options'
+import { join } from 'node:path'
+import {
+  addImports,
+  addServerHandler,
+  addVitePlugin,
+  createResolver,
+  defineNuxtModule,
+  useLogger,
+} from '@nuxt/kit'
+import { PREVIEW_PATH } from '../../shared/utils/blocks/preview-protocol'
+import {
+  BLOCK_MACRO,
+  BlockCatalog,
+  blocksModuleSource,
+  CONTENT_COMPONENTS_DIR,
+  MACRO_KEY,
+} from './blocks'
+import { buildContentManifest, manifestModuleSource } from './manifest'
+import { resolvePlutoDocsOptions } from './options'
+
+export type { PlutoDocsOptions, PlutoDocsTypeOptions } from './options'
+
+/** The routes of the run-time render mode, on the public API of Pluto. The composables of the layer call them. */
+export const RUNTIME_ROUTE_BASE = '/api/pluto/v1/docs'
+
+/**
+ * Registers the content types of the docs layer. It is OFF when the
+ * app turns the standalone CMS off (`plutoGithub.enabled: false`). The default
+ * `'auto'` and `true` both count as on. The module only checks the options
+ * and copies them to the public runtime config. The client plugin
+ * (`app/plugins/pluto-docs.ts`) and the Nitro plugin
+ * (`server/plugins/pluto-docs.ts`) of the layer build the types from
+ * that config. See the docs-setup skill.
+ */
+export default defineNuxtModule<PlutoDocsOptions>({
+  meta: { name: 'pluto-docs', configKey: 'plutoDocs' },
+  defaults: { types: [] },
+  setup(options, nuxt) {
+    const logger = useLogger('pluto-docs')
+    const cms = (
+      nuxt.options as { plutoGithub?: { enabled?: boolean | 'auto' } }
+    ).plutoGithub
+
+    const { enabled, render, runtimeMaxAge, types, blocks, errors } =
+      resolvePlutoDocsOptions(options, {
+        cmsEnabled: cms?.enabled !== false,
+      })
+
+    for (const error of errors) {
+      logger.error(error)
+    }
+    // `definePlutoBlock(...)` in a component file. A typed no-op at run time: the build reads the call.
+    addImports({
+      name: BLOCK_MACRO,
+      from: createResolver(import.meta.url).resolve(
+        '../../shared/utils/blocks/define'
+      ),
+    })
+
+    // The editor imports the virtual module even when the layer is off, so it always exists.
+    setupBlockDiscovery(nuxt, logger, enabled && blocks.discover)
+
+    if (!enabled) {
+      return
+    }
+
+    // The plugins read this key. It is absent when the layer is off.
+    nuxt.options.runtimeConfig.public.plutoDocs = {
+      types,
+      render,
+      runtimeMaxAge,
+    }
+
+    if (render === 'hybrid' || render === 'runtime') {
+      const { resolve } = createResolver(import.meta.url)
+      // The toc rules of Nuxt Content, so the run-time parser makes the same table of contents.
+      const content = (
+        nuxt.options as {
+          content?: {
+            build?: {
+              markdown?: { toc?: { depth?: number; searchDepth?: number } }
+            }
+          }
+        }
+      ).content
+      nuxt.options.runtimeConfig.plutoDocs = {
+        toc: content?.build?.markdown?.toc,
+      } as never
+      for (const name of ['page', 'navigation', 'surround']) {
+        addServerHandler({
+          route: `${RUNTIME_ROUTE_BASE}/:type/${name}`,
+          method: 'get',
+          handler: resolve(`../../server/handlers/${name}.get`),
+        })
+      }
+    }
+
+    // The build manifest: content id → version, for each type. The server compares it with the
+    // store (hybrid). It is a virtual module, so it is part of the server bundle and matches the
+    // pages that Nuxt Content builds in the same run. It is made once, when the bundler first asks.
+    const makeManifest = () => {
+      // The CMS module sets the storage and the content dir. They are known here, after all modules ran.
+      const cmsConfig = nuxt.options.runtimeConfig.public.plutoGithub as
+        { storage?: string; dir?: string } | undefined
+      return buildContentManifest(types, {
+        contentDir: join(nuxt.options.rootDir, cmsConfig?.dir ?? 'content'),
+        storage: cmsConfig?.storage ?? 'github',
+      })
+    }
+
+    const manifestId = '#pluto-docs/manifest'
+    let manifestSource: string | undefined
+    nuxt.hook('nitro:config', (nitroConfig) => {
+      nitroConfig.virtual ||= {}
+      nitroConfig.virtual[manifestId] = () => {
+        if (manifestSource === undefined) {
+          manifestSource = manifestModuleSource(
+            render === 'hybrid' ? makeManifest() : {}
+          )
+        }
+        return manifestSource
+      }
+    })
+
+    // The preview page of the visual editor is a client-only page that no search engine should list.
+    // It is not prerendered, and the response says noindex. An app rule for the same path wins.
+    nuxt.options.routeRules = {
+      [PREVIEW_PATH]: {
+        ssr: false,
+        prerender: false,
+        headers: { 'x-robots-tag': 'noindex, nofollow' },
+      },
+      ...nuxt.options.routeRules,
+    }
+
+    if (!nuxt.options._prepare) {
+      logger.info(
+        `Docs layer on. Types: ${types.map((type) => type.name).join(', ') || 'none'}. Render: ${render}.`
+      )
+    }
+  },
+})
+
+/**
+ * The component layer of the block editor. The virtual module `#pluto-docs/blocks` holds the blocks
+ * of the `components/content/` folders. Only the two lazy editor roots import it, so it stays out of
+ * the public pages. The server build gets an empty object: the editor never renders on the server.
+ * The cost comes when the editor first loads, not at start-up. A change to a component file makes
+ * the module load again, and the catalog parses only the file that changed.
+ */
+function setupBlockDiscovery(
+  nuxt: Nuxt,
+  logger: ReturnType<typeof useLogger>,
+  discover: boolean
+) {
+  const id = '#pluto-docs/blocks'
+  // The `\0` prefix marks a virtual id. No `#` in it: a browser URL ends at the `#`.
+  const resolved = '\0pluto-docs-blocks'
+  // The app first, then its layers (the order of priority).
+  const dirs = () =>
+    nuxt.options._layers.map((layer) =>
+      join(layer.config.srcDir, CONTENT_COMPONENTS_DIR)
+    )
+
+  // The cache of the parser names a file from the text after the last slash, so give it slashes.
+  const slashes = (path: string) => path.replaceAll('\\', '/')
+
+  type Reader = typeof import('nuxt-component-meta/parser').getComponentMeta
+  let getComponentMeta: Reader | undefined
+  const catalog = new BlockCatalog(
+    dirs,
+    (file) =>
+      getComponentMeta!(slashes(file), {
+        rootDir: nuxt.options.rootDir,
+        cache: true,
+        cacheDir: join(nuxt.options.buildDir, 'cache/pluto-blocks'),
+        extendMetaFunctions: [
+          { name: BLOCK_MACRO, transform: (arg) => ({ [MACRO_KEY]: arg }) },
+        ],
+      }) as never,
+    logger
+  )
+
+  const isBlockFile = (file: string) =>
+    file.endsWith('.vue') &&
+    dirs().some((dir) => slashes(file).startsWith(slashes(dir)))
+
+  addVitePlugin({
+    name: 'pluto-docs:blocks',
+    enforce: 'pre',
+    resolveId(source) {
+      return source === id ? resolved : undefined
+    },
+    async load(loaded, options) {
+      if (loaded !== resolved) {
+        return undefined
+      }
+      if (options?.ssr || !discover) {
+        return blocksModuleSource({})
+      }
+      // The parser loads here, not at start-up, because it brings the TypeScript compiler.
+      getComponentMeta ||= (await import('nuxt-component-meta/parser'))
+        .getComponentMeta
+      return blocksModuleSource(catalog.collect())
+    },
+    configureServer(server) {
+      const invalidate = (file: string) => {
+        if (!isBlockFile(file)) {
+          return
+        }
+        const module = server.moduleGraph.getModuleById(resolved)
+        if (module) {
+          server.moduleGraph.invalidateModule(module)
+          server.ws.send({ type: 'full-reload' })
+        }
+      }
+      server.watcher.on('add', invalidate)
+      server.watcher.on('unlink', invalidate)
+    },
+    handleHotUpdate({ file, server }) {
+      if (!isBlockFile(file)) {
+        return undefined
+      }
+      const module = server.moduleGraph.getModuleById(resolved)
+      return module ? [module] : undefined
+    },
+  })
+}

package/modules/pluto-docs/manifest.ts

@@ -0,0 +1,159 @@
+// The build manifest of the hybrid mode. Node code that runs at build time. It lists the
+// content files of each type with the version that the content store reports for the file
+// (`_version`). At run time the server compares the store with this list to see which pages
+// changed after the build. See the docs-setup skill.
+
+import { Buffer } from 'node:buffer'
+import { createHash } from 'node:crypto'
+import { readdirSync, readFileSync } from 'node:fs'
+import { join, relative, sep } from 'node:path'
+
+/** Content id → version, for one type. */
+export type TypeManifest = Record<string, string>
+
+/** Type name → its manifest. */
+export type ContentManifest = Record<string, TypeManifest>
+
+/** The same dirs that the CMS fs store skips. */
+const SKIPPED_DIRS = new Set([
+  'node_modules',
+  '.nuxt',
+  '.output',
+  '.data',
+  '.nitro',
+  '.cache',
+  'dist',
+])
+const SEGMENT_PATTERN = /^(?:\d+\.)?[a-z0-9]+(?:-[a-z0-9]+)*$/
+const MAX_SEGMENTS = 6
+const MAX_ID_LENGTH = 255
+
+/** The Git blob id of a text: sha1 of `blob <byteLength>\0` and the UTF-8 bytes. Core does the same. */
+export function gitBlobSha(text: string): string {
+  const bytes = Buffer.from(text, 'utf8')
+  return createHash('sha1')
+    .update(`blob ${bytes.length}\0`)
+    .update(bytes)
+    .digest('hex')
+}
+
+/**
+ * The version of a file text for a storage.
+ *
+ * - `github`: the store reports the Git blob id. Git keeps LF in the blob, so a CRLF working file
+ *   (Windows with `core.autocrlf`) is hashed with LF. A file that really has CRLF in the repository
+ *   looks changed at run time. That is safe: the page then comes from the store.
+ * - `fs` (and `memory`): the store hashes the file as it is on the disk, so the manifest does too.
+ */
+export function versionOf(text: string, storage: string): string {
+  return gitBlobSha(storage === 'github' ? text.replace(/\r\n/g, '\n') : text)
+}
+
+/** Whether a content id is one that the CMS lists: 1 to 6 safe segments, and not excluded. */
+export function isListedId(
+  id: string,
+  exclude: readonly string[] = []
+): boolean {
+  if (id.length === 0 || id.length > MAX_ID_LENGTH) {
+    return false
+  }
+  const segments = id.split('/')
+  if (
+    segments.length > MAX_SEGMENTS ||
+    segments.some((segment) => !SEGMENT_PATTERN.test(segment))
+  ) {
+    return false
+  }
+  return !exclude.some((entry) =>
+    entry.endsWith('/') ? id.startsWith(entry) : id === entry
+  )
+}
+
+/** One file in memory: the path is relative to the type folder, with `/`. */
+export interface ManifestFile {
+  path: string
+  text: string
+}
+
+/** Builds the manifest of one type from its files. A file that is not a listed `.md` page is skipped. */
+export function buildTypeManifest(
+  files: ManifestFile[],
+  options: { storage: string; exclude?: readonly string[] }
+): TypeManifest {
+  const manifest: TypeManifest = {}
+  for (const file of [...files].sort((a, b) =>
+    a.path < b.path ? -1 : a.path > b.path ? 1 : 0
+  )) {
+    if (!file.path.endsWith('.md')) {
+      continue
+    }
+    const id = file.path.slice(0, -'.md'.length)
+    if (!isListedId(id, options.exclude)) {
+      continue
+    }
+    manifest[id] = versionOf(file.text, options.storage)
+  }
+  return manifest
+}
+
+function walkMarkdown(dir: string, out: string[]): void {
+  let entries
+  try {
+    entries = readdirSync(dir, { withFileTypes: true })
+  } catch {
+    return
+  }
+  for (const entry of entries) {
+    if (entry.name.startsWith('.')) {
+      continue
+    }
+    const full = join(dir, entry.name)
+    if (entry.isDirectory()) {
+      if (!SKIPPED_DIRS.has(entry.name)) {
+        walkMarkdown(full, out)
+      }
+    } else if (entry.isFile() && entry.name.endsWith('.md')) {
+      out.push(full)
+    }
+  }
+}
+
+/** Reads the `.md` files under a type folder on the disk. A missing folder gives `[]`. */
+export function readTypeFiles(folder: string): ManifestFile[] {
+  const found: string[] = []
+  walkMarkdown(folder, found)
+  return found.map((full) => ({
+    path: relative(folder, full).split(sep).join('/'),
+    text: readFileSync(full, 'utf8'),
+  }))
+}
+
+export interface ManifestTypeInput {
+  name: string
+  /** The folder under the CMS content dir. `''` is the content dir itself. */
+  source: string
+  exclude?: string[]
+}
+
+/** Builds the manifest of every type. `contentDir` is the absolute path of the CMS content dir. */
+export function buildContentManifest(
+  types: ManifestTypeInput[],
+  options: { contentDir: string; storage: string }
+): ContentManifest {
+  const manifest: ContentManifest = {}
+  for (const type of types) {
+    const folder = type.source
+      ? join(options.contentDir, type.source)
+      : options.contentDir
+    manifest[type.name] = buildTypeManifest(readTypeFiles(folder), {
+      storage: options.storage,
+      exclude: type.exclude,
+    })
+  }
+  return manifest
+}
+
+/** The source of the virtual module that the server imports. */
+export function manifestModuleSource(manifest: ContentManifest): string {
+  return `export default ${JSON.stringify(manifest)}\n`
+}