@lincyaw/workspace 1.0.0-rc.0 → 1.0.0-rc.10

package/src/node/compile-session.ts

@@ -0,0 +1,187 @@
+import { existsSync, statSync, watch } from 'node:fs'
+import type { FSWatcher } from 'node:fs'
+import { basename, dirname, join } from 'node:path'
+import type { Compilation, Compiler } from '@lincyaw/compiler'
+import { errorMessage } from '@lincyaw/kernel'
+import type { Logger } from '@lincyaw/kernel'
+
+/** One document of the workspace server: its compiler, the latest compilation, and recompiling when a file it read changes. */
+export class CompileSession {
+  readonly dir: string
+  readonly compiler: Compiler
+  #latest?: Compilation
+  #first?: Promise<Compilation>
+  #running: Promise<unknown> = Promise.resolve()
+  #queued?: Promise<Compilation>
+  readonly #listeners = new Set<(compilation: Compilation, changed: string[]) => void | Promise<void>>()
+  readonly #failures = new Set<(error: unknown) => void>()
+  #changedFiles = new Set<string>()
+  readonly #watchers = new Map<string, FSWatcher>()
+  #relevant = new Map<string, Set<string>>()
+  #timer?: NodeJS.Timeout
+  #closed = false
+  #compiling = false
+  #version = 0
+  #controller?: AbortController
+  readonly #observed = new Map<string, number>()
+  readonly #log: Logger
+
+  constructor(dir: string, compiler: Compiler, log: Logger) {
+    this.dir = dir
+    this.compiler = compiler
+    this.#log = log
+  }
+
+  ready(): Promise<Compilation> {
+    if (this.#latest) return Promise.resolve(this.#latest)
+    if (!this.#first) {
+      const first = this.recompile()
+      first.catch(() => {
+        if (this.#first === first) this.#first = undefined
+      })
+      this.#first = first
+    }
+    return this.#first
+  }
+
+  get latest(): Compilation | undefined {
+    return this.#latest
+  }
+
+  onCompile(listener: (compilation: Compilation, changed: string[]) => void | Promise<void>): () => void {
+    this.#listeners.add(listener)
+    return () => this.#listeners.delete(listener)
+  }
+
+  onFailure(listener: (error: unknown) => void): () => void {
+    this.#failures.add(listener)
+    return () => { this.#failures.delete(listener) }
+  }
+
+  recompile(): Promise<Compilation> {
+    if (this.#queued) return this.#queued
+    const next = this.#running.then(async () => {
+      this.#queued = undefined
+      this.#compiling = true
+      try {
+        let compilation: Compilation | undefined
+        const changed = new Set<string>()
+        let version: number
+        do {
+          version = this.#version
+          for (const file of this.#changedFiles) changed.add(file)
+          this.#changedFiles.clear()
+          const started = Date.now()
+          this.#log.info(`[compile] document=${JSON.stringify(this.dir || '.')} state=started changed=${changed.size}`)
+          const controller = this.#controller = new AbortController()
+          try {
+            compilation = await this.compiler.compile({ signal: controller.signal })
+          } catch (error) {
+            if (!this.#closed && controller.signal.aborted && version !== this.#version) continue
+            throw error
+          }
+          this.#watch(compilation, started)
+          this.#log.info(`[compile] document=${JSON.stringify(this.dir || '.')} state=compiled elapsedMs=${Date.now() - started} sections=${compilation.doc.sections.length} diagnostics=${compilation.doc.diagnostics.length}`)
+        } while (!this.#closed && version !== this.#version)
+        if (!compilation) throw new Error('the compilation did not complete')
+        if (this.#closed) return compilation
+        this.#latest = compilation
+        for (const listener of this.#listeners) {
+          try {
+            await listener(compilation, [...changed])
+          } catch (error) {
+            this.#log.error(`after compiling ${this.dir || '.'}: ${errorMessage(error)}`)
+          }
+        }
+        return compilation
+      } finally {
+        this.#controller = undefined
+        this.#compiling = false
+        if (!this.#closed && this.#changedFiles.size > 0) this.#schedule()
+      }
+    })
+    this.#queued = next
+    this.#running = next.catch((error: unknown) => {
+      if (!this.#closed) {
+        this.#log.error(`compiling ${this.dir || '.'} failed: ${errorMessage(error)}`)
+        for (const listener of this.#failures) listener(error)
+      }
+    })
+    return next
+  }
+
+  close(): void {
+    this.#closed = true
+    this.#controller?.abort()
+    clearTimeout(this.#timer)
+    for (const watcher of this.#watchers.values()) watcher.close()
+    this.#watchers.clear()
+    this.#listeners.clear()
+    this.#failures.clear()
+  }
+
+  #watch(compilation: Compilation, started: number): void {
+    if (this.#closed) return
+    const files = new Set([...compilation.sourceFiles, ...compilation.inputs.files])
+    const next = new Map<string, Set<string>>()
+    for (const file of files) {
+      let dir = dirname(file)
+      let name = basename(file)
+      while (!existsSync(dir) && dirname(dir) !== dir) {
+        name = basename(dir)
+        dir = dirname(dir)
+      }
+      next.set(dir, (next.get(dir) ?? new Set<string>()).add(name))
+    }
+    this.#relevant = next
+    for (const [dir, watcher] of this.#watchers) {
+      if (next.has(dir)) continue
+      watcher.close()
+      this.#watchers.delete(dir)
+    }
+    for (const dir of next.keys()) {
+      if (this.#watchers.has(dir)) continue
+      const failed = (error: unknown) => {
+        if (this.#closed) return
+        const failure = new Error(`cannot watch ${dir}: ${errorMessage(error)}`)
+        this.#log.error(failure.message)
+        for (const listener of this.#failures) listener(failure)
+      }
+      try {
+        const watcher = watch(dir, (_event, name) => {
+          if (name && this.#relevant.get(dir)?.has(basename(name))) this.#changed(join(dir, name))
+        })
+        this.#watchers.set(dir, watcher)
+        watcher.on('error', (error) => {
+          watcher.close()
+          if (this.#watchers.get(dir) === watcher) this.#watchers.delete(dir)
+          failed(error)
+        })
+      } catch (error) {
+        failed(error)
+      }
+    }
+    const modified = [...files].filter((file) => {
+      const time = statSync(file, { throwIfNoEntry: false })?.mtimeMs ?? 0
+      const changed = time >= started - 20 && time <= Date.now() && this.#observed.get(file) !== time
+      this.#observed.set(file, time)
+      return changed
+    })
+    for (const file of this.#observed.keys()) if (!files.has(file)) this.#observed.delete(file)
+    for (const file of modified) this.#changed(file)
+  }
+
+  #changed(file: string): void {
+    if (this.#closed) return
+    this.#changedFiles.add(file)
+    this.#observed.set(file, statSync(file, { throwIfNoEntry: false })?.mtimeMs ?? 0)
+    this.#version++
+    this.#controller?.abort()
+    if (!this.#compiling) this.#schedule()
+  }
+
+  #schedule(): void {
+    clearTimeout(this.#timer)
+    this.#timer = setTimeout(() => void this.recompile().catch(() => undefined), 10)
+  }
+}

package/src/node/config.ts

@@ -1,26 +1,24 @@
 import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs'
 import { join, resolve } from 'node:path'
-import { DECK_ENTRY, readFrontmatter, WORKSPACE_CONFIG } from '@lincyaw/compiler'
+import { WORKSPACE_CONFIG } from '@lincyaw/compiler'
 import type { OperatorSettings } from '@lincyaw/compiler'
-import { errorMessage, isRecord } from '@lincyaw/kernel'
-import type { ApprovalKind } from '@lincyaw/kernel'
+import { APPROVAL_KINDS, DOCUMENT_ENTRY, errorMessage, isRecord } from '@lincyaw/kernel'
+import type { ApprovalKind, DocumentSummary } from '@lincyaw/kernel'
 import type { ApprovalPolicy } from '@lincyaw/server'
 import { parse as parseYaml } from 'yaml'
-import type { WorkspaceDocument } from '../domain/api.ts'
 import { matchesPattern, patternSegments } from '../domain/paths.ts'
 
-export const DEFAULT_DOCUMENTS = [`*/${DECK_ENTRY}`]
+export const DEFAULT_DOCUMENTS = [`*/${DOCUMENT_ENTRY}`]
 
 const SKIPPED = new Set(['node_modules', '.git', '.presmium', 'dist'])
 
-const KINDS: ApprovalKind[] = ['operate', 'spend', 'write', 'agent']
 const POLICIES = new Set<ApprovalPolicy>(['off', 'ask', 'allow'])
 
 export interface WorkspaceConfig {
   root: string
   title: string
   documents: string[]
-  single: boolean
+  singleDeck: boolean
   site: Record<string, unknown>
   approvals: Partial<Record<ApprovalKind, ApprovalPolicy>>
   templates?: string
@@ -31,7 +29,7 @@ function strings(value: unknown): string[] {
   return Array.isArray(value) ? value.filter((v): v is string => typeof v === 'string' && v.trim() !== '').map(v => v.trim()) : typeof value === 'string' && value.trim() ? [value.trim()] : []
 }
 
-export function readWorkspace(root: string): WorkspaceConfig {
+export function readWorkspaceConfig(root: string): WorkspaceConfig {
   const dir = realpathSync(resolve(root))
   const file = join(dir, WORKSPACE_CONFIG)
   let data: Record<string, unknown> = {}
@@ -44,16 +42,20 @@ export function readWorkspace(root: string): WorkspaceConfig {
     }
   }
   const declared = strings(data.documents)
-  const single = declared.length === 0 && existsSync(join(dir, DECK_ENTRY))
+  const singleDeck = declared.length === 0 && existsSync(join(dir, DOCUMENT_ENTRY))
   const approvals = isRecord(data.approvals) ? data.approvals : {}
   const settings = isRecord(data.settings) ? data.settings : {}
+  const renderers = isRecord(data.renderers) ? data.renderers : {}
+  for (const [path, value] of [['renderers.lecture', renderers.lecture], ['settings.presmium.lecture', settings['presmium.lecture']]] as const) {
+    if (isRecord(value) && Object.hasOwn(value, 'script')) throw new Error(`${file}: ${path}.script: External lecture scripts are not supported. Author narration in each document's lecture.json narrations.`)
+  }
   return {
     root: dir,
     title: typeof data.title === 'string' && data.title.trim() ? data.title.trim() : dir.split('/').at(-1) ?? 'Presmium',
-    documents: single ? [DECK_ENTRY] : declared.length > 0 ? declared : DEFAULT_DOCUMENTS,
-    single,
+    documents: singleDeck ? [DOCUMENT_ENTRY] : declared.length > 0 ? declared : DEFAULT_DOCUMENTS,
+    singleDeck,
     site: isRecord(data.site) ? data.site : {},
-    approvals: Object.fromEntries(KINDS.flatMap(kind => (POLICIES.has(approvals[kind] as ApprovalPolicy) ? [[kind, approvals[kind] as ApprovalPolicy]] : []))),
+    approvals: Object.fromEntries(APPROVAL_KINDS.flatMap(kind => (POLICIES.has(approvals[kind] as ApprovalPolicy) ? [[kind, approvals[kind] as ApprovalPolicy]] : []))),
     ...typeof data.templates === 'string' && data.templates.trim() ? { templates: resolve(dir, data.templates.trim()) } : {},
     settings: Object.fromEntries(Object.entries(settings).map(([id, value]) => [id, isRecord(value) ? value : {}])),
   }
@@ -62,7 +64,7 @@ export function readWorkspace(root: string): WorkspaceConfig {
 export function isWorkspaceRoot(dir: string): boolean {
   if (!existsSync(join(dir, WORKSPACE_CONFIG))) return false
   try {
-    return !readWorkspace(dir).single
+    return !readWorkspaceConfig(dir).singleDeck
   } catch {
     return true
   }
@@ -87,7 +89,7 @@ function entries(config: WorkspaceConfig): string[] {
       return names.flatMap((name) => {
         const path = join(dir, name)
         const next = rel ? `${rel}/${name}` : name
-        if (last) return existsSync(path) && statSync(path).isFile() ? [next] : []
+        if (last) return name === DOCUMENT_ENTRY && existsSync(path) && statSync(path).isFile() ? [next] : []
         return existsSync(path) && statSync(path).isDirectory() ? walk(path, next, i + 1) : []
       })
     }
@@ -96,15 +98,23 @@ function entries(config: WorkspaceConfig): string[] {
   return found
 }
 
-export interface FoundDocument extends WorkspaceDocument {
+export interface DocumentEntry {
+  dir: string
   entry: string
 }
 
-function documentOf(config: WorkspaceConfig, entry: string): FoundDocument {
-  const dir = entry.split('/').slice(0, -1).join('/')
+export interface FoundDocument extends DocumentSummary, DocumentEntry {}
+
+/** The configured documents that exist on disk, without reading their content. */
+export function listDocumentEntries(config: WorkspaceConfig): DocumentEntry[] {
+  return entries(config).map(entry => ({ dir: entry.split('/').slice(0, -1).join('/'), entry }))
+}
+
+export function readDocumentSummary(config: WorkspaceConfig, { dir, entry }: DocumentEntry): FoundDocument {
   let headmatter: Record<string, unknown> = {}
   try {
-    headmatter = readFrontmatter(readFileSync(join(config.root, entry), 'utf8'))
+    const parsed: unknown = JSON.parse(readFileSync(join(config.root, entry), 'utf8'))
+    headmatter = isRecord(parsed) && isRecord(parsed.metadata) ? parsed.metadata : {}
   } catch {
   }
   const tags = strings(headmatter.tags)
@@ -112,15 +122,20 @@ function documentOf(config: WorkspaceConfig, entry: string): FoundDocument {
     dir,
     entry,
     title: typeof headmatter.title === 'string' && headmatter.title.trim() ? headmatter.title.trim() : dir || config.title,
+    ...typeof headmatter.description === 'string' && headmatter.description.trim() ? { description: headmatter.description.trim() } : {},
     ...typeof headmatter.lang === 'string' && headmatter.lang.trim() ? { lang: headmatter.lang.trim() } : {},
     ...tags.length > 0 ? { tags } : {},
   }
 }
 
 export function listDocuments(config: WorkspaceConfig): FoundDocument[] {
-  return entries(config).map(entry => documentOf(config, entry))
+  return listDocumentEntries(config).map(entry => readDocumentSummary(config, entry))
+}
+
+export function documentSummaries(config: WorkspaceConfig): DocumentSummary[] {
+  return listDocuments(config).map(({ entry: _entry, ...document }) => document)
 }
 
-export function namedByPatterns(config: WorkspaceConfig, dir: string): boolean {
-  return config.documents.some(pattern => matchesPattern(pattern, dir ? `${dir}/${DECK_ENTRY}` : DECK_ENTRY))
+export function isDocumentDir(config: WorkspaceConfig, dir: string): boolean {
+  return config.documents.some(pattern => matchesPattern(pattern, dir ? `${dir}/${DOCUMENT_ENTRY}` : DOCUMENT_ENTRY))
 }

package/src/node/document-assets.ts

@@ -0,0 +1,30 @@
+import { createReadStream, realpathSync } from 'node:fs'
+import { basename, isAbsolute, relative, resolve, sep } from 'node:path'
+import { Readable } from 'node:stream'
+import { lookup } from 'mime-types'
+import type { TeachingDocument } from '@lincyaw/kernel'
+import { RequestError } from './errors.ts'
+
+/** Read only a file registered by the current canonical document, without compiling it. */
+export function documentAssetResponse(root: string, directory: string, document: TeachingDocument, id: string, readableFile: (path: string) => string, signal?: AbortSignal): Response {
+  const asset = document.assets[id]
+  if (!asset) throw new RequestError(404, 'document-asset-missing')
+  if (!asset.location || /^(?:[a-z][a-z\d+.-]*:|\/)/i.test(asset.location)) throw new RequestError(400, 'document-asset-not-local')
+  let location = asset.location.replace(/[?#].*$/s, '')
+  try { location = decodeURIComponent(location) } catch { throw new RequestError(400, 'document-asset-path') }
+  if (!location || isAbsolute(location) || location.includes('\\') || location.includes('\0')) throw new RequestError(400, 'document-asset-path')
+  const workspace = realpathSync(root), target = resolve(workspace, directory, location)
+  const path = relative(workspace, target)
+  if (!path || path === '..' || path.startsWith(`..${sep}`) || isAbsolute(path)) throw new RequestError(403, 'document-asset-outside-workspace')
+  const file = readableFile(path.split(sep).join('/'))
+  const type = lookup(file) || 'application/octet-stream'
+  const inline = /^(?:image\/(?:png|jpeg|gif|webp|avif|svg\+xml|bmp|x-icon)|audio\/|video\/)/i.test(type)
+  return new Response(Readable.toWeb(createReadStream(file, { signal })), { headers: {
+    'content-type': type,
+    'cache-control': 'private, no-store',
+    'content-disposition': `${inline ? 'inline' : 'attachment'}; filename*=UTF-8''${encodeURIComponent(basename(file))}`,
+    'x-content-type-options': 'nosniff',
+    'cross-origin-resource-policy': 'same-origin',
+    'content-security-policy': "sandbox; default-src 'none'; style-src 'unsafe-inline'",
+  } })
+}