@crombolo/server 0.0.0-stage → 0.9.1

package/src/log-viewer.ts

@@ -0,0 +1,190 @@
+/**
+ * The log viewer over the site's log files, as Umbraco's reads Serilog's: the
+ * files of each UTC day in the range (`*yyyyMMdd*.json`), one CLEF event per
+ * line (a line that is not one is skipped), and the entries whose time falls
+ * in the range, both ends included.
+ */
+import { existsSync } from 'node:fs'
+import { readdir, stat } from 'node:fs/promises'
+import { join } from 'node:path'
+import type { LogEntry, LogLevelName, LogRange, LogViewerPort } from '@crombolo/api-management'
+import { type Db, LogSearchRepository } from '@crombolo/data'
+import type { CromboloConfig } from './config.ts'
+import { type LogEvent, logFilter } from './log-expressions.ts'
+import { SERILOG_LEVELS } from './logging.ts'
+
+/** Umbraco's `FileSizeCap`: ranges whose files come to more are refused. */
+export const LOG_SIZE_CAP_MB = 100
+
+const DAY = 24 * 60 * 60 * 1000
+
+/** Umbraco's default: when either end is missing, the last day. */
+function resolveRange(range: LogRange, now = new Date()): { start: Date; end: Date } {
+  if (!range.start || !range.end) return { start: new Date(now.getTime() - DAY), end: now }
+  return { start: range.start, end: range.end }
+}
+
+const dayStamp = (date: Date) => date.toISOString().slice(0, 10).replaceAll('-', '')
+
+async function filesFor(dir: string, range: { start: Date; end: Date }): Promise<string[]> {
+  if (!existsSync(dir)) return []
+  const names = (await readdir(dir)).filter((n) => n.endsWith('.json'))
+  const days = new Set<string>()
+  for (
+    let day = Date.UTC(
+      range.start.getUTCFullYear(),
+      range.start.getUTCMonth(),
+      range.start.getUTCDate(),
+    );
+    day <= range.end.getTime();
+    day += DAY
+  )
+    days.add(dayStamp(new Date(day)))
+  return names.filter((n) => [...days].some((d) => n.includes(d))).map((n) => join(dir, n))
+}
+
+/** Serilog's rendering of a property value in a message: strings quoted, the rest as written. */
+function renderProperty(value: unknown, format: string | undefined): string {
+  if (typeof value === 'string') return format === 'l' ? value : `"${value}"`
+  if (value === null) return 'null'
+  if (typeof value === 'object') return JSON.stringify(value)
+  return String(value)
+}
+
+/** A message template with its properties filled in, as Serilog's `RenderMessage` does. */
+export function renderMessage(template: string, properties: Record<string, unknown>): string {
+  return template.replace(
+    /\{\{|\}\}|\{([@$]?)([A-Za-z0-9_]+)(?:,[^:}]*)?(?::([^}]*))?\}/g,
+    (whole, _hint: string | undefined, name: string | undefined, format: string | undefined) => {
+      if (whole === '{{') return '{'
+      if (whole === '}}') return '}'
+      if (!name || !(name in properties)) return whole
+      return renderProperty(properties[name], format)
+    },
+  )
+}
+
+const LEVEL_NAMES = new Set(Object.values(SERILOG_LEVELS))
+
+/** One CLEF line as an event the viewer shows and filters; undefined when it is not one. */
+export function readClefLine(line: string): (LogEvent & { level: LogLevelName }) | undefined {
+  let raw: Record<string, unknown>
+  try {
+    raw = JSON.parse(line)
+  } catch {
+    return undefined
+  }
+  if (!raw || typeof raw !== 'object' || typeof raw['@t'] !== 'string') return undefined
+  const properties: Record<string, unknown> = {}
+  for (const [name, value] of Object.entries(raw))
+    if (!name.startsWith('@')) properties[name] = value
+  const template =
+    typeof raw['@mt'] === 'string' ? raw['@mt'] : typeof raw['@m'] === 'string' ? raw['@m'] : ''
+  const level =
+    typeof raw['@l'] === 'string' && LEVEL_NAMES.has(raw['@l']) ? raw['@l'] : 'Information'
+  return {
+    timestamp: raw['@t'],
+    level: level as LogLevelName,
+    messageTemplate: template,
+    message: typeof raw['@m'] === 'string' ? raw['@m'] : renderMessage(template, properties),
+    exception: typeof raw['@x'] === 'string' ? raw['@x'] : null,
+    properties,
+  }
+}
+
+async function readEvents(dir: string, range: { start: Date; end: Date }) {
+  const events: Array<LogEvent & { level: LogLevelName }> = []
+  const from = range.start.getTime()
+  const to = range.end.getTime()
+  for (const file of await filesFor(dir, range)) {
+    let content: string
+    try {
+      content = await Bun.file(file).text()
+    } catch {
+      continue
+    }
+    for (const line of content.split('\n')) {
+      if (!line.trim()) continue
+      const event = readClefLine(line)
+      if (!event) continue
+      const time = new Date(event.timestamp).getTime()
+      if (time >= from && time <= to) events.push(event)
+    }
+  }
+  return events
+}
+
+const propertyValue = (value: unknown): string | null =>
+  value === null || value === undefined
+    ? null
+    : typeof value === 'object'
+      ? JSON.stringify(value)
+      : String(value)
+
+export function createLogViewerPort(
+  db: Db,
+  config: Pick<CromboloConfig, 'logsDir' | 'logLevel'>,
+): LogViewerPort {
+  const searches = new LogSearchRepository(db)
+  return {
+    async canView(range) {
+      let bytes = 0
+      for (const file of await filesFor(config.logsDir, resolveRange(range)))
+        bytes += (await stat(file)).size
+      return Math.floor(bytes / 1024 / 1024) <= LOG_SIZE_CAP_MB
+    },
+
+    async levelCounts(range) {
+      const counts = { information: 0, debug: 0, warning: 0, error: 0, fatal: 0 }
+      for (const event of await readEvents(config.logsDir, resolveRange(range))) {
+        // Verbose is not counted, as in Umbraco
+        const key = event.level.toLowerCase() as keyof typeof counts
+        if (key in counts) counts[key] += 1
+      }
+      return counts
+    },
+
+    async templates(range) {
+      const counts = new Map<string, number>()
+      for (const event of await readEvents(config.logsDir, resolveRange(range)))
+        counts.set(event.messageTemplate, (counts.get(event.messageTemplate) ?? 0) + 1)
+      return [...counts]
+        .map(([messageTemplate, count]) => ({ messageTemplate, count }))
+        .sort((a, b) => b.count - a.count)
+    },
+
+    async logs(range, query) {
+      const matches = logFilter(query.filter)
+      const levels = new Set(query.levels.map((l) => l.toLowerCase()))
+      const events = (await readEvents(config.logsDir, resolveRange(range)))
+        .filter(matches)
+        .filter((event) => levels.size === 0 || levels.has(event.level.toLowerCase()))
+      const sign = query.direction === 'Ascending' ? 1 : -1
+      events.sort((a, b) => (Date.parse(a.timestamp) - Date.parse(b.timestamp)) * sign)
+      return events.map(
+        (event): LogEntry => ({
+          timestamp: event.timestamp,
+          level: event.level,
+          messageTemplate: event.messageTemplate,
+          renderedMessage: event.message,
+          exception: event.exception,
+          properties: Object.entries(event.properties).map(([name, value]) => ({
+            name,
+            value: propertyValue(value),
+          })),
+        }),
+      )
+    },
+
+    loggers: () => [
+      { name: 'Global', level: SERILOG_LEVELS[config.logLevel] as LogLevelName },
+      // Everything the loggers pass reaches the file
+      { name: 'CromboloFile', level: 'Verbose' },
+    ],
+
+    savedSearches: () => searches.all(),
+    savedSearch: (name) => searches.byName(name),
+    saveSearch: (name, query) => searches.create(name, query),
+    deleteSearch: (name) => searches.delete(name),
+  }
+}

package/src/logging.ts

@@ -0,0 +1,126 @@
+/**
+ * Logging, through LogTape, into the files Umbraco's log viewer reads: Serilog's
+ * compact JSON (CLEF), one event per line, one file per machine per UTC day in
+ * the site's `logs/` — `CromboloTraceLog.<machine>.<yyyyMMdd>.json`, after
+ * Umbraco's `UmbracoTraceLog.<machine>.<yyyyMMdd>.json`. A message is a
+ * template (`'Published {name}'`) with its properties, as Serilog's are; an
+ * `error` property carries the exception.
+ */
+import { appendFileSync, mkdirSync } from 'node:fs'
+import { hostname } from 'node:os'
+import { join } from 'node:path'
+import {
+  configureSync,
+  getConsoleSink,
+  getLogger,
+  type Logger,
+  type LogLevel,
+  type LogRecord,
+  type Sink,
+} from '@logtape/logtape'
+
+/** Serilog's level names, which the log viewer filters and counts by. */
+export const SERILOG_LEVELS: Record<LogLevel, string> = {
+  trace: 'Verbose',
+  debug: 'Debug',
+  info: 'Information',
+  warning: 'Warning',
+  error: 'Error',
+  fatal: 'Fatal',
+}
+
+export const LOG_FILE_PREFIX = 'CromboloTraceLog'
+
+/** Today's file for this machine: the day is UTC, as the timestamps are. */
+export function logFileName(date: Date, machine = hostname()): string {
+  const day = date.toISOString().slice(0, 10).replaceAll('-', '')
+  return `${LOG_FILE_PREFIX}.${machine.replace(/[^A-Za-z0-9_-]/g, '_')}.${day}.json`
+}
+
+const renderValue = (value: unknown): unknown => {
+  if (value instanceof Error) return value.message
+  if (value instanceof Date) return value.toISOString()
+  return value
+}
+
+/** One CLEF event: `@t`, `@mt`, `@l` (left out for Information, as Serilog does), `@x`, then properties. */
+export function toClef(record: LogRecord): Record<string, unknown> {
+  const { error, ...properties } = record.properties
+  const event: Record<string, unknown> = {
+    '@t': new Date(record.timestamp).toISOString(),
+    '@mt': typeof record.rawMessage === 'string' ? record.rawMessage : record.rawMessage.join('{}'),
+  }
+  if (record.level !== 'info') event['@l'] = SERILOG_LEVELS[record.level]
+  if (error !== undefined)
+    event['@x'] =
+      error instanceof Error ? (error.stack ?? `${error.name}: ${error.message}`) : String(error)
+  for (const [name, value] of Object.entries(properties)) event[name] = renderValue(value)
+  event.SourceContext = record.category.join('.')
+  event.MachineName = hostname()
+  event.ProcessId = process.pid
+  return event
+}
+
+/** Appends each event to the day's file; written synchronously, so nothing is lost on a crash. */
+export function getClefFileSink(dir: string): Sink {
+  let made = false
+  return (record) => {
+    if (!made) {
+      mkdirSync(dir, { recursive: true })
+      made = true
+    }
+    appendFileSync(
+      join(dir, logFileName(new Date(record.timestamp))),
+      `${JSON.stringify(toClef(record))}\n`,
+    )
+  }
+}
+
+export interface LoggingOptions {
+  logsDir: string
+  /** The least severe level written, for the files and the console alike. */
+  level: LogLevel
+  /** Whether events are also printed; tests turn it off. */
+  console: boolean
+}
+
+/**
+ * Points every `crombolo` logger at the site's files (and the console). LogTape
+ * holds one configuration per process, so the most recently started server's
+ * wins.
+ */
+export function configureLogging(options: LoggingOptions): void {
+  configureSync({
+    reset: true,
+    sinks: {
+      file: getClefFileSink(options.logsDir),
+      console: getConsoleSink(),
+    },
+    loggers: [
+      {
+        category: ['crombolo'],
+        lowestLevel: options.level,
+        sinks: options.console ? ['file', 'console'] : ['file'],
+      },
+      // The development 501 log prints its own line; its events only go to the file
+      {
+        category: ['crombolo', 'not-implemented'],
+        sinks: ['file'],
+        parentSinks: 'override',
+      },
+      // The request trace is thousands of lines per backoffice boot: the console
+      // only, so the log viewer's files stay readable
+      {
+        category: ['crombolo', 'http'],
+        sinks: options.console ? ['console'] : [],
+        parentSinks: 'override',
+      },
+      { category: ['logtape', 'meta'], lowestLevel: 'warning', sinks: ['console'] },
+    ],
+  })
+}
+
+/** A logger for one part of the server; its category becomes the event's `SourceContext`. */
+export function logger(...category: string[]): Logger {
+  return getLogger(['crombolo', ...category])
+}

package/src/marketplace.ts

@@ -0,0 +1,297 @@
+/**
+ * The Bundles section's marketplace, over npm. `docs/17-bundles.md`.
+ *
+ * Discovery and declaration are two different things, because npm indexes only
+ * one of them: a bundle is *found* by the `crombolo-bundle` keyword, which is
+ * the only thing registry search will match, and it *declares* itself with a
+ * `crombolo` field in its `package.json`. Search for the keyword, then read each
+ * hit's packument — custom fields survive there — and offer only the packages
+ * that actually declare extensions. A keyword squatter never reaches the list.
+ *
+ * The search is proxied rather than done in the browser: it works where the
+ * browser cannot reach npm, and the backoffice keeps making same-origin
+ * requests only.
+ */
+import { join } from 'node:path'
+import { EXTENSION_KEYWORD, type ExtensionRegistry } from '@crombolo/backoffice-host'
+import { logger } from './logging.ts'
+
+const log = logger('packages')
+
+/** How long a search is reused. Long enough to page a list, short enough to stay fresh. */
+const CACHE_MS = 5 * 60 * 1000
+
+export interface MarketplaceListing {
+  name: string
+  version: string
+  description: string
+  publisher: string | null
+  links: { npm: string; homepage: string | null; repository: string | null }
+  /** Whether the latest version declares backoffice extensions. */
+  declaresExtensions: boolean
+  installedVersion: string | null
+}
+
+/** An installed extension as the Installed view lists it. */
+export interface InstalledListing {
+  /** The npm package name, which is what an uninstall names. */
+  packageName: string
+  /** The display name the package chose. */
+  name: string
+  id: string
+  version: string
+  extensionCount: number
+}
+
+export interface MarketplaceOptions {
+  registry: string
+  keyword: string
+  marketplaceUrl: string
+  extensions: ExtensionRegistry
+  fetch?: typeof fetch
+  /** Where `bun add` runs, which is the site that owns the package.json. */
+  siteDir: string
+  /** Overridden by tests; the real one shells out to Bun. */
+  install?: PackageInstaller
+}
+
+export interface InstallOutcome {
+  ok: boolean
+  /**
+   * The dependency line the site's `package.json` now holds, so the UI can show
+   * exactly what there is to commit: the range as written, and the version that
+   * resolved to.
+   */
+  dependency?: { name: string; range: string; version: string }
+  message: string
+}
+
+export type PackageInstaller = (
+  action: 'add' | 'remove',
+  specifier: string,
+  siteDir: string,
+) => Promise<{ ok: boolean; output: string }>
+
+interface SearchObject {
+  package?: {
+    name?: string
+    version?: string
+    description?: string
+    publisher?: { username?: string }
+    links?: Record<string, string>
+  }
+}
+
+/** A name npm would accept, so nothing shell-ish or path-ish reaches `bun add`. */
+export function isValidPackageName(name: string): boolean {
+  return /^(?:@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*$/.test(name) && name.length <= 214
+}
+
+/**
+ * A version or range that cannot be mistaken for another argument.
+ *
+ * Ranges start with an operator — `^2.1`, `>=1 <2` — so those lead; `-` never
+ * does, because `bun add` would read it as a flag. No shell is involved
+ * (`Bun.spawn` takes an argv), so the space in a range is harmless.
+ */
+export function isValidVersion(version: string): boolean {
+  return /^[A-Za-z0-9^~>=<*][A-Za-z0-9.^~>=<| *+-]*$/.test(version) && version.length <= 64
+}
+
+export interface Marketplace {
+  /** Where someone browsing should be sent, when they want more than this list. */
+  url(): string
+  search(query: string | undefined): Promise<MarketplaceListing[]>
+  installed(): InstalledListing[]
+  install(name: string, version: string | undefined): Promise<InstallOutcome>
+  uninstall(name: string): Promise<InstallOutcome>
+}
+
+/**
+ * `bun add` in the site directory: Bun owns resolution and writes the lockfile.
+ *
+ * The spawn itself can fail rather than the command — an unreadable site
+ * directory or a `bun` that is not on the path both throw out of `Bun.spawn` —
+ * and that has to come back as a failed install with a reason, not as an
+ * exception the route turns into a 500.
+ */
+export const bunInstaller: PackageInstaller = async (action, specifier, siteDir) => {
+  try {
+    const proc = Bun.spawn(['bun', action, specifier], {
+      cwd: siteDir,
+      stdout: 'pipe',
+      stderr: 'pipe',
+    })
+    const [stdout, stderr, code] = await Promise.all([
+      new Response(proc.stdout).text(),
+      new Response(proc.stderr).text(),
+      proc.exited,
+    ])
+    return { ok: code === 0, output: `${stdout}${stderr}`.trim() }
+  } catch (error) {
+    return {
+      ok: false,
+      output: `bun ${action} could not be run in ${siteDir}: ${
+        error instanceof Error ? error.message : String(error)
+      }`,
+    }
+  }
+}
+
+export function createMarketplace(options: MarketplaceOptions): Marketplace {
+  const call = options.fetch ?? fetch
+  const install = options.install ?? bunInstaller
+
+  /** The range `package.json` holds for a dependency, as `bun add` wrote it. */
+  const rangeFor = async (name: string): Promise<string | undefined> => {
+    try {
+      const json = (await Bun.file(join(options.siteDir, 'package.json')).json()) as {
+        dependencies?: Record<string, string>
+      }
+      return json.dependencies?.[name]
+    } catch {
+      return undefined
+    }
+  }
+  let cached: { at: number; query: string; listings: MarketplaceListing[] } | undefined
+
+  const installedVersion = (name: string): string | null =>
+    options.extensions.list().find((extension) => extension.packageName === name)?.version ?? null
+
+  /** Whether the registry says this package's latest version declares extensions. */
+  const declares = async (name: string): Promise<boolean> => {
+    try {
+      const response = await call(`${options.registry}/${encodeURIComponent(name)}`, {
+        headers: { accept: 'application/json' },
+      })
+      if (!response.ok) return false
+      const packument = (await response.json()) as {
+        'dist-tags'?: { latest?: string }
+        versions?: Record<string, { crombolo?: { extensions?: unknown[] } }>
+      }
+      const latest = packument['dist-tags']?.latest
+      const version = latest ? packument.versions?.[latest] : undefined
+      return Array.isArray(version?.crombolo?.extensions) && version.crombolo.extensions.length > 0
+    } catch {
+      return false
+    }
+  }
+
+  return {
+    url: () => options.marketplaceUrl,
+
+    async search(query) {
+      const text = [`keywords:${options.keyword}`, query?.trim()].filter(Boolean).join(' ')
+      if (cached && cached.query === text && Date.now() - cached.at < CACHE_MS)
+        return cached.listings.map((listing) => ({
+          ...listing,
+          installedVersion: installedVersion(listing.name),
+        }))
+
+      let objects: SearchObject[] = []
+      try {
+        const response = await call(
+          `${options.registry}/-/v1/search?text=${encodeURIComponent(text)}&size=50`,
+          { headers: { accept: 'application/json' } },
+        )
+        if (!response.ok) throw new Error(`the registry answered ${response.status}`)
+        objects = ((await response.json()) as { objects?: SearchObject[] }).objects ?? []
+      } catch (error) {
+        // An unreachable registry is an empty list with a log line, not a 500:
+        // an air-gapped site should still be able to open the section.
+        log.warning('The package registry could not be reached: {detail}', {
+          detail: error instanceof Error ? error.message : String(error),
+        })
+        return []
+      }
+
+      // The packuments are read together: one request per hit, and a page of
+      // hits read one at a time would make the view feel broken.
+      const listings = await Promise.all(
+        objects
+          .map((object) => object.package)
+          .filter((pkg): pkg is NonNullable<typeof pkg> =>
+            Boolean(pkg?.name && isValidPackageName(pkg.name)),
+          )
+          .map(async (pkg): Promise<MarketplaceListing> => {
+            const name = pkg.name as string
+            return {
+              name,
+              version: pkg.version ?? '0.0.0',
+              description: pkg.description ?? '',
+              publisher: pkg.publisher?.username ?? null,
+              links: {
+                npm: pkg.links?.npm ?? `https://www.npmjs.com/package/${name}`,
+                homepage: pkg.links?.homepage ?? null,
+                repository: pkg.links?.repository ?? null,
+              },
+              declaresExtensions: await declares(name),
+              installedVersion: installedVersion(name),
+            }
+          }),
+      )
+      // Only what really declares extensions: the keyword is a claim, the field
+      // is the thing itself.
+      const offered = listings.filter((listing) => listing.declaresExtensions)
+      cached = { at: Date.now(), query: text, listings: offered }
+      return offered
+    },
+
+    installed() {
+      return options.extensions.list().map((extension) => ({
+        packageName: extension.packageName,
+        name: extension.manifest.name,
+        id: extension.manifest.id ?? extension.packageName,
+        version: extension.version,
+        extensionCount: extension.manifest.extensions.length,
+      }))
+    },
+
+    async install(name, version) {
+      if (!isValidPackageName(name))
+        return { ok: false, message: `'${name}' is not a package name npm would accept.` }
+      if (version !== undefined && !isValidVersion(version))
+        return { ok: false, message: `'${version}' is not a version npm would accept.` }
+
+      const specifier = version ? `${name}@${version}` : name
+      const result = await install('add', specifier, options.siteDir)
+      options.extensions.invalidate()
+      if (!result.ok)
+        return {
+          ok: false,
+          message: result.output || `Installing ${specifier} failed.`,
+        }
+
+      const range = (await rangeFor(name)) ?? version ?? 'latest'
+      const found = options.extensions.list().find((extension) => extension.packageName === name)
+      if (!found)
+        return {
+          ok: true,
+          dependency: { name, range, version: version ?? range },
+          message: `${name} was installed, but it declares no backoffice extensions, so nothing was added to the backoffice.`,
+        }
+      return {
+        ok: true,
+        dependency: { name, range, version: found.version },
+        // The install lives in this node's node_modules until the dependency is
+        // committed; saying so is the difference between a surprise and a plan.
+        message: `${name}@${found.version} is installed and active. Commit the changed package.json and bun.lock to keep it across deployments.`,
+      }
+    },
+
+    async uninstall(name) {
+      if (!isValidPackageName(name))
+        return { ok: false, message: `'${name}' is not a package name npm would accept.` }
+      const result = await install('remove', name, options.siteDir)
+      options.extensions.invalidate()
+      return result.ok
+        ? {
+            ok: true,
+            message: `${name} was removed. Commit the changed package.json and bun.lock.`,
+          }
+        : { ok: false, message: result.output || `Removing ${name} failed.` }
+    },
+  }
+}
+
+export { EXTENSION_KEYWORD }