@crombolo/server 0.9.1 → 0.11.0

package/src/jobs.ts

@@ -1,13 +1,21 @@
 /**
  * Background jobs, as Umbraco runs them: scheduled publishing every minute and
- * version cleanup every hour when it is enabled. Each run can also be called directly, which is
+ * version cleanup every hour when it is enabled, plus the search indexer
+ * draining its queue and the audit log ageing out old entries hourly. Each run can also be called directly, which is
  * how tests drive them. On a multi-node Postgres site every node runs them;
- * each schedule is claimed atomically, so it runs once.
+ * each schedule and queued item is claimed atomically, so it runs once.
  */
 
 import type { NodeSchemaState } from '@crombolo/data'
-import { type Db, DocumentRepository, PublishBlockedError } from '@crombolo/data'
+import {
+  AuditLog,
+  type Db,
+  DocumentRepository,
+  PublishBlockedError,
+  SearchIndex,
+} from '@crombolo/data'
 import type { PublishedCache } from '@crombolo/render'
+import { auditRetainedSince, recordAudit } from './adapters/audit.ts'
 import type { WorkflowRunner, WorkflowRunReport } from './form-workflows.ts'
 import { logger } from './logging.ts'
 import type { MediaFileStore } from './media-files.ts'
@@ -33,6 +41,10 @@ export interface JobsOptions {
    */
   formWorkflows?: WorkflowRunner
   formWorkflowsMs?: number
+  /** How often the search indexer looks for queued items; a write here also starts it at once. */
+  searchIndexingMs?: number
+  /** Days the audit log keeps an entry; 0 or absent keeps them forever. */
+  auditRetentionDays?: number
 }
 
 export interface ScheduledRun {
@@ -45,6 +57,12 @@ export interface BackgroundJobs {
   runScheduledPublishing(now?: Date): Promise<ScheduledRun>
   /** Runs the form workflows that are due; a no-op when none are configured. */
   runFormWorkflows(now?: Date): Promise<WorkflowRunReport>
+  /** Indexes every queued item, and returns how many. docs/search.md. */
+  runSearchIndexing(): Promise<number>
+  /** Something was queued here: index it shortly, rather than at the next tick. */
+  kickSearchIndexing(): void
+  /** Deletes audit entries older than the retention period, and returns how many. */
+  runAuditRetention(now?: Date): Promise<number>
   /** Runs whether or not the hourly job is enabled; the policy decides what goes. */
   runVersionCleanup(now?: Date): Promise<{ nodes: number; versionsDeleted: number }>
   start(): void
@@ -60,6 +78,9 @@ export function createBackgroundJobs(options: JobsOptions): BackgroundJobs {
   })
   const timers: Array<ReturnType<typeof setInterval>> = []
   let running = false
+  const search = new SearchIndex(options.db)
+  let indexing: Promise<number> | undefined
+  let kicked: ReturnType<typeof setTimeout> | undefined
 
   async function runScheduledPublishing(now = new Date()): Promise<ScheduledRun> {
     const run: ScheduledRun = { published: [], unpublished: [], failed: [] }
@@ -67,13 +88,20 @@ export function createBackgroundJobs(options: JobsOptions): BackgroundJobs {
       if (!(await documents.claimSchedule(due.id))) continue
       const cultures = due.culture ? [due.culture] : null
       try {
-        if (due.action === 'Release') {
-          await documents.publish(due.key, cultures)
-          run.published.push(due.key)
-        } else {
-          await documents.unpublish(due.key, cultures)
-          run.unpublished.push(due.key)
-        }
+        const release = due.action === 'Release'
+        const changed = release
+          ? await documents.publish(due.key, cultures)
+          : await documents.unpublish(due.key, cultures)
+        ;(release ? run.published : run.unpublished).push(due.key)
+        await recordAudit(options.db, {
+          category: 'content',
+          action: release ? 'Publish' : 'Unpublish',
+          source: 'job',
+          entityType: 'Document',
+          entityKey: due.key,
+          entityName: changed?.variants[0]?.name ?? null,
+          detail: `Scheduled${due.culture ? ` (${due.culture})` : ''}`,
+        })
         options.onDocumentChanged?.(due.key)
       } catch (error) {
         if (!(error instanceof PublishBlockedError)) throw error
@@ -84,6 +112,11 @@ export function createBackgroundJobs(options: JobsOptions): BackgroundJobs {
     return run
   }
 
+  async function runAuditRetention(now = new Date()): Promise<number> {
+    const before = auditRetainedSince(options.auditRetentionDays ?? 0, now)
+    return before ? new AuditLog(options.db).prune(before) : 0
+  }
+
   async function runVersionCleanup(now = new Date()) {
     return documents.cleanupVersions(now, options.versionCleanup)
   }
@@ -99,6 +132,20 @@ export function createBackgroundJobs(options: JobsOptions): BackgroundJobs {
     )
   }
 
+  // Its own guard rather than `guarded`: indexing a big rebuild must not hold up
+  // a scheduled publish, and a kick while it runs joins the run in progress.
+  function runSearchIndexing(): Promise<number> {
+    indexing ??= search.indexAll().finally(() => {
+      indexing = undefined
+    })
+    return indexing
+  }
+
+  const indexQuietly = () =>
+    runSearchIndexing().catch((error: unknown) => {
+      logger('jobs').error('Search indexing failed', { error })
+    })
+
   const guarded = (job: () => Promise<unknown>) => async () => {
     if (running) return
     running = true
@@ -120,18 +167,35 @@ export function createBackgroundJobs(options: JobsOptions): BackgroundJobs {
   return {
     runScheduledPublishing,
     runVersionCleanup,
+    runAuditRetention,
     runFormWorkflows,
+    runSearchIndexing,
+    kickSearchIndexing() {
+      if (timers.length === 0 || kicked) return
+      kicked = setTimeout(() => {
+        kicked = undefined
+        void indexQuietly()
+      }, 250)
+      kicked.unref()
+    },
     start() {
       every(options.scheduledPublishingMs ?? 60_000, runScheduledPublishing)
+      // The queue is shared, so this also picks up what other nodes wrote.
+      const searchTimer = setInterval(indexQuietly, options.searchIndexingMs ?? 5_000)
+      searchTimer.unref()
+      timers.push(searchTimer)
       // More often than the other jobs: somebody waiting on a form notification
       // notices a minute, and a queue row is cheap to look for.
       if (options.formWorkflows) every(options.formWorkflowsMs ?? 15_000, runFormWorkflows)
       if (options.versionCleanup.enabled) every(60 * 60_000, runVersionCleanup)
+      if (options.auditRetentionDays) every(60 * 60_000, runAuditRetention)
       const files = options.mediaFiles
       if (files) every(60 * 60_000, () => files.cleanupExpired())
     },
     stop() {
       for (const timer of timers.splice(0)) clearInterval(timer)
+      if (kicked) clearTimeout(kicked)
+      kicked = undefined
     },
     started() {
       return timers.length > 0

package/src/marketplace.ts

@@ -1,19 +1,30 @@
 /**
- * The Bundles section's marketplace, over npm. `docs/17-bundles.md`.
+ * The Bundles section's marketplace, over npm. `docs/bundles.md`.
  *
  * Discovery and declaration are two different things, because npm indexes only
  * one of them: a bundle is *found* by the `crombolo-bundle` keyword, which is
  * the only thing registry search will match, and it *declares* itself with a
  * `crombolo` field in its `package.json`. Search for the keyword, then read each
  * hit's packument — custom fields survive there — and offer only the packages
- * that actually declare extensions. A keyword squatter never reaches the list.
+ * whose manifest is a valid bundle. A keyword squatter never reaches the list.
+ *
+ * An install is `bun add`, and then — unless the site or the request says not
+ * to — the content the bundle carries, through the same installer as
+ * `crombolo bundle install`. A bundle's server half is never loaded by an
+ * install: that still takes an edit to `crombolo.config.ts` and a deploy.
  *
  * The search is proxied rather than done in the browser: it works where the
  * browser cannot reach npm, and the backoffice keeps making same-origin
  * requests only.
  */
 import { join } from 'node:path'
-import { EXTENSION_KEYWORD, type ExtensionRegistry } from '@crombolo/backoffice-host'
+import {
+  type BundlePackage,
+  EXTENSION_KEYWORD,
+  type ExtensionRegistry,
+  type PackageJson,
+  validateBundlePackage,
+} from '@crombolo/backoffice-host'
 import { logger } from './logging.ts'
 
 const log = logger('packages')
@@ -27,12 +38,12 @@ export interface MarketplaceListing {
   description: string
   publisher: string | null
   links: { npm: string; homepage: string | null; repository: string | null }
-  /** Whether the latest version declares backoffice extensions. */
-  declaresExtensions: boolean
+  /** What the latest version carries; only a valid bundle is listed at all. */
+  carries: BundlePackage['carries']
   installedVersion: string | null
 }
 
-/** An installed extension as the Installed view lists it. */
+/** A bundle the site depends on, as the Installed view lists it. */
 export interface InstalledListing {
   /** The npm package name, which is what an uninstall names. */
   packageName: string
@@ -41,6 +52,19 @@ export interface InstalledListing {
   id: string
   version: string
   extensionCount: number
+  carries: BundlePackage['carries']
+  /** Whether its server half is imported into `crombolo.config.ts`; only meaningful when it carries one. */
+  wired: boolean
+  /** The version whose content was applied here, or null when none has been. */
+  contentApplied: string | null
+}
+
+/** What became of the content half of an install. */
+export interface ContentOutcome {
+  applied: boolean
+  /** Set when it was applied: the transfer run that can revert it. */
+  runId?: string
+  message: string
 }
 
 export interface MarketplaceOptions {
@@ -53,6 +77,17 @@ export interface MarketplaceOptions {
   siteDir: string
   /** Overridden by tests; the real one shells out to Bun. */
   install?: PackageInstaller
+  /** The ids of the bundles whose server half this node has mounted. */
+  wired?: () => readonly string[]
+  /** The version of a package whose content has been applied here. */
+  contentApplied?: (packageName: string) => Promise<string | null>
+  /**
+   * Installs the content half of a bundle the site now depends on. Absent where
+   * content cannot be applied, which declines it for every install.
+   */
+  applyContent?: (packageName: string) => Promise<ContentOutcome>
+  /** The site's `applyBundleContentOnInstall`. */
+  applyContentOnInstall?: boolean
 }
 
 export interface InstallOutcome {
@@ -64,6 +99,8 @@ export interface InstallOutcome {
    */
   dependency?: { name: string; range: string; version: string }
   message: string
+  /** What happened to the content the bundle carries; absent when it carries none. */
+  content?: ContentOutcome
 }
 
 export type PackageInstaller = (
@@ -102,8 +139,12 @@ export interface Marketplace {
   /** Where someone browsing should be sent, when they want more than this list. */
   url(): string
   search(query: string | undefined): Promise<MarketplaceListing[]>
-  installed(): InstalledListing[]
-  install(name: string, version: string | undefined): Promise<InstallOutcome>
+  installed(): Promise<InstalledListing[]>
+  install(
+    name: string,
+    version: string | undefined,
+    options?: { applyContent?: boolean },
+  ): Promise<InstallOutcome>
   uninstall(name: string): Promise<InstallOutcome>
 }
 
@@ -156,24 +197,35 @@ export function createMarketplace(options: MarketplaceOptions): Marketplace {
   let cached: { at: number; query: string; listings: MarketplaceListing[] } | undefined
 
   const installedVersion = (name: string): string | null =>
-    options.extensions.list().find((extension) => extension.packageName === name)?.version ?? null
+    options.extensions.bundles().bundles.find((bundle) => bundle.packageName === name)?.version ??
+    null
 
-  /** Whether the registry says this package's latest version declares extensions. */
-  const declares = async (name: string): Promise<boolean> => {
+  /** What the registry says this package's latest version carries, when it is a valid bundle. */
+  const declares = async (name: string): Promise<BundlePackage['carries'] | undefined> => {
     try {
       const response = await call(`${options.registry}/${encodeURIComponent(name)}`, {
         headers: { accept: 'application/json' },
       })
-      if (!response.ok) return false
+      if (!response.ok) return undefined
       const packument = (await response.json()) as {
         'dist-tags'?: { latest?: string }
-        versions?: Record<string, { crombolo?: { extensions?: unknown[] } }>
+        versions?: Record<string, PackageJson>
       }
       const latest = packument['dist-tags']?.latest
       const version = latest ? packument.versions?.[latest] : undefined
-      return Array.isArray(version?.crombolo?.extensions) && version.crombolo.extensions.length > 0
+      if (!version || validateBundlePackage(version).length > 0) return undefined
+      const field = version.crombolo as {
+        extensions?: unknown[]
+        content?: string
+        server?: string
+      }
+      return {
+        extensions: Array.isArray(field.extensions) && field.extensions.length > 0,
+        content: field.content !== undefined,
+        server: field.server !== undefined,
+      }
     } catch {
-      return false
+      return undefined
     }
   }
 
@@ -213,8 +265,10 @@ export function createMarketplace(options: MarketplaceOptions): Marketplace {
           .filter((pkg): pkg is NonNullable<typeof pkg> =>
             Boolean(pkg?.name && isValidPackageName(pkg.name)),
           )
-          .map(async (pkg): Promise<MarketplaceListing> => {
+          .map(async (pkg): Promise<MarketplaceListing | undefined> => {
             const name = pkg.name as string
+            const carries = await declares(name)
+            if (!carries) return undefined
             return {
               name,
               version: pkg.version ?? '0.0.0',
@@ -225,29 +279,35 @@ export function createMarketplace(options: MarketplaceOptions): Marketplace {
                 homepage: pkg.links?.homepage ?? null,
                 repository: pkg.links?.repository ?? null,
               },
-              declaresExtensions: await declares(name),
+              carries,
               installedVersion: installedVersion(name),
             }
           }),
       )
-      // Only what really declares extensions: the keyword is a claim, the field
-      // is the thing itself.
-      const offered = listings.filter((listing) => listing.declaresExtensions)
+      // Only what really is a bundle: the keyword is a claim, the manifest is
+      // the thing itself.
+      const offered = listings.filter((listing): listing is MarketplaceListing => Boolean(listing))
       cached = { at: Date.now(), query: text, listings: offered }
       return offered
     },
 
-    installed() {
-      return options.extensions.list().map((extension) => ({
-        packageName: extension.packageName,
-        name: extension.manifest.name,
-        id: extension.manifest.id ?? extension.packageName,
-        version: extension.version,
-        extensionCount: extension.manifest.extensions.length,
-      }))
+    async installed() {
+      const wired = new Set(options.wired?.() ?? [])
+      return Promise.all(
+        options.extensions.bundles().bundles.map(async (bundle) => ({
+          packageName: bundle.packageName,
+          name: bundle.name,
+          id: bundle.id,
+          version: bundle.version,
+          extensionCount: bundle.field.extensions?.length ?? 0,
+          carries: bundle.carries,
+          wired: wired.has(bundle.id),
+          contentApplied: (await options.contentApplied?.(bundle.packageName)) ?? null,
+        })),
+      )
     },
 
-    async install(name, version) {
+    async install(name, version, request = {}) {
       if (!isValidPackageName(name))
         return { ok: false, message: `'${name}' is not a package name npm would accept.` }
       if (version !== undefined && !isValidVersion(version))
@@ -263,19 +323,44 @@ export function createMarketplace(options: MarketplaceOptions): Marketplace {
         }
 
       const range = (await rangeFor(name)) ?? version ?? 'latest'
-      const found = options.extensions.list().find((extension) => extension.packageName === name)
+      const found = options.extensions
+        .bundles()
+        .bundles.find((bundle) => bundle.packageName === name)
       if (!found)
         return {
           ok: true,
           dependency: { name, range, version: version ?? range },
-          message: `${name} was installed, but it declares no backoffice extensions, so nothing was added to the backoffice.`,
+          message: `${name} was installed, but it is not a valid bundle, so nothing was added to the backoffice.`,
         }
+
+      // The content half lands after the dependency, so a refused check leaves
+      // the package installed and says why, rather than undoing an install that
+      // succeeded.
+      let content: ContentOutcome | undefined
+      if (found.carries.content) {
+        const declined = request.applyContent === false || options.applyContentOnInstall === false
+        content =
+          declined || !options.applyContent
+            ? {
+                applied: false,
+                message: `Its content was not applied. Run \`crombolo bundle install ${name}\` to apply it.`,
+              }
+            : await options.applyContent(name).catch((error: unknown) => ({
+                applied: false,
+                message: `Its content could not be applied: ${error instanceof Error ? error.message : String(error)}`,
+              }))
+      }
       return {
         ok: true,
         dependency: { name, range, version: found.version },
         // The install lives in this node's node_modules until the dependency is
         // committed; saying so is the difference between a surprise and a plan.
-        message: `${name}@${found.version} is installed and active. Commit the changed package.json and bun.lock to keep it across deployments.`,
+        message: `${name}@${found.version} is installed. Commit the changed package.json and bun.lock to keep it across deployments.${
+          found.carries.server
+            ? ' Its server half runs only once crombolo.config.ts imports it and the site is deployed.'
+            : ''
+        }`,
+        ...(content ? { content } : {}),
       }
     },
 

package/src/no-nodes.ts

@@ -1,6 +1,6 @@
 /**
  * The holding page for a site with nothing published — Umbraco's "no nodes"
- * screen, which `docs/06-features.md` asks for.
+ * screen, which `docs/features.md` asks for.
  *
  * It answers 404, not 200: a brand-new site is not a working one, and anything
  * watching the URL should keep saying so until a page is published. It is only

package/src/ports.ts

@@ -35,6 +35,7 @@ import {
 } from '@crombolo/data'
 import type { PublishedCache } from '@crombolo/render'
 import { createNodeLookup } from './access.ts'
+import { createAuditHistory } from './adapters/audit.ts'
 import { createBlueprintPort } from './adapters/blueprints.ts'
 import { createPackagePort } from './adapters/bundles.ts'
 import {
@@ -58,6 +59,7 @@ import {
   type SchemaFiles,
   type SchemaFileWriter,
 } from './adapters/schema-files.ts'
+import { createSearchIndexPort } from './adapters/search-index.ts'
 import { createTemplateFileStore } from './adapters/template-files.ts'
 import { createTemplateQueryPort } from './adapters/template-query.ts'
 import { avatarUrls, createUserPorts } from './adapters/users.ts'
@@ -89,7 +91,7 @@ const CULTURES = [
 
 function createServerPort(config: CromboloConfig, canSendUserLinks: boolean): ServerPort {
   return {
-    // Phase 6 derives this from migration state; until then the site is always up.
+    // Story 080 derives this from migration state; until then the site is always up.
     status: () => ({ serverStatus: 'Run' }),
     configuration: () => ({
       // Reported false when no link could be delivered, so the client does not
@@ -374,6 +376,8 @@ export interface DepsOptions {
    * tells the other nodes to look; without it they keep the view they imported.
    */
   onViewsChange?: (alias: string) => void | Promise<void>
+  /** Items were queued for the search index; the indexer can start on them now. */
+  onSearchQueued?: () => void
 }
 
 function createTemporaryFilePort(files: MediaFileStore): TemporaryFilePort {
@@ -514,12 +518,14 @@ export function createDeps(options: DepsOptions): ManagementApiDeps {
       stylesheets: deps.stylesheets,
       scripts: deps.scripts,
     })
+    deps.auditHistory = createAuditHistory(options.db, options.config.auditRetentionDays)
     if (options.cache) {
       const cache = options.cache
       deps.templateQuery = createTemplateQueryPort({
         cache,
         types: new ContentTypeRepository(options.db),
       })
+      deps.searchIndex = createSearchIndexPort(options.db, options.onSearchQueued)
       deps.publishedCache = {
         reload: () => cache.invalidate(),
         rebuild: async () => {

package/src/schema.ts

@@ -1,7 +1,7 @@
 /**
  * Schema-as-code at runtime: the boot sync, the development watcher that
  * re-syncs on file changes, and the cache-instruction poll that keeps this
- * node's caches coherent with the others. docs/09-schema-as-code.md.
+ * node's caches coherent with the others. docs/schema-as-code.md.
  */
 
 import { existsSync, watch } from 'node:fs'