@crombolo/cli 0.15.0 → 0.16.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crombolo/cli",
-  "version": "0.15.0",
+  "version": "0.16.0",
   "description": "The crombolo command line: scaffolding a site, schema, views, domains, content transfer and upgrades.",
   "license": "MIT",
   "author": "Crombolo",
@@ -19,7 +19,8 @@
   "type": "module",
   "exports": {
     ".": "./src/index.ts",
-    "./bin": "./bin/crombolo.ts"
+    "./bin": "./bin/crombolo.ts",
+    "./plugin": "./src/plugin.ts"
   },
   "bin": {
     "crombolo": "./bin/crombolo.ts"
@@ -35,12 +36,13 @@
     "access": "public"
   },
   "dependencies": {
-    "@crombolo/auth": "0.15.0",
-    "@crombolo/core": "0.15.0",
-    "@crombolo/data": "0.15.0",
-    "@crombolo/schema": "0.15.0",
-    "@crombolo/server": "0.15.0",
-    "@crombolo/transfer": "0.15.0",
+    "@crombolo/api-management": "0.16.0",
+    "@crombolo/auth": "0.16.0",
+    "@crombolo/core": "0.16.0",
+    "@crombolo/data": "0.16.0",
+    "@crombolo/schema": "0.16.0",
+    "@crombolo/server": "0.16.0",
+    "@crombolo/transfer": "0.16.0",
     "@oclif/core": "^5.1.8"
   },
   "oclif": {
@@ -79,7 +81,7 @@
         "description": "pause and resume editors' saves, cluster-wide"
       },
       "media": {
-        "description": "a pulled site's media files"
+        "description": "the media library in bulk: import a folder, export one; a pulled site's files"
       },
       "remote": {
         "description": "the remotes this site pulls from and pushes to (remotes.toml)"
@@ -89,7 +91,25 @@
       },
       "upgrade": {
         "description": "framework migrations: check, plan, apply, ledger"
+      },
+      "users": {
+        "description": "the back-office users: unlock, enable, reset a password or a second factor, invite"
+      },
+      "api-client": {
+        "description": "the API client a site seeds from its environment"
+      },
+      "index": {
+        "description": "the search indexes: what they hold, and rebuilding them"
+      },
+      "cache": {
+        "description": "the published cache on every node"
+      },
+      "recycle-bin": {
+        "description": "what has been deleted, before it is gone for good"
       }
-    }
+    },
+    "optionalPlugins": [
+      "@crombolo/cloud"
+    ]
   }
 }

package/src/base.ts

@@ -5,11 +5,9 @@
  * result is the output, and in text mode the command says it as it goes.
  */
 import { Command, Errors, Flags } from '@oclif/core'
-import { classify, reportFailure } from './classify.ts'
 import { CliError, type ErrorKind, exitCodeOf, OUTCOMES, type OutcomeName } from './errors.ts'
-import * as ops from './operations/commands.ts'
+import type * as ops from './operations/commands.ts'
 import type { Io } from './support/site.ts'
-import { siteConfig } from './support/site.ts'
 
 /** A field of one output record, in clispec's closed set of types. */
 export interface OutputField {
@@ -107,6 +105,9 @@ export abstract class BaseCommand extends Command implements Io {
 
   protected override async catch(error: Error): Promise<never> {
     if (error instanceof Errors.ExitError) throw error
+    // Loaded when something fails, so a command — or a plugin's — that succeeds
+    // never pays for the site packages it knows how to name.
+    const { classify, reportFailure } = await import('./classify.ts')
     const failure = classify(error)
     reportFailure(failure, this.jsonEnabled())
     throw new Errors.ExitError(exitCodeOf(failure.kind))
@@ -115,7 +116,11 @@ export abstract class BaseCommand extends Command implements Io {
   protected override async finally(error: Error | undefined): Promise<void> {
     const audit = (this.constructor as typeof BaseCommand).audit
     if (error || !audit || this.changedNothing) return
-    await ops.recordCommand(await siteConfig(), {
+    const [{ recordCommand }, { siteConfig }] = await Promise.all([
+      import('./operations/commands.ts'),
+      import('./support/site.ts'),
+    ])
+    await recordCommand(await siteConfig(), {
       ...audit,
       command: `crombolo ${(this.id ?? '').replaceAll(':', ' ')}`,
       entityKey: this.auditKey,

package/src/commands/api-client/rotate.ts

@@ -0,0 +1,57 @@
+import { randomBytes } from 'node:crypto'
+import { hashPassword } from '@crombolo/auth'
+import { UserRepository } from '@crombolo/data'
+import { bootstrapDatabase } from '@crombolo/server'
+import { Flags } from '@oclif/core'
+import { BaseCommand } from '../../base.ts'
+import { siteConfig } from '../../support/site.ts'
+
+/** A new secret for the API client a site seeds from its environment (story 034). */
+export default class ApiClientRotate extends BaseCommand {
+  static override summary = 'issue a new secret for the API client, and print it once'
+  static override description =
+    'The old secret stops working at once. Put the new one in CROMBOLO_API_CLIENT_SECRET: a start keeps the client it finds, so it never sets the secret back.'
+  static override flags = {
+    client: Flags.string({
+      summary: 'the client id; CROMBOLO_API_CLIENT_ID when not given',
+      helpValue: '<id>',
+    }),
+  }
+  static override audit = {
+    category: 'user',
+    action: 'RotateClientSecret',
+    entityType: 'User',
+  } as const
+  static override spec = {
+    effects: 'non_idempotent',
+    cardinality: 'single',
+    outputFields: [
+      { name: 'client', type: 'string' },
+      { name: 'secret', type: 'string' },
+    ],
+    errors: ['usage', 'not_found'],
+  } as const
+
+  async run() {
+    const { flags } = await this.parse(ApiClientRotate)
+    const config = await siteConfig()
+    const client = flags.client ?? config.apiClient?.id
+    if (!client)
+      this.fail('usage', 'Which API client?', 'Pass --client <id>, or set CROMBOLO_API_CLIENT_ID.')
+    const { db } = await bootstrapDatabase(config)
+    try {
+      const users = new UserRepository(db)
+      const owner = await users.clientCredentialOwner(client)
+      if (!owner) this.fail('not_found', `There is no API client "${client}".`)
+      this.auditKey = client
+      const secret = randomBytes(32).toString('base64url')
+      await users.removeClientCredential(owner.userId, client)
+      await users.addClientCredential(owner.userId, client, await hashPassword(secret))
+      this.say(`  rotated  ${client}; put this in CROMBOLO_API_CLIENT_SECRET:`)
+      this.say(secret)
+      return { client, secret }
+    } finally {
+      await db.close()
+    }
+  }
+}

package/src/commands/cache/rebuild.ts

@@ -0,0 +1,56 @@
+import { appendCacheInstruction, latestCacheInstructionId, liveNodes } from '@crombolo/data'
+import { bootstrapDatabase } from '@crombolo/server'
+import { BaseCommand } from '../../base.ts'
+import { siteConfig } from '../../support/site.ts'
+
+/**
+ * Each node keeps its own published cache, so this asks all of them, the way an
+ * editor's save does: a cache instruction every node acts on at its next poll.
+ */
+export default class CacheRebuild extends BaseCommand {
+  static override summary = 'rebuild the published cache on every node of the site'
+  static override description =
+    'Each node rebuilds at its next poll, within a few seconds. Nothing has to be reachable from here but the database.'
+  static override audit = {
+    category: 'system',
+    action: 'RebuildCache',
+    entityType: 'Cache',
+  } as const
+  static override spec = {
+    effects: 'idempotent',
+    cardinality: 'single',
+    outputFields: [
+      { name: 'instruction', type: 'integer' },
+      {
+        name: 'nodes',
+        type: 'array',
+        items: { type: 'string' },
+        description: 'the nodes seen in the last minute',
+      },
+    ],
+  } as const
+
+  async run() {
+    await this.parse(CacheRebuild)
+    const config = await siteConfig()
+    const { db } = await bootstrapDatabase(config)
+    try {
+      // Stamped as the command line, never as a node: a node skips its own.
+      await appendCacheInstruction(db, {
+        kind: 'content',
+        payload: { all: true, reason: 'crombolo cache rebuild' },
+        by: `cli:${config.nodeId}`,
+      })
+      const instruction = await latestCacheInstructionId(db)
+      const nodes = (await liveNodes(db, { seenWithinMs: 60_000 })).map((node) => node.nodeId)
+      this.say(
+        nodes.length > 0
+          ? `  asked    ${nodes.length} node(s) to rebuild: ${nodes.join(', ')}`
+          : '  asked    every node to rebuild; none has been seen in the last minute',
+      )
+      return { instruction, nodes }
+    } finally {
+      await db.close()
+    }
+  }
+}

package/src/commands/health.ts

@@ -0,0 +1,90 @@
+import {
+  bootstrapDatabase,
+  createDiagnosticsPort,
+  mediaStoreFor,
+  resolveEmailPort,
+} from '@crombolo/server'
+import { Flags } from '@oclif/core'
+import { BaseCommand } from '../base.ts'
+import { siteConfig } from '../support/site.ts'
+
+/** The checks the Settings section's Health Check dashboard runs, for a monitor or a deploy to gate on. */
+export default class Health extends BaseCommand {
+  static override summary = "run the site's health checks"
+  static override description =
+    'The same checks as Settings › Health Check. Exits 20 when a check fails; a warning is reported, not counted.'
+  static override flags = {
+    group: Flags.string({ summary: 'only this group of checks', helpValue: '<name>' }),
+  }
+  static override spec = {
+    effects: 'read_only',
+    cardinality: 'bounded',
+    outputFields: [
+      {
+        name: 'groups',
+        type: 'array',
+        items: {
+          type: 'object',
+          fields: [
+            { name: 'name', type: 'string' },
+            { name: 'checks', type: 'array', items: { type: 'object' } },
+          ],
+        },
+      },
+      { name: 'failed', type: 'integer' },
+      { name: 'warnings', type: 'integer' },
+    ],
+    errors: ['not_found'],
+    outcomes: ['problems_found'],
+  } as const
+
+  async run() {
+    const { flags } = await this.parse(Health)
+    const config = await siteConfig()
+    const { db } = await bootstrapDatabase(config)
+    try {
+      const diagnostics = createDiagnosticsPort({
+        config,
+        db,
+        media: await mediaStoreFor(config),
+        email: resolveEmailPort(config),
+      })
+      const groups = diagnostics.groups()
+      const chosen = flags.group
+        ? groups.filter((group) => group.name.toLowerCase() === flags.group?.toLowerCase())
+        : groups
+      if (chosen.length === 0)
+        this.fail(
+          'not_found',
+          `There is no group "${flags.group}".`,
+          `The groups: ${groups.map((g) => g.name).join(', ')}.`,
+        )
+      let failed = 0
+      let warnings = 0
+      const report = []
+      for (const group of chosen) {
+        const results = (await diagnostics.run(group.name)) ?? []
+        const checks = group.checks.map((check) => {
+          const found = results.find((result) => result.id === check.id)?.results ?? []
+          for (const result of found) {
+            if (result.resultType === 'Error') failed += 1
+            if (result.resultType === 'Warning') warnings += 1
+          }
+          return {
+            name: check.name,
+            results: found.map((r) => ({ type: r.resultType, message: r.message })),
+          }
+        })
+        this.say(group.name)
+        for (const check of checks)
+          for (const result of check.results)
+            this.say(`  [${result.type.padEnd(7)}] ${check.name}: ${result.message}`)
+        report.push({ name: group.name, checks })
+      }
+      if (failed > 0) this.outcome = 'problems_found'
+      return { groups: report, failed, warnings }
+    } finally {
+      await db.close()
+    }
+  }
+}

package/src/commands/index/list.ts

@@ -0,0 +1,45 @@
+import { SEARCH_INDEXES, SearchIndex } from '@crombolo/data'
+import { bootstrapDatabase } from '@crombolo/server'
+import { BaseCommand } from '../../base.ts'
+import { siteConfig } from '../../support/site.ts'
+
+export default class IndexList extends BaseCommand {
+  static override summary = 'the search indexes: what each holds, and what is waiting to be indexed'
+  static override spec = {
+    effects: 'read_only',
+    cardinality: 'bounded',
+    outputFields: [
+      { name: 'name', type: 'string' },
+      { name: 'documents', type: 'integer' },
+      { name: 'fields', type: 'integer' },
+      { name: 'pending', type: 'integer' },
+      { name: 'healthy', type: 'boolean', description: 'nothing waiting to be indexed' },
+    ],
+  } as const
+
+  async run() {
+    await this.parse(IndexList)
+    const { db } = await bootstrapDatabase(await siteConfig())
+    try {
+      const search = new SearchIndex(db)
+      const items = []
+      for (const index of SEARCH_INDEXES) {
+        const stats = await search.stats(index)
+        items.push({
+          name: index.name,
+          documents: stats.documentCount,
+          fields: stats.fieldCount,
+          pending: stats.pending,
+          healthy: stats.pending === 0,
+        })
+      }
+      for (const index of items)
+        this.say(
+          `  ${index.name.padEnd(14)} ${index.documents} item(s), ${index.fields} field(s)${index.pending > 0 ? `, ${index.pending} waiting` : ''}`,
+        )
+      return { items }
+    } finally {
+      await db.close()
+    }
+  }
+}

package/src/commands/index/rebuild.ts

@@ -0,0 +1,66 @@
+import { SEARCH_INDEXES, SearchIndex } from '@crombolo/data'
+import { bootstrapDatabase } from '@crombolo/server'
+import { Args, Flags } from '@oclif/core'
+import { BaseCommand } from '../../base.ts'
+import { siteConfig } from '../../support/site.ts'
+
+/**
+ * The index lives in the database, so rebuilding it here rebuilds it for every
+ * node: there is no copy per server to reach. Search keeps answering from the
+ * old entries while each item is indexed again.
+ */
+export default class IndexRebuild extends BaseCommand {
+  static override summary = 'index every item again, for every node: the index is in the database'
+  static override description =
+    'Search keeps answering while it runs. --queue-only queues everything and leaves the indexing to the running site.'
+  static override args = {
+    index: Args.string({
+      description: 'one index; both when not given',
+      options: SEARCH_INDEXES.map((index) => index.name),
+    }),
+  }
+  static override flags = {
+    'queue-only': Flags.boolean({ summary: 'queue everything, and let the site index it' }),
+  }
+  static override audit = {
+    category: 'system',
+    action: 'RebuildIndex',
+    entityType: 'Index',
+  } as const
+  static override spec = {
+    effects: 'idempotent',
+    cardinality: 'single',
+    outputFields: [
+      { name: 'indexes', type: 'array', items: { type: 'string' } },
+      { name: 'indexed', type: 'integer', description: 'items indexed here; 0 with --queue-only' },
+      { name: 'queued', type: 'boolean' },
+    ],
+  } as const
+
+  async run() {
+    const { args, flags } = await this.parse(IndexRebuild)
+    const chosen = SEARCH_INDEXES.filter((index) => !args.index || index.name === args.index)
+    this.auditKey = args.index
+    const { db } = await bootstrapDatabase(await siteConfig())
+    try {
+      const search = new SearchIndex(db)
+      for (const index of chosen) await search.rebuild(index)
+      const names = chosen.map((index) => index.name)
+      if (flags['queue-only']) {
+        this.say(`  queued   ${names.join(', ')}: the running site indexes it`)
+        return { indexes: names, indexed: 0, queued: true }
+      }
+      let indexed = 0
+      for (;;) {
+        const batch = await search.indexQueued()
+        if (batch === 0) break
+        indexed += batch
+        this.progress(`  indexed  ${indexed}`)
+      }
+      this.say(`  rebuilt  ${names.join(', ')}: ${indexed} item(s) indexed`)
+      return { indexes: names, indexed, queued: false }
+    } finally {
+      await db.close()
+    }
+  }
+}

package/src/commands/media/export.ts

@@ -0,0 +1,91 @@
+import { mkdirSync, writeFileSync } from 'node:fs'
+import { extname, join, resolve } from 'node:path'
+import type { NodeRow } from '@crombolo/data'
+import { bootstrapDatabase, mediaFileOf, safeFileName } from '@crombolo/server'
+import { Args, Flags } from '@oclif/core'
+import { BaseCommand } from '../../base.ts'
+import { here } from '../../support/describe.ts'
+import { childrenByName, mediaParent, openMedia } from '../../support/media-tree.ts'
+import { siteConfig } from '../../support/site.ts'
+
+/** The media library, or a folder of it, back out as files in folders. */
+export default class MediaExport extends BaseCommand {
+  static override summary = 'write media files back out as a folder tree'
+  static override examples = ['<%= config.bin %> media export /Gallery --out ./gallery']
+  static override args = {
+    folder: Args.string({
+      description: 'a media folder, as a path or uuid; everything when not given',
+    }),
+  }
+  static override flags = {
+    out: Flags.string({ summary: 'the folder to write into', helpValue: '<dir>', required: true }),
+  }
+  static override spec = {
+    effects: 'idempotent',
+    cardinality: 'bounded',
+    outputFields: [
+      { name: 'out', type: 'string' },
+      { name: 'files', type: 'integer' },
+      { name: 'bytes', type: 'integer' },
+      {
+        name: 'missing',
+        type: 'array',
+        items: { type: 'string' },
+        description: 'items whose file is not in the store',
+      },
+    ],
+    errors: ['not_found'],
+  } as const
+
+  async run() {
+    const { args, flags } = await this.parse(MediaExport)
+    const config = await siteConfig()
+    const { db } = await bootstrapDatabase(config)
+    try {
+      const tree = await openMedia(db, config)
+      const start = await mediaParent(db, args.folder)
+      const out = resolve(flags.out)
+      let files = 0
+      let bytes = 0
+      const missing: string[] = []
+
+      const write = async (node: NodeRow, dir: string) => {
+        const media = await tree.repo.byKey(node.key)
+        const src = media ? mediaFileOf(media) : null
+        const name = node.text ?? node.key
+        if (!src) {
+          // A folder, or an item with nothing uploaded: its children follow.
+          const children = await childrenByName(tree, node)
+          if (children.size === 0 && media) return
+          const into = join(dir, safeFileName(name))
+          for (const child of children.values()) await write(child, into)
+          return
+        }
+        const stored = await tree.files.open(src)
+        if (!stored) {
+          missing.push(name)
+          return
+        }
+        const extension = extname(src)
+        const fileName = safeFileName(
+          extname(name).toLowerCase() === extension.toLowerCase() ? name : `${name}${extension}`,
+        )
+        const content = await stored.bytes()
+        mkdirSync(dir, { recursive: true })
+        writeFileSync(join(dir, fileName), content)
+        files += 1
+        bytes += content.length
+      }
+
+      if (start) await write(start, out)
+      else for (const child of (await childrenByName(tree, null)).values()) await write(child, out)
+      mkdirSync(out, { recursive: true })
+      this.say(`  wrote    ${files} file(s) to ${here(out)}`)
+      for (const name of missing)
+        this.warning(`  missing  ${name}: its file is not in the media store`)
+      return { out: here(out), files, bytes, missing }
+    } finally {
+      await db.close()
+    }
+  }
+}