@mercury-fw/cli 0.52.0 → 0.53.1

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @mercury-fw/cli
 
+## 0.53.1
+
+### Patch Changes
+
+- @mercury-fw/core@0.53.1
+- @mercury-fw/utils@0.53.1
+
+## 0.53.0
+
+### Minor Changes
+
+- 79a232e: - New package `@mercury-fw/web-search-searxng`: a SearXNG instance as the web search plugin's backend, through its JSON format.
+  - `mfw create --plugins web-search` adds the web search plugin with the SearXNG backend, a `searxng` service in the compose file (on the compose network only), its `searxng/settings.yml` with the JSON format on, and `SEARXNG_URL` and `SEARXNG_SECRET` in the env example.
+
+### Patch Changes
+
+- Updated dependencies [56e4ed5]
+- Updated dependencies [c83b129]
+  - @mercury-fw/core@0.53.0
+  - @mercury-fw/utils@0.53.0
+
 ## 0.52.0
 
 ### Minor Changes

package/README.md

@@ -61,7 +61,7 @@ Then it runs `bun install` in the app and creates a git repository on `main` wit
 | `--assistant-name <name>` | The assistant's name (default `Mercury`). |
 | `--role <text>` | Completes "You are <name>, …" (default `an internal assistant`). |
 | `--channels <ids>` | Comma-separated: `google-chat`, `http`. |
-| `--plugins <ids>` | Comma-separated: `jira`, `bitbucket`, `atlassian-admin`, `zitadel`. |
+| `--plugins <ids>` | Comma-separated: `jira`, `bitbucket`, `atlassian-admin`, `zitadel`, `web-search` (with a SearXNG service next to the app: set `SEARXNG_SECRET` in the env file). |
 | `--auth <id>` | The HTTP channel's auth provider, `oidc` or `static`: required with `http`, refused without it, since the channel doesn't start without one. The wizard asks for it once `http` is chosen. |
 | `--directory <id>` | The user directory, `static` or `zitadel`: who the people are and their roles. Without one, the person is whoever the channel says, with no roles. |
 | `--local-packages <folder>` | Installs the packages packed in `<folder>` (`bun pm pack`) instead of the registry's, as [`mfw local-packages`](#mfw-local-packages-folder--mfw-local-packages---off) does on an existing app: their versions come from the tarballs, so a package that isn't published yet works too, and no newer `mfw` is looked up on the registry. For trying unreleased packages in a new app; the test bed makes its apps this way. |

package/dist/src/catalog.d.ts

@@ -26,6 +26,24 @@ export type CatalogEntry = {
     /** Dependencies whose install scripts Bun must run for this entry (Bun runs
      * none it isn't told to trust), on top of a tool plugin's own package. */
     trusts?: string[];
+    /** For a plugin built by calling its export: what the config writes in
+     * `plugins` (indented as an element of it) and the other imports that needs,
+     * whose packages the app depends on too. */
+    build?: {
+        expression: string;
+        imports: Array<{
+            names: string;
+            package: string;
+        }>;
+    };
+    /** A compose service the entry needs next to the app (its lines under
+     * `services:`), which the app then depends on. */
+    service?: {
+        name: string;
+        lines: string[];
+    };
+    /** Files the entry adds to the app, by path. */
+    files?: Record<string, string>;
 };
 /** Starting formatter rules for a plugin that hands the user lists: without a
  * rule a kind of list isn't shown, so the scaffolded config wraps the plugin

package/dist/src/render.d.ts

@@ -1,3 +1,4 @@
+import { type CatalogEntry } from "./catalog.ts";
 /** What the new app is made of. `versions` maps each package the app depends
  * on to the version its range is written against. */
 export type RenderInput = {
@@ -28,3 +29,5 @@ export declare function selectionError(channels: string[], plugins: string[], au
  * Undefined when they go together. Shared with the command, which checks it
  * before writing when nothing will ask (`--yes`). */
 export declare function pairingError(channels: string[], auth?: string): string | undefined;
+/** The packages an entry brings into the app: its own, and those its config imports. */
+export declare function entryPackages(e: CatalogEntry): string[];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mercury-fw/cli",
-  "version": "0.52.0",
+  "version": "0.53.1",
   "type": "module",
   "license": "MIT",
   "repository": {
@@ -41,22 +41,24 @@
   "comment:shape": "The Mercury CLI (`mfw`). `mfw create <dir>` writes a new Mercury app from the template. The framework packages move in lockstep with it, so a new app gets them at the CLI's own version; plugins and channels at the registry's latest. The plugin and channel packages are devDependencies only for the catalog test. `main(argv)` is exported for create-mercury-agent.",
   "dependencies": {
     "@clack/prompts": "^1.8.1",
-    "@mercury-fw/core": "0.52.0",
-    "@mercury-fw/utils": "0.52.0",
+    "@mercury-fw/core": "0.53.1",
+    "@mercury-fw/utils": "0.53.1",
     "commander": "^15.0.0"
   },
   "devDependencies": {
     "@mercury-fw/auth-oidc": "0.2.0",
     "@mercury-fw/auth-static": "0.2.1",
-    "@mercury-fw/channel-google-chat": "0.8.0",
+    "@mercury-fw/channel-google-chat": "0.8.2",
     "@mercury-fw/channel-http": "0.10.1",
     "@mercury-fw/directory-static": "0.1.1",
-    "@mercury-fw/formatter": "0.52.0",
+    "@mercury-fw/formatter": "0.53.1",
     "@mercury-fw/plugin-atlassian-admin": "0.3.2",
     "@mercury-fw/plugin-bitbucket": "0.4.1",
     "@mercury-fw/plugin-jira": "0.6.5",
+    "@mercury-fw/plugin-web-search": "0.1.2",
     "@mercury-fw/plugin-zitadel": "0.3.2",
     "@mercury-fw/typescript-config": "*",
+    "@mercury-fw/web-search-searxng": "0.1.0",
     "@types/bun": "^1.4.2",
     "typescript": "^6.0.3"
   }

package/src/catalog.ts

@@ -24,8 +24,33 @@ export type CatalogEntry = {
   /** Dependencies whose install scripts Bun must run for this entry (Bun runs
    * none it isn't told to trust), on top of a tool plugin's own package. */
   trusts?: string[];
+  /** For a plugin built by calling its export: what the config writes in
+   * `plugins` (indented as an element of it) and the other imports that needs,
+   * whose packages the app depends on too. */
+  build?: { expression: string; imports: Array<{ names: string; package: string }> };
+  /** A compose service the entry needs next to the app (its lines under
+   * `services:`), which the app then depends on. */
+  service?: { name: string; lines: string[] };
+  /** Files the entry adds to the app, by path. */
+  files?: Record<string, string>;
 };
 
+/** SearXNG's settings for the web search plugin, mounted into its container. */
+const SEARXNG_SETTINGS = `# SearXNG for the web search plugin. Only the app reaches it, on the compose
+# network: nothing of it is published on the host. The secret comes from
+# SEARXNG_SECRET in the app's env file.
+use_default_settings: true
+server:
+  # Only Mercury asks: no rate limiting, no image proxy.
+  limiter: false
+  image_proxy: false
+search:
+  # The plugin reads the results as JSON, which is off by default.
+  formats:
+    - html
+    - json
+`;
+
 /** Starting formatter rules for a plugin that hands the user lists: without a
  * rule a kind of list isn't shown, so the scaffolded config wraps the plugin
  * with these. `displaysType` is the type the plugin exports for its kinds,
@@ -124,6 +149,41 @@ export const CATALOG: CatalogEntry[] = [
     exportName: "zitadelPlugin",
     env: [],
   },
+  {
+    id: "web-search",
+    kind: "tool",
+    package: "@mercury-fw/plugin-web-search",
+    exportName: "webSearchPlugin",
+    env: [
+      { name: "SEARXNG_URL", comment: "SearXNG, the compose service the web search asks", value: "http://searxng:8080" },
+      { name: "SEARXNG_SECRET", comment: "Required: SearXNG's secret key, any long random string (openssl rand -hex 32)" },
+    ],
+    build: {
+      expression: [
+        "    webSearchPlugin({",
+        '      backend: searxngBackend({ url: process.env.SEARXNG_URL || "http://searxng:8080" }),',
+        "      // Hosts whose pages anyone may read without being asked; each covers its subdomains.",
+        "      allowedHosts: [],",
+        "    })",
+      ].join("\n"),
+      imports: [{ names: "searxngBackend", package: "@mercury-fw/web-search-searxng" }],
+    },
+    service: {
+      name: "searxng",
+      lines: [
+        "  searxng:",
+        "    # SearXNG publishes no major version tags, only latest and dated ones.",
+        "    image: searxng/searxng:latest",
+        "    restart: unless-stopped",
+        "    environment:",
+        "      # SearXNG exits on its default key: without one, compose says so instead.",
+        "      SEARXNG_SECRET: ${SEARXNG_SECRET:?set SEARXNG_SECRET in .env (see .env.example)}",
+        "    volumes:",
+        "      - ./searxng/settings.yml:/etc/searxng/settings.yml:ro",
+      ],
+    },
+    files: { "searxng/settings.yml": SEARXNG_SETTINGS },
+  },
   {
     id: "static",
     kind: "directory",

package/src/main.ts

@@ -11,7 +11,7 @@ import type { CreateArgs } from "./args.ts";
 import { CATALOG } from "./catalog.ts";
 import { kebabCase } from "./naming.ts";
 import { runProgram } from "./program.ts";
-import { pairingError, renderApp, selectionError } from "./render.ts";
+import { entryPackages, pairingError, renderApp, selectionError } from "./render.ts";
 import { appVersions, cliVersion, newerCli, registryFrom } from "./versions.ts";
 import { appCommands, terminalDeps, type AppDeps } from "./app/commands.ts";
 import { copyPacks, readPacks, withLocalOverrides } from "./app/local-packages.ts";
@@ -147,7 +147,7 @@ async function create(args: CreateArgs, rawArgs: string[], relaunch: Relaunch, g
           : e.kind === "directory"
             ? e.id === answers.directory
             : (e.kind === "channel" ? answers.channels : answers.plugins).includes(e.id),
-      ).map((e) => e.package),
+      ).flatMap(entryPackages),
     ),
   ];
   const versions = await appVersions(chosen, {

package/src/render.ts

@@ -76,13 +76,16 @@ export function renderApp(input: RenderInput): Map<string, string> {
   const directory = selected("directory", input.directory === undefined ? [] : [input.directory])[0];
   const assistantName = input.assistantName.trim();
 
+  const extraFiles = [...channels, ...tools].flatMap((e) => Object.entries(e.files ?? {}));
+  const services = tools.flatMap((t) => (t.service ? [t.service] : []));
   return new Map([
+    ...extraFiles,
     [".dockerignore", dockerignore],
     [".env.example", renderEnv(channels, auth, tools, directory)],
     [".gitignore", gitignore],
     ["Dockerfile", dockerfile],
     ["README.md", renderReadme(input.name, channels, auth, tools, directory)],
-    ["docker-compose.yml", renderCompose(input.name, channels.some((c) => c.id === "http"))],
+    ["docker-compose.yml", renderCompose(input.name, channels.some((c) => c.id === "http"), services)],
     ["markdown.d.ts", markdownDts],
     ["mercury.config.ts", renderConfig(assistantName, channels, auth, tools, directory)],
     ["package.json", renderPackageJson(input.name, channels, auth, tools, input.versions, directory)],
@@ -184,6 +187,7 @@ function renderConfig(
   for (const t of tools) {
     const names = t.formatter ? `${t.exportName}, type ${t.formatter.displaysType}` : t.exportName;
     imports.push(`import { ${names} } from "${t.package}";`);
+    for (const extra of t.build?.imports ?? []) imports.push(`import { ${extra.names} } from "${extra.package}";`);
   }
   for (const c of [...channels, ...(auth ? [auth] : []), ...(directory ? [directory] : [])]) {
     imports.push(`import { ${c.exportName} } from "${c.package}";`);
@@ -216,6 +220,7 @@ function renderConfig(
 
 /** One element of the config's `plugins` array. */
 function renderPluginEntry(t: CatalogEntry): string {
+  if (t.build) return `${t.build.expression},`;
   if (!t.formatter) {
     return `    ${t.exportName},`;
   }
@@ -229,6 +234,11 @@ function renderPluginEntry(t: CatalogEntry): string {
   ].join("\n");
 }
 
+/** The packages an entry brings into the app: its own, and those its config imports. */
+export function entryPackages(e: CatalogEntry): string[] {
+  return [e.package, ...(e.build?.imports ?? []).map((i) => i.package)];
+}
+
 /** `package.json`: the core, the formatter when a plugin is wrapped in it, the
  * chosen packages, and every tool plugin trusted to run its postinstall (it
  * downloads the plugin's CLI), plus what a chosen entry says it needs trusted. */
@@ -241,7 +251,7 @@ function renderPackageJson(
   directory: CatalogEntry | undefined,
 ): string {
   const chosen = [...channels, ...(auth ? [auth] : []), ...tools, ...(directory ? [directory] : [])];
-  const packages = [...new Set(["@mercury-fw/core", ...chosen.map((e) => e.package)])];
+  const packages = [...new Set(["@mercury-fw/core", ...chosen.flatMap(entryPackages)])];
   if (tools.some((t) => t.formatter)) {
     packages.push("@mercury-fw/formatter");
   }
@@ -297,8 +307,8 @@ function renderEnv(
 /** `docker-compose.yml`: the app and Qdrant, with named volumes prefixed by the
  * app's name, the CLI credentials one included whatever was chosen (a plugin
  * with a CLI can come later); the HTTP surface's port published on the host
- * only with the HTTP channel. */
-function renderCompose(name: string, hasHttp: boolean): string {
+ * only with the HTTP channel; and the services chosen plugins need. */
+function renderCompose(name: string, hasHttp: boolean, services: Array<{ name: string; lines: string[] }>): string {
   const httpPort = hasHttp
     ? [
         "    # The HTTP surface, on the host: every route but its OpenAPI document needs a token.",
@@ -325,12 +335,14 @@ function renderCompose(name: string, hasHttp: boolean): string {
     '      - "host.docker.internal:host-gateway"',
     "    depends_on:",
     "      - qdrant",
+    ...services.map((s) => `      - ${s.name}`),
     "",
     "  qdrant:",
     "    image: qdrant/qdrant:v1",
     "    volumes:",
     "      - qdrant-data:/qdrant/storage",
     "",
+    ...services.flatMap((s) => [...s.lines, ""]),
     "volumes:",
     "  wiki-vault:",
     `    name: ${name}_wiki-vault`,
@@ -351,6 +363,13 @@ function renderReadme(
   directory: CatalogEntry | undefined,
 ): string {
   const list = (entries: CatalogEntry[]) => (entries.length > 0 ? entries.map((e) => e.id).join(", ") : "none");
+  // What the app can't start without, named where the reader is about to start it.
+  const required = [...channels, ...(auth ? [auth] : []), ...tools, ...(directory ? [directory] : [])]
+    .flatMap((e) => e.env)
+    .filter((v) => v.comment.startsWith("Required:"))
+    .map((v) => `\`${v.name}\``);
+  const requiredNote =
+    required.length > 0 ? `\nBefore \`mfw start\`, fill in the variables the chosen pieces require: ${required.join(", ")}.\n` : "";
   return `# ${name}
 
 A Mercury app, scaffolded by \`mfw create\`.
@@ -381,7 +400,7 @@ cp .env.example .env
 mfw start
 mfw repl
 \`\`\`
-
+${requiredNote}
 \`bun install\` here gives your editor, \`bun run typecheck\` and the app's own \`mfw\` (the one a global \`mfw\` runs inside the app) the packages (tool plugins download their CLI binary as they install); the image installs its own copy when it builds. \`mfw start\` builds the image and starts the app with Qdrant in the background, \`mfw repl\` opens a terminal conversation with the assistant. \`mfw --help\` lists the rest: stopping and restarting, logs, a shell in the container, the wiki and the memory, and resetting them.
 ${renderHttpSection(channels, auth)}${CREDENTIALS_SECTION}`;
 }