@every-app/sdk 0.3.0-rc.9 → 0.3.0

package/dist/internal/appSettings.js

@@ -8,17 +8,20 @@
  * (`providers`) and a capability (`capabilities`) are the Gateway's: their
  * keys and clients live on the Gateway Worker, set once for every app
  * with `everyapp secret set <NAME>`. The app's own secrets (`secrets`)
- * live on its Worker, set from its directory. Each declaration carries a
- * need: required (the default; the app doesn't open until it is set),
- * optional (the app opens, and the feature says what is missing where it
- * is used), or a member of a named any-of group (the app opens once any
- * member is set; only for an app that picks among the set members at run
- * time). Each may say `why` it is needed and where it comes from
- * (`docs`, an https URL); for providers and capabilities the registries
- * here supply a default. An app's plain settings (`settings`: a host, a
- * project id, a region) are held like its secrets, on its Worker, but are
- * values rather than keys: each says what it takes (a URL, a number, one
- * of a list) and may give a default.
+ * live on its Worker, set from its directory. Everything the manifest
+ * lists at its top level is required: the app doesn't open until it is
+ * set. The rest is grouped into features (`features`): each a named set of
+ * secrets, settings and the Gateway capabilities and providers it needs,
+ * with numbered setup steps. A required feature's values are required; an
+ * optional feature is all or nothing, never blocking: off while none of
+ * its own values is set, incomplete while some are, ready once all are.
+ * Each declaration may say `why` it is needed, where it comes from
+ * (`docs`, an https URL) and, for an app's own value, `how` to get it;
+ * for providers and capabilities the registries here supply a default. An
+ * app's plain settings (`settings`: a host, a project id, a region) are
+ * held like its secrets, on its Worker, but are values rather than keys:
+ * each says what it takes (a URL, a number, one of a list) and may give a
+ * default.
  *
  * Everything in this module is pure and tenant-neutral: a manifest ships
  * in a release, and what is set is asked of the caller (`isSet`), which
@@ -28,18 +31,8 @@
 import { CATALOG, capabilitySettings, } from "./capabilities.js";
 import { aiDefaultSetupHint, aiRequiredSettings, resolveAiDefault, } from "./aiDefault.js";
 import { isInstallOwnedGatewaySetting, isSettableGatewaySetting, } from "./gatewaySettings.js";
-import { remoteMcpAuthLabel, remoteMcpOf, remoteMcpSecretOf, } from "./remoteMcp.js";
-/** The need an entry declares; required unless it says otherwise. */
-export function needOf(entry) {
-    if (!entry)
-        return { kind: "required" };
-    if (typeof entry.anyOf === "string" && entry.anyOf.trim()) {
-        return { kind: "anyOf", group: entry.anyOf.trim() };
-    }
-    if (entry.optional === true)
-        return { kind: "optional" };
-    return { kind: "required" };
-}
+import { oauthSetupGuide } from "./oauthPresets.js";
+import { hasRemoteMcpBlock, remoteMcpBlockProblem } from "./remoteMcp.js";
 function needFields(entry) {
     if (!entry || typeof entry !== "object")
         return {};
@@ -47,11 +40,66 @@ function needFields(entry) {
     return {
         ...(typeof record["why"] === "string" ? { why: record["why"] } : {}),
         ...(typeof record["docs"] === "string" ? { docs: record["docs"] } : {}),
-        ...(typeof record["optional"] === "boolean"
-            ? { optional: record["optional"] }
-            : {}),
-        ...(typeof record["setup"] === "string" ? { setup: record["setup"] } : {}),
-        ...(typeof record["anyOf"] === "string" ? { anyOf: record["anyOf"] } : {}),
+    };
+}
+/** `how`, `generate` and `file` as a stored entry has them. */
+function ownValueFields(entry) {
+    if (!entry || typeof entry !== "object")
+        return {};
+    const record = entry;
+    return {
+        ...(typeof record["how"] === "string" ? { how: record["how"] } : {}),
+        ...(record["generate"] === true ? { generate: true } : {}),
+        ...(record["file"] === true ? { file: true } : {}),
+    };
+}
+const stringList = (value) => Array.isArray(value)
+    ? value.filter((each) => typeof each === "string")
+    : [];
+/** The features the manifest declares, in order, each read whatever a stored row holds. */
+export function featureEntries(manifest) {
+    const features = manifest.features;
+    if (!features || typeof features !== "object")
+        return [];
+    return Object.entries(features).flatMap(([id, raw]) => {
+        if (!raw || typeof raw !== "object")
+            return [];
+        const record = raw;
+        return [
+            {
+                id,
+                name: typeof record["name"] === "string" && record["name"].trim()
+                    ? record["name"]
+                    : id,
+                required: record["required"] === true,
+                ...(typeof record["why"] === "string" ? { why: record["why"] } : {}),
+                steps: stringList(record["steps"]),
+                capabilities: stringList(record["capabilities"]),
+                providers: stringList(record["providers"]),
+            },
+        ];
+    });
+}
+/** One feature's raw `secrets` or `settings` list, with the feature id. */
+function featureLists(manifest, key) {
+    const features = manifest.features;
+    if (!features || typeof features !== "object")
+        return [];
+    return Object.entries(features).flatMap(([feature, raw]) => {
+        const list = raw && typeof raw === "object"
+            ? raw[key]
+            : undefined;
+        return Array.isArray(list) ? [{ feature, list }] : [];
+    });
+}
+/** The need of a capability or provider: optional when an optional feature lists it, else required. */
+function gatewayNeedOf(features, kind, id) {
+    const owner = features.find((feature) => feature[kind].includes(id));
+    if (!owner)
+        return { need: { kind: "required" } };
+    return {
+        need: { kind: owner.required ? "required" : "optional" },
+        feature: owner.id,
     };
 }
 /** The providers the manifest lists, each as an entry whatever form it took. */
@@ -69,50 +117,68 @@ export function providerEntries(manifest) {
 export function providerIds(manifest) {
     return providerEntries(manifest).map((entry) => entry.id);
 }
-/** The app's own secrets the manifest lists, each as an entry whatever form it took. */
-export function secretEntries(manifest) {
-    return (manifest.secrets ?? []).flatMap((entry) => {
-        if (typeof entry === "string")
-            return [{ name: entry }];
-        if (entry && typeof entry === "object" && typeof entry.name === "string") {
-            return [{ name: entry.name, ...needFields(entry) }];
+function secretEntryOf(entry, feature) {
+    const tag = feature === undefined ? {} : { feature };
+    if (typeof entry === "string")
+        return [{ name: entry, ...tag }];
+    if (entry && typeof entry === "object") {
+        const name = entry.name;
+        if (typeof name === "string") {
+            return [{ name, ...needFields(entry), ...ownValueFields(entry), ...tag }];
         }
-        return [];
-    });
+    }
+    return [];
+}
+/**
+ * The app's own secrets the manifest lists, each as an entry whatever form
+ * it took: the top-level ones, then each feature's (tagged with it).
+ */
+export function secretEntries(manifest) {
+    return [
+        ...(manifest.secrets ?? []).flatMap((entry) => secretEntryOf(entry)),
+        ...featureLists(manifest, "secrets").flatMap(({ feature, list }) => list.flatMap((entry) => secretEntryOf(entry, feature))),
+    ];
 }
 /** The names of the app's own secrets, in order. */
 export function secretNames(manifest) {
     return secretEntries(manifest).map((entry) => entry.name);
 }
-/** The app's plain settings the manifest lists, each as an entry. */
+function settingEntryOf(entry, feature) {
+    if (!entry || typeof entry !== "object")
+        return [];
+    const record = entry;
+    if (typeof record["name"] !== "string")
+        return [];
+    const type = record["type"];
+    const values = record["values"];
+    return [
+        {
+            name: record["name"],
+            ...needFields(entry),
+            ...(typeof record["how"] === "string" ? { how: record["how"] } : {}),
+            ...(type === "url" ||
+                type === "number" ||
+                type === "boolean" ||
+                type === "text"
+                ? { type }
+                : {}),
+            ...(Array.isArray(values) &&
+                values.every((value) => typeof value === "string")
+                ? { values: values }
+                : {}),
+            ...(typeof record["default"] === "string"
+                ? { default: record["default"] }
+                : {}),
+            ...(feature === undefined ? {} : { feature }),
+        },
+    ];
+}
+/** The app's plain settings the manifest lists, each as an entry: the top-level ones, then each feature's. */
 export function settingEntries(manifest) {
-    return (manifest.settings ?? []).flatMap((entry) => {
-        if (!entry || typeof entry !== "object" || typeof entry.name !== "string") {
-            return [];
-        }
-        const record = entry;
-        const type = record["type"];
-        const values = record["values"];
-        return [
-            {
-                name: entry.name,
-                ...needFields(entry),
-                ...(type === "url" ||
-                    type === "number" ||
-                    type === "boolean" ||
-                    type === "text"
-                    ? { type }
-                    : {}),
-                ...(Array.isArray(values) &&
-                    values.every((value) => typeof value === "string")
-                    ? { values: values }
-                    : {}),
-                ...(typeof record["default"] === "string"
-                    ? { default: record["default"] }
-                    : {}),
-            },
-        ];
-    });
+    return [
+        ...(manifest.settings ?? []).flatMap((entry) => settingEntryOf(entry)),
+        ...featureLists(manifest, "settings").flatMap(({ feature, list }) => list.flatMap((entry) => settingEntryOf(entry, feature))),
+    ];
 }
 /** The names of the app's plain settings, in order. */
 export function settingNames(manifest) {
@@ -256,23 +322,62 @@ function providerHint(capability, names) {
     ];
     return lines.length > 0 ? { hint: lines.join(" ") } : {};
 }
-function withNeed(setting, entry, defaults = {}) {
+function withNeed(setting, entry, need, defaults = {}) {
     const docs = entry?.docs ?? defaults.docs;
-    // The registry's guide for a Gateway setting; the manifest's for the app's own secret.
-    const setup = setting.holder === "gateway" ? defaults.setup : entry?.setup;
     return {
         ...setting,
-        need: needOf(entry),
+        need: need.need,
+        ...(need.feature === undefined ? {} : { feature: need.feature }),
         ...(entry?.why ? { why: entry.why } : {}),
         ...(docs ? { docs } : {}),
-        ...(setup ? { setup } : {}),
+        ...(defaults.setup ? { setup: defaults.setup } : {}),
         ...(defaults.hint ? { hint: defaults.hint } : {}),
     };
 }
+/**
+ * A stored manifest from before features (deployed before 0.3.0-rc.14) marks a
+ * value `optional: true`. The strict schema refuses that key now, but a stored
+ * row keeps it until the app is redeployed, so it still reads as optional:
+ * updating the Gateway never puts a working app into Needs setup.
+ */
+function storedOptional(need, raw) {
+    if (need.feature !== undefined || need.need.kind === "optional")
+        return need;
+    const optional = raw !== null &&
+        typeof raw === "object" &&
+        raw["optional"] === true;
+    return optional ? { need: { kind: "optional" } } : need;
+}
+/** The raw entry named `name` in a stored list (`secrets`, `settings`, `providers`). */
+function rawEntry(list, name, key) {
+    if (!Array.isArray(list))
+        return undefined;
+    return list.find((entry) => entry !== null &&
+        typeof entry === "object" &&
+        entry[key] === name);
+}
+/** An app's own value's need: its feature's (required, or optional), or required at the top level. */
+function ownNeedOf(features, feature) {
+    if (feature === undefined)
+        return { need: { kind: "required" } };
+    const owner = features.find((each) => each.id === feature);
+    return {
+        need: { kind: owner?.required ? "required" : "optional" },
+        feature,
+    };
+}
+/** `how`, `generate` and `file` onto an app's own setting, when its entry says them. */
+function withOwnValue(setting, entry) {
+    return {
+        ...setting,
+        ...(entry.how ? { how: entry.how } : {}),
+        ...(entry.generate ? { generate: true } : {}),
+        ...(entry.file ? { file: true } : {}),
+    };
+}
 /**
  * Every setting the manifest declares, in manifest order: its AI
- * providers, its capabilities, the AI proxy's extra routes when `ai.routes`
- * asks for any, then the app's own secrets. A provider or capability the
+ * providers, its capabilities, then the app's own secrets. A provider or capability the
  * gateway doesn't offer contributes nothing (the manifest validator refuses
  * it elsewhere); so does a capability with nothing to set (`members`),
  * which is always set. `lookup` is where ids are found: the catalog alone
@@ -280,6 +385,7 @@ function withNeed(setting, entry, defaults = {}) {
  */
 export function appSettings(manifest, lookup = CATALOG, options = {}) {
     const settings = [];
+    const features = featureEntries(manifest);
     const chat = manifest.ai?.chat;
     if (chat === true || (chat && typeof chat === "object")) {
         // The default resolves on what is set: with nothing set, the first of
@@ -287,7 +393,7 @@ export function appSettings(manifest, lookup = CATALOG, options = {}) {
         // ask for; with one configured, that provider's. `isSet` here only
         // orders the candidates; evaluateAppSettings decides what is set.
         const resolution = resolveAiDefault({
-            provider: options.aiDefault ?? null,
+            choice: null,
             isSet: options.isSet ?? (() => false),
             lookup,
         });
@@ -303,24 +409,25 @@ export function appSettings(manifest, lookup = CATALOG, options = {}) {
             alternatives: resolution.candidates
                 .map((candidate) => settable(candidate.settings))
                 .filter((names) => names.length > 0),
-        }, chat === true ? undefined : chat, {
+        }, chat === true ? undefined : needFields(chat), storedOptional({ need: { kind: "required" } }, chat), {
             ...(provider?.mode === "ai" ? { docs: provider.docs } : {}),
             setup: "setup/ai.md",
             hint: aiDefaultSetupHint(resolution),
         }));
     }
-    // The default's settings: an optional pin that needs only those is the
-    // same row (one key, set once), so it isn't listed twice. A required
-    // pin stays: it needs that provider's key even while another provider
-    // serves the default.
+    // The default's settings: an optional pin (a provider an optional
+    // feature lists) that needs only those is the same row (one key, set
+    // once), so it isn't listed twice. A required pin stays: it needs that
+    // provider's key even while another provider serves the default.
     const defaultNames = settings.find((setting) => setting.kind === "ai-default")?.names;
     for (const entry of providerEntries(manifest)) {
         const capability = lookup.get(entry.id);
         if (capability?.mode !== "ai")
             continue;
         const names = providerNames(capability);
+        const need = storedOptional(gatewayNeedOf(features, "providers", entry.id), rawEntry(manifest.providers, entry.id, "id"));
         if (defaultNames &&
-            needOf(entry).kind === "optional" &&
+            need.need.kind === "optional" &&
             names.length > 0 &&
             names.every((name) => defaultNames.includes(name))) {
             continue;
@@ -332,7 +439,7 @@ export function appSettings(manifest, lookup = CATALOG, options = {}) {
             holder: "gateway",
             names,
             alsoReads: [],
-        }, entry, {
+        }, entry, need, {
             docs: capability.docs,
             setup: capability.setup,
             ...providerHint(capability, names),
@@ -351,67 +458,58 @@ export function appSettings(manifest, lookup = CATALOG, options = {}) {
             holder: "gateway",
             names,
             alsoReads,
-        }, needFields(entry), {
+        }, needFields(entry), storedOptional(gatewayNeedOf(features, "capabilities", id), entry), {
             docs: capabilityDocs(capability),
-            setup: capability.setup,
+            setup: capability.mode === "oauth_client"
+                ? oauthSetupGuide(id)
+                : capability.setup,
             ...providerHint(capability, names),
         }));
     }
-    if (manifest.ai?.routes?.length) {
+    const remoteProblem = hasRemoteMcpBlock(manifest)
+        ? remoteMcpBlockProblem(manifest)
+        : null;
+    if (remoteProblem) {
+        const label = typeof manifest.name === "string" && manifest.name.trim()
+            ? manifest.name.trim()
+            : "the MCP server";
         settings.push({
-            kind: "ai-routes",
-            id: "ai-routes",
-            label: "AI proxy routes",
+            kind: "remote-mcp",
+            id: "remote",
+            label: `${label} registration`,
             holder: "gateway",
-            names: ["AI_PROXY_EXTRA_ROUTES"],
+            names: [],
             alsoReads: [],
             need: { kind: "required" },
-            why: `Lets the app call ${manifest.ai.routes.join(", ")} through the AI proxy.`,
-            setup: "setup/openai.md",
+            why: `${label}'s registration can't be read: ${remoteProblem}.`,
+            setup: "setup/remote-mcp.md",
         });
     }
-    const remote = remoteMcpOf(manifest);
-    if (remote) {
-        const secret = remoteMcpSecretOf(remote.auth);
-        if (secret) {
-            const label = typeof manifest.name === "string" && manifest.name.trim()
-                ? manifest.name.trim()
-                : "the MCP server";
-            settings.push({
-                kind: "remote-mcp",
-                id: secret,
-                label: `${label} credential`,
-                holder: "gateway",
-                names: [secret],
-                alsoReads: [],
-                need: { kind: "required" },
-                why: `What ${label} expects: ${remoteMcpAuthLabel(remote.auth)}. The Gateway sends it to ${remote.url} and nowhere else.`,
-                setup: "setup/remote-mcp.md",
-            });
-        }
-    }
     for (const entry of secretEntries(manifest)) {
-        settings.push(withNeed({
+        settings.push(withOwnValue(withNeed({
             kind: "secret",
             id: entry.name,
             label: entry.name,
             holder: "app",
             names: [entry.name],
             alsoReads: [],
-        }, entry));
+        }, entry, storedOptional(ownNeedOf(features, entry.feature), rawEntry(manifest.secrets, entry.name, "name"))), entry));
     }
     for (const entry of settingEntries(manifest)) {
-        // A setting with a default is never unset for long: the deploy writes it.
-        const need = entry.default !== undefined ? { ...entry, optional: true } : entry;
+        const own = storedOptional(ownNeedOf(features, entry.feature), rawEntry(manifest.settings, entry.name, "name"));
         settings.push({
-            ...withNeed({
+            ...withOwnValue(withNeed({
                 kind: "setting",
                 id: entry.name,
                 label: entry.name,
                 holder: "app",
                 names: [entry.name],
                 alsoReads: [],
-            }, need),
+            }, entry, 
+            // A setting with a default is never unset for long: the deploy writes it.
+            entry.default !== undefined
+                ? { ...own, need: { kind: "optional" } }
+                : own), entry),
             value: settingValueRule(entry),
         });
     }
@@ -419,6 +517,10 @@ export function appSettings(manifest, lookup = CATALOG, options = {}) {
 }
 /** Whether each name of `setting` is set: all → true, one unset → false, else unknown. */
 function settingIsSet(setting, isSet) {
+    // An unreadable remote registration has nothing to set: it is fixed by
+    // adding the app again, never by a value.
+    if (setting.kind === "remote-mcp")
+        return false;
     const groupIsSet = (names) => {
         let unknown = false;
         for (const name of names) {
@@ -444,12 +546,24 @@ function settingIsSet(setting, isSet) {
     }
     return unknown ? null : false;
 }
+/**
+ * A feature's state from its settings (see {@link FeatureState}). Whether
+ * it is started is read from its own values that have no default (a
+ * default is written by the deploy, so it says nothing), or, for a
+ * feature of Gateway settings alone, from those.
+ */
+function featureState(settings) {
+    if (settings.every((setting) => setting.set !== false))
+        return "ready";
+    const own = settings.filter((setting) => setting.holder === "app" && setting.value?.default === undefined);
+    const started = (own.length > 0 ? own : settings).some((setting) => setting.set === true);
+    return started ? "incomplete" : "off";
+}
 /**
  * Every setting against what is set. `isSet` answers per holder and name:
  * true, false, or null when it can't tell (the registry never recorded
- * the app Worker's secret names), which never blocks. An any-of group is
- * met once any member is set; while none is and none is unknown, every
- * member blocks.
+ * the app Worker's secret names), which never blocks. A required setting
+ * that is unset blocks; an optional one (an optional feature's) never does.
  */
 export function evaluateAppSettings(settings, isSet, options = {}) {
     const evaluated = settings.map((setting) => {
@@ -460,51 +574,38 @@ export function evaluateAppSettings(settings, isSet, options = {}) {
             blocking: false,
         };
     });
-    const groups = [];
-    for (const setting of evaluated) {
-        if (setting.need.kind !== "anyOf")
-            continue;
-        const group = setting.need.group;
-        let found = groups.find((each) => each.group === group);
-        if (!found) {
-            found = { group, members: [], met: false };
-            groups.push(found);
-        }
-        found.members.push(setting);
-    }
-    for (const group of groups) {
-        group.met = group.members.some((member) => member.set === true);
-        const unknown = group.members.some((member) => member.set === null);
-        if (!group.met && !unknown) {
-            for (const member of group.members)
-                member.blocking = true;
-        }
-    }
     for (const setting of evaluated) {
         if (setting.need.kind === "required" && setting.set === false) {
             setting.blocking = true;
         }
     }
     const unmet = evaluated.filter((setting) => setting.blocking);
-    return { ready: unmet.length === 0, settings: evaluated, groups, unmet };
+    const features = (options.features ?? []).map((feature) => {
+        const own = evaluated.filter((setting) => setting.feature === feature.id);
+        return { ...feature, settings: own, state: featureState(own) };
+    });
+    return { ready: unmet.length === 0, settings: evaluated, unmet, features };
 }
 /**
  * What to run for an app that needs setup: `everyapp configure <app>`,
  * which reads the app's manifest (from the registry, so it runs from
- * anywhere), prompts for each unmet setting (one member of an any-of group
- * is enough: it asks which) and sets them where they are held. Null when
+ * anywhere), prompts for each unmet setting, feature by feature, and sets
+ * them where they are held. Null when
  * the app is ready. `cli` is how the operator runs everyapp (`pnpm
  * everyapp` from a checkout). Scripts that set one name at a time use
  * `everyapp secret set NAME [--app <id>]` instead.
  *
  * A provider app (`provider`: the gateway's own, with no Worker and no
  * directory; `everyapp mcp on <id>`, an MCP app) has nothing to configure: its
- * command is the `secret set` of its Gateway key (the first member of an
- * unmet any-of group), as `mcp list` says it.
+ * command is the `secret set` of its Gateway key, as `mcp list` says it.
  */
 export function setupCommand(report, { cli, appId, provider = false, }) {
     if (report.unmet.length === 0)
         return null;
+    if (report.unmet.some((setting) => setting.kind === "remote-mcp")) {
+        // Nothing to set: the registration predates provider credentials.
+        return { command: `${cli} mcp remove ${appId}` };
+    }
     if (!provider)
         return { command: `${cli} configure ${appId}` };
     const names = [];
@@ -513,15 +614,8 @@ export function setupCommand(report, { cli, appId, provider = false, }) {
             if (!names.includes(name))
                 names.push(name);
     };
-    for (const setting of report.unmet) {
-        if (setting.need.kind === "required")
-            add(setting);
-    }
-    for (const group of report.groups) {
-        const first = group.members.find((member) => member.blocking);
-        if (first)
-            add(first);
-    }
+    for (const setting of report.unmet)
+        add(setting);
     return { command: `${cli} secret set ${names.join(" ")}` };
 }
 export function appSetupOf(report, options) {
@@ -530,10 +624,9 @@ export function appSetupOf(report, options) {
         ready: report.ready,
         settings: report.settings.map(strip),
         unmet: report.unmet.map(strip),
-        groups: report.groups.map((group) => ({
-            group: group.group,
-            met: group.met,
-            members: group.members.map((member) => member.id),
+        features: report.features.map((feature) => ({
+            ...feature,
+            settings: feature.settings.map((setting) => setting.id),
         })),
         command: options.withCommand === false ? null : setupCommand(report, options),
     };
@@ -547,3 +640,18 @@ export function apiKeyCapabilityOf(id, lookup = CATALOG) {
     const capability = lookup.get(id);
     return capability?.mode === "api_key" ? capability : undefined;
 }
+/**
+ * A fresh value for a secret the manifest marks `generate: true`: 32
+ * bytes from the platform's cryptographic random source, base64url
+ * without padding (43 characters).
+ */
+export function generatedSecret() {
+    const bytes = crypto.getRandomValues(new Uint8Array(32));
+    let binary = "";
+    for (const byte of bytes)
+        binary += String.fromCharCode(byte);
+    return btoa(binary)
+        .replace(/\+/g, "-")
+        .replace(/\//g, "_")
+        .replace(/=+$/, "");
+}