@shardflux/cli 0.20.0 → 0.23.0

package/dist/cli-proxy-hub.js

@@ -11,10 +11,10 @@
  * Hub info (`cli-proxy.json`, 0600, in the config directory, in the hub and in the machine template so every VM has
  * it): { version, api_url, workspace_id, callback_url, callback_created_at, management_key }. Never printed or logged.
  */
-import { randomBytes } from 'node:crypto';
+import { createHash, randomBytes } from 'node:crypto';
 import { readFileSync } from 'node:fs';
 import { join } from 'node:path';
-import { Document, isMap, parseDocument } from 'yaml';
+import { Document, isMap, isScalar, isSeq, parseDocument } from 'yaml';
 import { ShardfluxApiError } from '@shardflux/sdk';
 import { writePrivateFile } from "./credentials.js";
 import { CodedError, UsageError } from "./errors.js";
@@ -42,21 +42,14 @@ export const HUB_TEMPLATE = 'codex-connector';
 /**
  * The codex-connector version a hub runs at least: the first with the hub's boot service and an sf-agent that serves
  * the VM port (cli-proxy-filter). New hubs are created on it; an older hub is upgraded in place (its home, and so its
- * logins and settings, stay) before VMs get the hub's address.
+ * logins and settings, stay) before VMs get the hub's address. A hub on a newer version keeps it (never downgraded).
  */
 export const HUB_TEMPLATE_VERSION = 4;
-/**
- * The codex-connector version whose sf-agent runs the account pool router: a hub set up for pooled accounts is created
- * on it, or upgraded to it. HUB_TEMPLATE_VERSION moves to it once it is registered in every environment.
- */
-export const POOL_HUB_TEMPLATE_VERSION = 5;
 export const HUB_CAPS = { cpu_millis: 1000, memory_mib: 256, disk_gib: 2, allocation_mode: 'fixed' };
 export const HUB_INFO_FILE = 'cli-proxy.json';
 /** Paths in the hub workspace. */
 export const HUB_DIR = '/home/user/.cli-proxy-hub';
 export const HUB_AUTH_DIR = '/home/user/.cli-proxy-api';
-/** Pool mode: the router reads the API URL and the hub token (`sfh_…`) from these hub files (0600). */
-export const HUB_POOL_FILES = { apiUrl: `${HUB_DIR}/api-url`, apiToken: `${HUB_DIR}/api-token` };
 const CALLBACK_PATH = /^\/__shardflux\/callback\/sfcb_[A-Za-z0-9_-]+\/$/;
 function callbackUrlOk(cb) {
     try {
@@ -315,7 +308,7 @@ export function sdkHubCloud(cloud, opts = {}) {
                     }
                 },
                 async authFiles() {
-                    const r = await hub.sh(`[ -d ${HUB_AUTH_DIR} ] || exit 0; cd ${HUB_AUTH_DIR} && find . -type f -name '*.json' -printf '%P\\n'`);
+                    const r = await hub.sh(AUTH_FILES_SCRIPT);
                     if (r.code !== 0)
                         throw new HubError(`Could not list the hub’s logins: ${r.stderr.trim() || `exit ${r.code}`}`);
                     return r.stdout.split('\n').filter(Boolean);
@@ -380,6 +373,11 @@ export function sdkHubCloud(cloud, opts = {}) {
         probe: (url) => probeCallback(url),
     };
 }
+/** The hub's auth files, oldest first (modification time, then path in byte order), one relative path per line. */
+export const AUTH_FILES_SCRIPT = `set -euo pipefail
+[ -d ${HUB_AUTH_DIR} ] || exit 0; cd ${HUB_AUTH_DIR}
+find . -type f -name '*.json' -printf '%T@ %P\\n' | LC_ALL=C sort -t ' ' -k1,1n -k2 | cut -d ' ' -f 2-
+`;
 /** Installs the pinned release once: download, verify its SHA-256, then move the binary into place. */
 export const INSTALL_SCRIPT = `set -euo pipefail
 v=${CLI_PROXY_VERSION}; dir=${HUB_DIR}; bin="$dir/bin/cli-proxy-api-$v"
@@ -600,12 +598,12 @@ export async function ensureCliProxyHub(cloud, opts) {
     const progress = opts.progress ?? (() => { });
     const local = readHubInfo(opts.dir, opts.apiUrl);
     progress('Opening the CLIProxyAPI hub…');
-    // Pooled accounts need the router's template; everything else keeps the general one.
-    const want = opts.pool ? POOL_HUB_TEMPLATE_VERSION : HUB_TEMPLATE_VERSION;
+    const want = HUB_TEMPLATE_VERSION;
     let opened = await hubCloud.open(!opts.repairOnly, want);
     if (!opened)
         throw new HubError('This account has no hosted CLIProxyAPI yet.', 'move yours: shard cli-proxy import');
-    // A hub on an older template is upgraded before VMs get its address (a plain repair only starts it again).
+    // A hub on an older template is upgraded before VMs get its address (a plain repair only starts it again). One on a
+    // newer version (codex-connector 5) keeps it: its VM port serves exactly as on 4 once retirePool has run.
     const template = opened.template;
     if (!opened.created && (!opts.repairOnly || opts.renewInvalid) && hubCloud.upgrade && template?.version != null && template.version < want && (template.upgradeTo ?? 0) >= want) {
         progress('Updating the CLIProxyAPI hub…');
@@ -654,14 +652,11 @@ export async function ensureCliProxyHub(cloud, opts) {
     }
     if ((await ws.list(HUB_AUTH_DIR)) === null)
         await ws.sh(`mkdir -p -m 0700 ${HUB_AUTH_DIR}`);
-    if (opts.pool) {
-        // The router is in the hub's sf-agent: an older hub that could not be upgraded above cannot pool.
-        const running = opened.template?.version;
-        if (running != null && running < POOL_HUB_TEMPLATE_VERSION) {
-            throw new HubError(`Pooled accounts need the hub on ${HUB_TEMPLATE} ${POOL_HUB_TEMPLATE_VERSION}; it runs ${running} and no update is published yet.`);
-        }
-        await setUpPool(ws, opts.apiUrl, opts.pool);
-    }
+    // A hub that once ran the retired account pool router: its files go, and its sf-agent starts again without it.
+    if (await retirePool(ws))
+        restart = true;
+    // The user's logins stay as they are; only the retired account pool router's files are kept disabled.
+    const logins = await ws.withLock('logins', () => settleHubLogins(ws));
     const started = await ws.sh(START_SCRIPT, [restart ? '1' : '0'], { timeoutMs: 180_000 });
     const startLines = started.stdout.trim().split('\n');
     const state = startLines.find((l) => ['running', 'started', 'restarted'].includes(l));
@@ -681,46 +676,199 @@ export async function ensureCliProxyHub(cloud, opts) {
     };
     if (!local || hubInfoText(local) !== hubInfoText(info))
         writeHubInfo(opts.dir, info);
-    return { info, created, process: state, callbackReplaced, config };
+    return { info, created, process: state, callbackReplaced, config, logins };
+}
+const docOf = (text) => {
+    try {
+        const v = JSON.parse(text ?? '');
+        return v && typeof v === 'object' && !Array.isArray(v) ? v : null;
+    }
+    catch {
+        return null;
+    }
+};
+const publicLogins = (logins) => logins.filter((l) => !l.pool).map(({ file, provider, email, disabled, active }) => ({ file, provider, email, disabled, active }));
+/** The hub's logins, oldest first, as they are (nothing changes). Files the retired account pool router left are not listed. */
+export async function listHubLogins(ws) {
+    return publicLogins(await readHubLogins(ws));
+}
+/** The hub's logins, oldest first (the order authFiles lists them in). */
+async function readHubLogins(ws) {
+    const files = await ws.authFiles();
+    const texts = await Promise.all(files.map((f) => ws.read(`${HUB_AUTH_DIR}/${f}`)));
+    return files.map((file, i) => {
+        const text = texts[i] ?? null;
+        const doc = docOf(text);
+        const type = typeof doc?.type === 'string' && doc.type.trim() ? doc.type.trim().toLowerCase() : null;
+        const email = typeof doc?.email === 'string' && doc.email ? doc.email : null;
+        // The retired pool router's files (`sfa-<account id>.json`, `"prefix": "sfa-…"`) were never the user's logins.
+        const pool = /^sfa-[^/]+\.json$/.test(file) || (typeof doc?.prefix === 'string' && doc.prefix.startsWith('sfa-'));
+        const disabled = doc?.disabled === true;
+        return { file, provider: type, email, disabled, active: !pool && !disabled && type !== null, text, doc, pool };
+    });
+}
+/** The auth file's text with `disabled` set (true) or removed (false), every other field kept; null when unchanged. */
+export function withDisabled(text, disabled) {
+    const doc = docOf(text);
+    if (!doc || (doc.disabled === true) === disabled)
+        return null;
+    if (disabled)
+        doc.disabled = true;
+    else
+        delete doc.disabled;
+    return `${JSON.stringify(doc, null, text.trimEnd().includes('\n') ? 2 : undefined)}${text.endsWith('\n') ? '\n' : ''}`;
+}
+/**
+ * Replaces one auth file atomically (0600, a temporary file in its folder then rename), only while it still holds what
+ * the device read ($2, its SHA-256): CLIProxyAPI rewrites a file when it refreshes the login, and a newer token must
+ * never be overwritten. The modification time stays, so the order of the hub's logins (oldest first) does not change.
+ * $1 is the path relative to the auth directory, the new content comes on standard input. Prints `written`, `changed`
+ * or `gone`.
+ */
+export const SET_AUTH_SCRIPT = `set -euo pipefail
+a=${HUB_AUTH_DIR}; rel=$1; want=$2
+case "$rel" in ''|/*|*..*) echo "not a path inside the auth folder" >&2; exit 2 ;; esac
+f="$a/$rel"
+[ -f "$f" ] || { echo gone; exit 0; }
+tmp=$(mktemp "$(dirname "$f")/.writing.XXXXXX"); trap 'rm -f "$tmp"' EXIT
+chmod 0600 "$tmp"; cat > "$tmp"; touch -r "$f" "$tmp"
+if [ "$(sha256sum < "$f" | cut -d' ' -f1)" != "$want" ]; then echo changed; exit 0; fi
+mv -f "$tmp" "$f"; trap - EXIT
+echo written
+`;
+const sha256 = (text) => createHash('sha256').update(text, 'utf8').digest('hex');
+/** Sets one login's `disabled` (true) or clears it (false), reading the file again when CLIProxyAPI changed it meanwhile. */
+async function setDisabled(ws, login, disabled) {
+    let text = login.text;
+    for (let attempt = 0; attempt < 4; attempt++) {
+        if (text === null)
+            return;
+        const next = withDisabled(text, disabled);
+        if (next === null)
+            return;
+        const r = await ws.sh(SET_AUTH_SCRIPT, [login.file, sha256(text)], { stdin: next });
+        const out = r.stdout.trim();
+        if (r.code !== 0)
+            throw new HubError(`Could not update ${login.file} in the hub: ${r.stderr.trim().split('\n').at(-1) || `exit ${r.code}`}`, 'run: shard cli-proxy import');
+        if (out === 'written' || out === 'gone')
+            return;
+        text = await ws.read(`${HUB_AUTH_DIR}/${login.file}`);
+    }
+    throw new HubError(`${login.file} kept changing in the hub while it was being ${disabled ? 'disabled' : 'enabled'}.`, 'run: shard cli-proxy import');
+}
+/**
+ * Under the hub's `logins` lock: the retired account pool router's files stay disabled (they were never the user's);
+ * every other login keeps the state its owner gave it. Returns the logins as they are now, oldest first.
+ */
+export async function settleHubLogins(ws) {
+    const logins = await readHubLogins(ws);
+    for (const l of logins)
+        if (l.pool && !l.disabled)
+            await setDisabled(ws, l, true);
+    return publicLogins(logins);
 }
 /**
- * The hub config with the providers' rate-limit headers passed through (`requests.passthrough-headers`, or the legacy
- * top-level key): CLIProxyAPI 8.0.23 records them per login only for non-streaming answers, so the pool router reads
- * them on its own answers. Returns the text unchanged when it already passes them.
+ * The user's choice of which logins serve requests (`shard cli-proxy use`; call it under the hub's `logins` lock).
+ * `only[provider]` names the one login of that provider that stays enabled (it is enabled first, then the provider's
+ * other logins are disabled, so the provider is never without one); `all` enables every login of the hub, which then
+ * routes across them as its config says. Providers the choice does not name are left as they are. Returns the logins
+ * as they are now, oldest first.
  */
-export function poolHubConfig(text) {
-    let doc = parseDocument(text || '{}');
+export async function useHubLogins(ws, choice) {
+    const logins = await readHubLogins(ws);
+    const only = choice.only ?? {};
+    for (const [p, file] of Object.entries(only)) {
+        if (!logins.some((l) => !l.pool && l.provider === p && l.file === file))
+            throw new HubError(`The hub has no ${p} login ${file} any more.`, 'see the hub’s logins: shard cli-proxy');
+    }
+    const wanted = (l) => {
+        if (l.pool || l.provider === null)
+            return null;
+        if (only[l.provider] !== undefined)
+            return only[l.provider] === l.file;
+        return choice.all ? true : null;
+    };
+    for (const l of logins)
+        if (wanted(l) === true && l.disabled)
+            await setDisabled(ws, l, false);
+    for (const l of logins)
+        if (wanted(l) === false && !l.disabled)
+            await setDisabled(ws, l, true);
+    // Computers still on CLI 0.22.0 keep the login active.json names for each provider: it names this choice too.
+    if (Object.keys(only).length) {
+        const text = await ws.read(LEGACY_ACTIVE_FILE);
+        let stored = {};
+        try {
+            const v = JSON.parse(text ?? '');
+            if (v && typeof v === 'object' && !Array.isArray(v))
+                stored = v;
+        }
+        catch { /* none yet */ }
+        const next = `${JSON.stringify({ ...stored, ...only }, null, 2)}\n`;
+        if (next !== text) {
+            const r = await ws.sh(LEGACY_ACTIVE_SCRIPT, [], { stdin: next });
+            if (r.code !== 0)
+                throw new HubError(`The hub did not record the choice: ${r.stderr.trim().split('\n').at(-1) || `exit ${r.code}`}`, 'run: shard cli-proxy use <email>');
+        }
+    }
+    return listHubLogins(ws);
+}
+/** Where CLI 0.22.0 reads each provider's one login ({"<provider>": "<auth file>"}); written atomically (0600). */
+export const LEGACY_ACTIVE_FILE = `${HUB_DIR}/active.json`;
+export const LEGACY_ACTIVE_SCRIPT = `set -euo pipefail
+dir=${HUB_DIR}; mkdir -p "$dir"; umask 077
+tmp=$(mktemp "$dir/.active.XXXXXX"); trap 'rm -f "$tmp"' EXIT
+cat > "$tmp"; mv -f "$tmp" "$dir/active.json"; trap - EXIT
+echo written
+`;
+/** What the retired account pool router kept in the hub (codex-connector 5 with pooled accounts). */
+const POOL_LEFTOVERS = ['api-token', 'api-url', 'pool.json', 'pool-creds.json', 'pool-status.json', 'pool-client.key', 'pool.lock'];
+/** Removes the router's files; prints `retired` when its hub token was there (its sf-agent then starts again without it). */
+export const RETIRE_POOL_SCRIPT = `set -euo pipefail
+cd ${HUB_DIR} 2>/dev/null || exit 0
+router=no; if [ -e api-token ] || [ -e api-url ]; then router=yes; fi
+rm -f ${POOL_LEFTOVERS.join(' ')}
+if [ "$router" = yes ]; then echo retired; fi
+`;
+/** The config without the router's own client key (`api-keys`, or v8 `access.api-keys`); the text itself when absent. */
+export function withoutClientKey(text, key) {
+    const doc = parseDocument(text);
     if (doc.errors.length > 0 || !isMap(doc.contents))
-        doc = new Document({});
-    const v8 = doc.hasIn(['config-version']) || V8_SECTIONS.some((k) => isMap(doc.getIn([k])));
-    const path = v8 ? ['requests', 'passthrough-headers'] : ['passthrough-headers'];
-    if (doc.getIn(path) === true)
         return text;
-    if (v8 && doc.hasIn(['passthrough-headers']))
-        doc.deleteIn(['passthrough-headers']);
-    doc.setIn(path, true);
-    return doc.toString();
-}
-const HUB_TOKEN_RE = /^sfh_[A-Za-z0-9_-]{43}$/;
-/** Pool mode in the hub: the API URL, the hub token (minted once, or again with renew) and header passthrough. */
-async function setUpPool(ws, apiUrl, pool) {
-    await ws.withLock('pool', async () => {
-        if ((await ws.read(HUB_POOL_FILES.apiUrl))?.trim() !== apiUrl)
-            await ws.write(HUB_POOL_FILES.apiUrl, `${apiUrl}\n`, '0600');
-        const have = (await ws.read(HUB_POOL_FILES.apiToken))?.trim() ?? '';
-        if (pool.renew || !HUB_TOKEN_RE.test(have)) {
-            const token = (await pool.hubToken(ws.id)).trim();
-            if (!HUB_TOKEN_RE.test(token))
-                throw new HubError('The Shardflux API did not return a hub token.');
-            await ws.write(HUB_POOL_FILES.apiToken, `${token}\n`, '0600');
+    let changed = false;
+    for (const path of [['api-keys'], ['access', 'api-keys']]) {
+        const seq = doc.getIn(path, true);
+        if (!isSeq(seq))
+            continue;
+        const keep = seq.items.filter((item) => (isScalar(item) ? item.value : item) !== key);
+        if (keep.length !== seq.items.length) {
+            seq.items = keep;
+            changed = true;
         }
-        const configPath = `${HUB_DIR}/config.yaml`;
-        const config = (await ws.read(configPath)) ?? '';
-        const next = poolHubConfig(config);
-        // A running CLIProxyAPI reloads its config by itself; the router starts within a minute of the token.
+    }
+    return changed ? doc.toString() : text;
+}
+/**
+ * A hub that ran the account pool router (codex-connector 5): its hub token, API URL and router state go, and so does
+ * the router's client key in the hub's config. Returns whether the router could be running (the hub then restarts its
+ * sf-agent, which starts without it). A hub that never pooled costs one directory listing.
+ */
+async function retirePool(ws) {
+    const names = (await ws.list(HUB_DIR)) ?? [];
+    if (!names.some((n) => POOL_LEFTOVERS.includes(n)))
+        return false;
+    const key = (await ws.read(`${HUB_DIR}/pool-client.key`))?.trim();
+    const configPath = `${HUB_DIR}/config.yaml`;
+    const config = key ? await ws.read(configPath) : null;
+    if (key && config !== null) {
+        const next = withoutClientKey(config, key);
         if (next !== config)
             await ws.write(configPath, next, '0600');
-    });
+    }
+    const r = await ws.sh(RETIRE_POOL_SCRIPT);
+    if (r.code !== 0)
+        throw new HubError(`The hub did not remove the account pool’s files: ${r.stderr.trim().split('\n').at(-1) || `exit ${r.code}`}`, 'run: shard cli-proxy import');
+    return r.stdout.trim() === 'retired';
 }
 /**
  * The hub's one callback URL. Under the hub's callback lock: the stored one is kept while the port's callback still

package/dist/cli-proxy.d.ts

@@ -211,6 +211,10 @@ export interface HubAccount {
     email: string | null;
     status: string | null;
     disabled: boolean;
+    /** The auth file, relative to the hub's auth directory (null when CLIProxyAPI does not name it). */
+    file: string | null;
+    /** It serves requests (enabled); a disabled login stays in the hub unused. */
+    active: boolean;
 }
 /** The hub's logins from its management API (wakes the hub). */
 export declare function hubAccounts(info: HubInfo, timeoutMs?: number): Promise<HubAccount[]>;
@@ -230,6 +234,12 @@ export interface ImportOptions {
     machine?: boolean;
     platform?: NodeJS.Platform;
 }
+/** A login named by provider, email and auth file (never its tokens). */
+export interface LoginRef {
+    provider: string;
+    email: string | null;
+    file: string;
+}
 export interface ImportResult {
     hub: {
         workspace_id: string;
@@ -238,7 +248,8 @@ export interface ImportResult {
     };
     /**
      * added: accounts this import gave the hub; already: accounts the hub had (its tokens stay); moved_aside: where this
-     * computer's auth files went once the hub had them (`<dir>.moved-to-shardflux`).
+     * computer's auth files went once the hub had them (`<dir>.moved-to-shardflux`). active: the logins that serve
+     * requests; disabled: logins kept in the hub unused (as they were on the computer, or by --use).
      */
     logins: {
         uploaded: string[];
@@ -252,6 +263,8 @@ export interface ImportResult {
             email: string | null;
         }>;
         moved_aside: string[];
+        active: LoginRef[];
+        disabled: LoginRef[];
     };
     /** Whether the hub's config was made from this computer's, or the hub kept the one it has. */
     config: 'from_this_computer' | 'kept';
@@ -271,11 +284,21 @@ export interface ImportResult {
     dropped: string[];
     accounts: HubAccount[] | null;
 }
+/** The logins a `--use` or `shard cli-proxy use` value names: by email (any case), path in the auth folder, or file name. */
+export declare function matchLogins<T extends {
+    provider: string | null;
+    email: string | null;
+    file: string;
+}>(spec: string, logins: readonly T[]): T[];
 /**
  * Moves CLIProxyAPI from this computer into the account (or connects this computer to the account's hub): ensure the
  * hub from the user's config, stop the local copy, upload the auth files the hub does not have yet (never overwriting
  * one: the hub's copy may hold a newer refresh token), start the forwarder on 127.0.0.1:8317 and put the hub info into
  * the machine template.
+ *
+ * Each login arrives as it is on this computer (enabled or disabled), and the hub routes across the enabled ones as
+ * the user's config says. `--use <email|file>` (repeatable, one per provider) makes that login the only one serving
+ * its provider (the provider's others are kept in the hub, disabled), as `shard cli-proxy use` does.
  */
 export declare function importCliProxy(ctx: Ctx, opts?: ImportOptions): Promise<ImportResult>;
 export interface ExportOptions {
@@ -306,6 +329,26 @@ export interface ExportResult {
  * Undone by `shard cli-proxy import`.
  */
 export declare function exportCliProxy(ctx: Ctx, opts?: ExportOptions): Promise<ExportResult>;
+export interface UseResult {
+    /** Each provider this command set to one login, with the login that served it alone before (null if not one) and the one now. */
+    switched: Array<{
+        provider: string;
+        from: LoginRef | null;
+        to: LoginRef;
+    }>;
+    /** The logins that serve requests now, and the ones kept in the hub, disabled. */
+    active: LoginRef[];
+    disabled: LoginRef[];
+}
+/**
+ * `shard cli-proxy use <email|file>`: that login alone serves its provider (an email with a Claude and a Codex login
+ * sets both); the provider's other logins stay in the hub, disabled. `--all`: every login serves again, and the hub
+ * routes across them as its config says. Under the hub's logins lock.
+ */
+export declare function useCliProxyLogin(ctx: Ctx, spec: string, opts?: {
+    hub?: HubCloud;
+    all?: boolean;
+}): Promise<UseResult>;
 /**
  * After the edge refused the hub's callback URL (`callback_invalid`: the API key that created it was revoked):
  * replaces the URLs the edge refuses (under the hub's lock), saves them locally and gives VMs the new VM address.