@tikoci/centrs 0.1.1 → 0.1.3

package/src/retrieve-fanout.ts

@@ -1,28 +1,33 @@
 /**
- * Group fanout for `retrieve`.
+ * Multi-target fan-out for `retrieve`.
  *
- * A `--group <name>` selector expands a CDB group into N targets and runs each
- * through the single-target retrieve tail (`runResolvedRetrieve`), collecting a
+ * A target selection (`--group` / `--where` / `--all` / `--default` / multiple
+ * positionals) expands to N members and runs each through the single-target
+ * retrieve tail (`buildResolvedRetrieve` → `runResolvedRetrieve`), collecting a
  * per-target inner `CentrsEnvelope` into the LOCKED `FanoutData` shape from
  * `src/core/envelope.ts`. The outer envelope's `ok` means the orchestration
  * produced a complete per-target result set; a per-target failure is an INNER
  * `ok: false` envelope, never an outer failure and never a thrown error out of
  * the batch.
  *
- * Design rules implemented here (see `commands/retrieve/README.md`):
- *   - The CDB is loaded + decrypted ONCE for the whole fanout (group expansion).
- *   - Each target is resolved + validated independently (validation is
- *     per-target via live `/console/inspect`; schemas differ by version).
+ * Boundaries (see `commands/retrieve/README.md`, `docs/CONSTITUTION.md`), shared
+ * with `api-fanout.ts`:
+ *   - The CDB is loaded + decrypted ONCE (`expandCdbSelection`); a cdb member is
+ *     resolved with its pre-resolved record, so `resolveCdb`'s `__default__`
+ *     synthetic fallback never collides ad-hoc literals.
+ *   - A `--default` / `__default__` member is guarded from being dialed as the
+ *     literal hostname `"__default__"` — it fails that one target deterministically
+ *     with `target/unresolved`.
+ *   - Each target is resolved + validated independently (validation is per-target
+ *     via live `/console/inspect`; schemas differ by version).
  *   - Concurrency is a bounded worker pool; defaults are transport-aware
  *     (REST 8, native-api 4) and overridable with `--concurrency`.
- *   - Transient drops are retried with jittered backoff on an allowlisted
- *     retryable-code set only; router-side and client-side errors are not.
- *   - `data.targets[]` is ordered by CDB `recordIndex` regardless of completion
- *     order; `summary` counts are derived from inner `ok`.
+ *   - `data.targets[]` is ordered by CDB `recordIndex` (literals appended in
+ *     positional order); `summary` counts are derived from inner `ok`.
  *
- * Outer `ok: false` is reserved for fanout failing BEFORE reliable per-target
+ * Outer `ok: false` is reserved for fan-out failing BEFORE reliable per-target
  * results exist (bad request shape, CDB decrypt failure); those throw out of
- * {@link retrieveGroup} and the CLI renders them with
+ * {@link retrieveFanout} and the CLI renders them with
  * {@link buildRetrieveFanoutErrorEnvelope}.
  */
 
@@ -32,13 +37,25 @@ import type {
 	FanoutData,
 	FanoutSummary,
 } from "./core/envelope.ts";
+import {
+	buildFanoutEnvelope,
+	buildFanoutResolveFailure,
+	commonVia,
+	defaultFanoutSleep,
+	resolveFanoutConcurrency,
+	runFanout,
+	summarizeFanout,
+} from "./core/fanout.ts";
 import { CentrsError, serializeCentrsError } from "./errors.ts";
 import { plannedProtocols, type RouterOsProtocol } from "./protocols/index.ts";
 import {
-	type CdbGroupExpansion,
-	type CdbGroupResolveInput,
-	type CdbResolution,
-	expandCdbGroup,
+	type CdbSelectionExpansion,
+	type CdbSelectionMember,
+	type CdbSelectionResolveInput,
+	expandCdbSelection,
+	isDefaultRecordTarget,
+	loadEnvFileDefaults,
+	type TargetSelection,
 } from "./resolver/index.ts";
 import {
 	buildResolvedRetrieve,
@@ -52,38 +69,27 @@ import {
 	type RetrieveSuccessEnvelope,
 	resolveMacForRetrieve,
 	resolveRetrieveGlobalContext,
+	resolveRetrieveRequest,
 	runResolvedRetrieve,
 	toYaml,
 	validateRetrieveRequestShape,
 } from "./retrieve.ts";
 
-/** Transport-aware default worker-pool size (REST drops parallel POSTs above ~8). */
-export const RETRIEVE_FANOUT_CONCURRENCY_DEFAULTS: Record<
-	"rest-api" | "native-api",
-	number
-> = {
-	"rest-api": 8,
-	"native-api": 4,
-};
-
-/** Extra attempts after the first try (jittered backoff between them). */
-export const RETRIEVE_FANOUT_MAX_RETRIES = 2;
-
-/** Base backoff in milliseconds; doubled per attempt with up to +base jitter. */
-export const RETRIEVE_FANOUT_BACKOFF_BASE_MS = 200;
-
-/** RouterOS-style codes that are safe to retry under fanout. */
-export const RETRIEVE_FANOUT_RETRYABLE_CODES = [
-	"transport/network",
-	"transport/connection-closed",
-] as const;
+/** Compact description of the selector that produced the fan-out. */
+export interface RetrieveSelectionSummary {
+	groups: readonly string[];
+	where: readonly string[];
+	all: boolean;
+	default: boolean;
+	positionals: readonly string[];
+}
 
 export interface RetrieveFanoutOperationMeta {
 	kind: "fanout";
-	group: string;
 	concurrency: number;
 	summary: FanoutSummary;
 	request: RetrieveRequestSummary;
+	selection: RetrieveSelectionSummary;
 }
 
 export type RetrieveFanoutData = FanoutData<unknown, RetrieveOperationMeta>;
@@ -95,60 +101,109 @@ export type RetrieveFanoutErrorEnvelope =
 	CentrsErrorEnvelope<RetrieveFanoutOperationMeta>;
 
 /** Test/override seams; production callers pass nothing. */
-export interface RetrieveGroupInternals {
+export interface RetrieveFanoutInternals {
 	/** Per-target executor (attempts; throws on failure). Defaults to the live tail. */
 	execute?: (
 		resolved: ResolvedRetrieveRequest,
-		recordIndex: number,
 	) => Promise<RetrieveSuccessEnvelope>;
-	/** Group expansion. Defaults to the real CDB-backed expansion. */
+	/** Selection expansion. Defaults to the real CDB-backed expansion. */
 	expand?: (
-		input: CdbGroupResolveInput,
+		selection: TargetSelection,
+		input: CdbSelectionResolveInput,
 		env: Record<string, string | undefined>,
-	) => Promise<CdbGroupExpansion>;
+		config?: Record<string, string | undefined>,
+	) => Promise<CdbSelectionExpansion>;
 	/** Backoff sleeper. Defaults to a real timer; tests pass a no-op. */
 	sleep?: (ms: number) => Promise<void>;
 }
 
+export interface RetrieveFanoutOptions {
+	concurrency?: number;
+	/** CLI: true (literal positionals allowed). MCP: false (CDB is the allowlist). */
+	allowAdhoc?: boolean;
+}
+
+function summarizeSelection(
+	selection: TargetSelection,
+): RetrieveSelectionSummary {
+	return {
+		groups: selection.groups,
+		where: selection.where.map((clause) => `${clause.key}=${clause.value}`),
+		all: selection.all,
+		default: selection.default,
+		positionals: selection.positionals,
+	};
+}
+
+function memberTargetMeta(member: CdbSelectionMember): {
+	input?: string;
+	identity?: string;
+	recordIndex?: number;
+} {
+	if (member.kind === "cdb") {
+		return {
+			input: member.resolution.identity,
+			identity: member.resolution.identity,
+			recordIndex: member.recordIndex,
+		};
+	}
+	return { input: member.input };
+}
+
+/**
+ * Ad-hoc literal targets have no CDB record. `resolveRetrieveRequest` may borrow
+ * the `__default__` record's index when filling fallback creds, so the inner
+ * envelope would carry that borrowed `recordIndex` — making distinct literals
+ * collide on one index. Drop it for literal members and stamp the caller's `input`
+ * so the target stays identifiable in renderers (mirrors `api-fanout.ts`).
+ */
+function relabelAdhocMember(
+	envelope: RetrieveEnvelope,
+	member: CdbSelectionMember,
+): RetrieveEnvelope {
+	if (member.kind !== "literal") {
+		return envelope;
+	}
+	const { recordIndex: _borrowed, ...target } = envelope.meta.target;
+	return {
+		...envelope,
+		meta: { ...envelope.meta, target: { ...target, input: member.input } },
+	};
+}
+
 /**
- * Run a `retrieve` group fanout. Always returns a success envelope when the
+ * Run a `retrieve` fan-out. Always returns a success envelope when the
  * orchestration completes (even if every target failed); throws only for
  * pre-flight failures (bad request shape, CDB decrypt) the caller renders as an
  * outer error.
  */
-export async function retrieveGroup(
+export async function retrieveFanout(
 	request: RetrieveRequest,
+	selection: TargetSelection,
 	env: Record<string, string | undefined> = Bun.env,
-	internals: RetrieveGroupInternals = {},
+	internals: RetrieveFanoutInternals = {},
+	options: RetrieveFanoutOptions = {},
 ): Promise<RetrieveFanoutEnvelope> {
-	const group = request.group;
-	if (group === undefined || group.length === 0) {
-		throw new CentrsError({
-			code: "usage/missing-group",
-			summary: "retrieve group fanout requires a non-empty `--group` value.",
-			remediation:
-				"Pass `--group <name>` matching a CDB group, e.g. `centrs retrieve --group prod /system/resource`.",
-		});
-	}
-
 	const attributeSelections = validateRetrieveRequestShape(request);
+	const config = await loadEnvFileDefaults(env);
 	const global = resolveRetrieveGlobalContext(
 		request,
 		env,
 		attributeSelections,
+		config,
 	);
-	if (request.concurrency !== undefined) {
-		resolveFanoutConcurrency(request.concurrency, global.via);
-	}
+	const selectionSummary = summarizeSelection(selection);
 
-	const expand = internals.expand ?? expandCdbGroup;
+	const expand = internals.expand ?? expandCdbSelection;
 	const expansion = await expand(
+		selection,
 		{
-			group,
 			cdbFile: request.cdbFile,
 			cdbPassword: request.cdbPassword,
+			allowAdhoc: options.allowAdhoc ?? true,
 		},
 		env,
+		config,
 	);
 
 	const concurrencyVia = resolveFanoutConcurrencyProtocol(
@@ -158,13 +213,13 @@ export async function retrieveGroup(
 		expansion,
 	);
 	const concurrency = resolveFanoutConcurrency(
-		request.concurrency,
+		options.concurrency ?? request.concurrency,
 		concurrencyVia,
 	);
 
 	if (expansion.empty) {
-		return buildFanoutEnvelope({
-			group,
+		return retrieveFanoutEnvelope({
+			selection: selectionSummary,
 			concurrency,
 			summary: { total: 0, ok: 0, failed: 0 },
 			requestSummary: global.summary,
@@ -176,38 +231,70 @@ export async function retrieveGroup(
 	}
 
 	const execute = internals.execute ?? runResolvedRetrieve;
-	const sleep = internals.sleep ?? defaultSleep;
-
-	const targets = await runBoundedPool(
-		expansion.targets,
+	const sleep = internals.sleep ?? defaultFanoutSleep;
+
+	const targets = await runFanout<
+		CdbSelectionMember,
+		ResolvedRetrieveRequest,
+		RetrieveEnvelope
+	>({
+		members: expansion.targets,
 		concurrency,
-		async (member): Promise<RetrieveEnvelope> => {
-			let resolved: ResolvedRetrieveRequest;
-			try {
+		resolve: async (member) => {
+			if (member.kind === "cdb") {
+				if (isDefaultRecordTarget(member.resolution.target)) {
+					throw new CentrsError({
+						code: "target/unresolved",
+						summary:
+							"The `__default__` record is a credential fallback, not a connectable router.",
+						remediation:
+							"Select real devices with `--group` / `--where` / `--all` or a positional; `--default` cannot be a RouterOS target.",
+						context: { target: member.resolution.target },
+					});
+				}
 				const macResolution = await resolveMacForRetrieve(
 					request,
 					env,
 					member.resolution,
+					config,
 				);
-				resolved = buildResolvedRetrieve(
+				return buildResolvedRetrieve(
 					request,
 					env,
 					member.resolution,
 					attributeSelections,
 					macResolution,
+					config,
 				);
-			} catch (error) {
-				return buildResolveFailureEnvelope(member.resolution, error);
 			}
-			return runTargetWithRetry(resolved, member.recordIndex, execute, sleep);
+			// Ad-hoc literal: full single-target resolution (CDB load → `__default__`
+			// fallback creds), mirroring single-target `retrieve`.
+			return resolveRetrieveRequest(
+				{ ...request, targetInput: member.input },
+				env,
+			);
 		},
-	);
+		onResolveError: (member, error) =>
+			buildFanoutResolveFailure<RetrieveOperationMeta>({
+				error,
+				target: memberTargetMeta(member),
+				warnings: member.kind === "cdb" ? [...member.resolution.warnings] : [],
+				summary: "Failed to resolve a fanout target.",
+			}),
+		execute: async (resolved, member) =>
+			relabelAdhocMember(await execute(resolved), member),
+		onExecuteError: (resolved, member, error) =>
+			relabelAdhocMember(
+				buildRetrieveErrorEnvelopeFromResolved(resolved, error),
+				member,
+			),
+		sleep,
+	});
 
-	const summary = summarizeFanout(targets);
-	return buildFanoutEnvelope({
-		group,
+	return retrieveFanoutEnvelope({
+		selection: selectionSummary,
 		concurrency,
-		summary,
+		summary: summarizeFanout(targets),
 		requestSummary: global.summary,
 		settings: global.settings,
 		via: commonVia(targets),
@@ -216,6 +303,37 @@ export async function retrieveGroup(
 	});
 }
 
+/**
+ * Back-compat shim for the `--group`-only fan-out: builds a single-group
+ * {@link TargetSelection} and delegates to {@link retrieveFanout}. Kept for the
+ * MCP `centrs_retrieve` tool, whose surface exposes only `group`.
+ */
+export async function retrieveGroup(
+	request: RetrieveRequest,
+	env: Record<string, string | undefined> = Bun.env,
+	internals: RetrieveFanoutInternals = {},
+): Promise<RetrieveFanoutEnvelope> {
+	const group = request.group;
+	if (group === undefined || group.length === 0) {
+		throw new CentrsError({
+			code: "usage/missing-group",
+			summary: "retrieve group fanout requires a non-empty `--group` value.",
+			remediation:
+				"Pass `--group <name>` matching a CDB group, e.g. `centrs retrieve --group prod /system/resource`.",
+		});
+	}
+	const selection: TargetSelection = {
+		positionals: [],
+		groups: [group],
+		all: false,
+		default: false,
+		where: [],
+	};
+	return retrieveFanout(request, selection, env, internals, {
+		concurrency: request.concurrency,
+	});
+}
+
 /** Outer error envelope for a fanout that failed before per-target results. */
 export function buildRetrieveFanoutErrorEnvelope(
 	request: RetrieveRequest,
@@ -226,7 +344,7 @@ export function buildRetrieveFanoutErrorEnvelope(
 			? error
 			: new CentrsError({
 					code: "internal/unhandled",
-					summary: "retrieve group fanout failed with an unexpected error.",
+					summary: "retrieve fan-out failed with an unexpected error.",
 					remediation:
 						"Re-run with `--format json` to capture the structured error details for debugging.",
 					cause: error,
@@ -251,40 +369,24 @@ export function buildRetrieveFanoutErrorEnvelope(
 	};
 }
 
-/** Resolve and validate the worker-pool size. */
-export function resolveFanoutConcurrency(
-	requested: number | undefined,
-	via: RouterOsProtocol,
-): number {
-	if (requested !== undefined) {
-		if (!Number.isInteger(requested) || requested < 1) {
-			throw new CentrsError({
-				code: "usage/invalid-concurrency",
-				summary: `--concurrency must be an integer >= 1. Received: ${requested}`,
-				remediation: "Pass a positive integer, e.g. `--concurrency 4`.",
-				context: { concurrency: requested },
-			});
-		}
-		return requested;
-	}
-	if (via === "native-api") {
-		return RETRIEVE_FANOUT_CONCURRENCY_DEFAULTS["native-api"];
-	}
-	return RETRIEVE_FANOUT_CONCURRENCY_DEFAULTS["rest-api"];
-}
-
 function resolveFanoutConcurrencyProtocol(
 	request: RetrieveRequest,
 	env: Record<string, string | undefined>,
 	globalVia: RouterOsProtocol,
-	expansion: CdbGroupExpansion,
+	expansion: CdbSelectionExpansion,
 ): RouterOsProtocol {
+	// Deliberately CLI/env only, not `config` — a `centrs.env` default is the
+	// weakest precedence tier and must not be treated as a hard pin the way an
+	// explicit `--via`/`CENTRS_VIA` is; doing so would defeat per-target CDB
+	// `via=` overrides for members that disagree with the global default.
 	if (request.via !== undefined || env["CENTRS_VIA"] !== undefined) {
 		return globalVia;
 	}
 	if (
 		expansion.targets.some(
-			(target) => target.resolution.overrides.via?.value === "native-api",
+			(member) =>
+				member.kind === "cdb" &&
+				member.resolution.overrides.via?.value === "native-api",
 		)
 	) {
 		return "native-api";
@@ -292,124 +394,8 @@ function resolveFanoutConcurrencyProtocol(
 	return globalVia;
 }
 
-/**
- * Classify a per-target failure as retryable. Only the locked allowlist
- * retries: generic transport/network failures and dropped/closed connections
- * (including REST 5xx mapped to `transport/connection-closed`). Router-side
- * (`routeros/*`), validation, auth, cdb, target, input, usage, timeout, TLS,
- * connection-refused, and DNS failures are deterministic and never retried.
- */
-export function isRetryableFanoutError(error: unknown): boolean {
-	if (!(error instanceof CentrsError)) {
-		return false;
-	}
-	return (RETRIEVE_FANOUT_RETRYABLE_CODES as readonly string[]).includes(
-		error.code,
-	);
-}
-
-export function summarizeFanout(
-	targets: readonly RetrieveEnvelope[],
-): FanoutSummary {
-	let ok = 0;
-	let failed = 0;
-	for (const target of targets) {
-		if (target.ok) {
-			ok += 1;
-		} else {
-			failed += 1;
-		}
-	}
-	return { total: targets.length, ok, failed };
-}
-
-async function runTargetWithRetry(
-	resolved: ResolvedRetrieveRequest,
-	recordIndex: number,
-	execute: (
-		resolved: ResolvedRetrieveRequest,
-		recordIndex: number,
-	) => Promise<RetrieveSuccessEnvelope>,
-	sleep: (ms: number) => Promise<void>,
-): Promise<RetrieveEnvelope> {
-	let attempt = 0;
-	for (;;) {
-		try {
-			return await execute(resolved, recordIndex);
-		} catch (error) {
-			if (
-				attempt < RETRIEVE_FANOUT_MAX_RETRIES &&
-				isRetryableFanoutError(error)
-			) {
-				attempt += 1;
-				await sleep(backoffMs(attempt));
-				continue;
-			}
-			return buildRetrieveErrorEnvelopeFromResolved(resolved, error);
-		}
-	}
-}
-
-/**
- * Ordered bounded worker pool. Runs `worker` over `items` with at most
- * `concurrency` in flight, returning results in INPUT order regardless of
- * completion order.
- */
-export async function runBoundedPool<I, O>(
-	items: readonly I[],
-	concurrency: number,
-	worker: (item: I, index: number) => Promise<O>,
-): Promise<O[]> {
-	const results = new Array<O>(items.length);
-	let next = 0;
-	const width = Math.max(1, Math.min(concurrency, items.length));
-
-	async function drain(): Promise<void> {
-		for (;;) {
-			const index = next;
-			next += 1;
-			if (index >= items.length) {
-				return;
-			}
-			results[index] = await worker(items[index] as I, index);
-		}
-	}
-
-	await Promise.all(Array.from({ length: width }, () => drain()));
-	return results;
-}
-
-function buildResolveFailureEnvelope(
-	resolution: CdbResolution,
-	error: unknown,
-): RetrieveEnvelope {
-	const centrsError =
-		error instanceof CentrsError
-			? error
-			: new CentrsError({
-					code: "internal/unhandled",
-					summary: "Failed to resolve a fanout target.",
-					cause: error,
-				});
-	return {
-		ok: false,
-		error: serializeCentrsError(centrsError),
-		warnings: [...resolution.warnings],
-		tips: [],
-		meta: {
-			target: {
-				input: resolution.identity,
-				identity: resolution.identity,
-				recordIndex: resolution.recordIndex,
-			},
-			via: null,
-			settings: {},
-		},
-	};
-}
-
 interface FanoutEnvelopeInput {
-	group: string;
+	selection: RetrieveSelectionSummary;
 	concurrency: number;
 	summary: FanoutSummary;
 	requestSummary: RetrieveRequestSummary;
@@ -419,54 +405,28 @@ interface FanoutEnvelopeInput {
 	targets: readonly RetrieveEnvelope[];
 }
 
-function buildFanoutEnvelope(
+/** retrieve's outer fan-out success envelope (delegates to the shared core). */
+function retrieveFanoutEnvelope(
 	input: FanoutEnvelopeInput,
 ): RetrieveFanoutEnvelope {
-	return {
-		ok: true,
-		data: {
-			summary: input.summary,
-			targets: input.targets,
-		},
+	return buildFanoutEnvelope<
+		unknown,
+		RetrieveOperationMeta,
+		RetrieveFanoutOperationMeta
+	>({
+		summary: input.summary,
+		targets: input.targets,
 		warnings: input.warnings,
-		tips: [],
-		meta: {
-			target: {},
-			via: input.via,
-			settings: input.settings,
-			operation: {
-				kind: "fanout",
-				group: input.group,
-				concurrency: input.concurrency,
-				summary: input.summary,
-				request: input.requestSummary,
-			},
+		settings: input.settings,
+		via: input.via,
+		operation: {
+			kind: "fanout",
+			concurrency: input.concurrency,
+			summary: input.summary,
+			request: input.requestSummary,
+			selection: input.selection,
 		},
-	};
-}
-
-function commonVia(
-	targets: readonly RetrieveEnvelope[],
-): RouterOsProtocol | null {
-	let via: RouterOsProtocol | null | undefined;
-	for (const target of targets) {
-		const candidate = target.meta.via;
-		if (via === undefined) {
-			via = candidate;
-		} else if (via !== candidate) {
-			return null;
-		}
-	}
-	return via ?? null;
-}
-
-function backoffMs(attempt: number): number {
-	const base = RETRIEVE_FANOUT_BACKOFF_BASE_MS * 2 ** (attempt - 1);
-	return base + Math.floor(Math.random() * RETRIEVE_FANOUT_BACKOFF_BASE_MS);
-}
-
-function defaultSleep(ms: number): Promise<void> {
-	return new Promise((resolve) => setTimeout(resolve, ms));
+	});
 }
 
 /**
@@ -504,12 +464,16 @@ function renderFanoutText(
 	const operation = envelope.meta.operation;
 	const { total, ok, failed } = envelope.data.summary;
 	const lines: string[] = [
-		`group ${operation?.group ?? "?"}: ${ok}/${total} ok, ${failed} failed (concurrency ${operation?.concurrency ?? "?"})`,
+		`retrieve ${operation?.request.path ?? "?"}: ${ok}/${total} ok, ${failed} failed (concurrency ${operation?.concurrency ?? "?"})`,
 	];
 
 	for (const target of envelope.data.targets) {
 		const meta = target.meta;
-		const label = meta.target.identity ?? meta.target.host ?? "(unknown)";
+		const label =
+			meta.target.identity ??
+			meta.target.host ??
+			meta.target.input ??
+			"(unknown)";
 		const index = meta.target.recordIndex ?? "-";
 		if (target.ok) {
 			lines.push(