@untangle-code/cli 0.0.0-stage → 0.1.1

package/src/commands/init/services.ts

@@ -0,0 +1,271 @@
+/**
+ * Probing the two pieces of infrastructure Untangle needs.
+ *
+ * READ-ONLY, ON PURPOSE. `HttpProbePort` can issue a GET and nothing else, so
+ * `init` is structurally incapable of changing a service it finds — which is
+ * what spec.md §3.2 demands of External Infrastructure mode ("Untangle DARF in
+ * diesem Modus keine fremden Prozesse oder Container verändern").
+ *
+ * WHICH URLS ARE PROBED AT ALL IS NOT DECIDED HERE ANY MORE (WP-222). That
+ * rule lives in `cli/vector-store-policy.ts` and is the SAME rule
+ * `untangle doctor` obeys. It used to be written out twice — once here, once
+ * (by omission) in `doctor` — and the two disagreed: `init` refused upstream's
+ * default port 6333 citing spec.md §3.2, while `doctor` fell back onto it,
+ * found the foreign instance living there and reported it as healthy. Two
+ * commands of one tool, two answers to a safety question. One rule, one
+ * file, one answer.
+ *
+ * NOT FINDING EITHER SERVICE IS NOT AN ERROR (WP-125): the configuration is
+ * written regardless, and the command names exactly one next step.
+ */
+
+import { compareCodepoints } from "@untangle-code/domain";
+
+import { MANAGED_QDRANT_URL, qdrantProbePlan } from "../../cli/vector-store-policy.js";
+
+import type {
+  EmbeddingModelInfoV1,
+  HttpProbePort,
+  InitLimitsV1,
+  OllamaFindingsV1,
+  QdrantFindingsV1,
+} from "./types.js";
+
+/** Native Ollama probe target (spec.md §3.1). */
+export const DEFAULT_OLLAMA_URL = "http://127.0.0.1:11434";
+
+/**
+ * Managed Qdrant REST port (spec.md §3.1) — re-exported, NOT redefined. The
+ * value belongs to the shared policy; a second copy here is exactly how the
+ * two commands drifted apart in the first place.
+ */
+export const DEFAULT_QDRANT_URL = MANAGED_QDRANT_URL;
+
+interface OllamaModelDetails {
+  context_length?: unknown;
+  embedding_length?: unknown;
+}
+
+interface OllamaModel {
+  name?: unknown;
+  capabilities?: unknown;
+  details?: OllamaModelDetails;
+}
+
+/**
+ * Reads `/api/tags` into the embedding models it describes.
+ *
+ * The provider reports `capabilities`, `embedding_length` and
+ * `context_length` per model, so the vector width is READ, never guessed and
+ * never obtained by sending the model something to embed. A model that
+ * predates the `capabilities` field is still recognised by the presence of an
+ * `embedding_length`, which only embedding models carry.
+ */
+export function readEmbeddingModels(payload: unknown): EmbeddingModelInfoV1[] {
+  if (typeof payload !== "object" || payload === null) {
+    return [];
+  }
+  const models = (payload as { models?: unknown }).models;
+  if (!Array.isArray(models)) {
+    return [];
+  }
+  const found: EmbeddingModelInfoV1[] = [];
+  for (const raw of models as OllamaModel[]) {
+    if (typeof raw !== "object" || raw === null) {
+      continue;
+    }
+    const name = typeof raw.name === "string" ? raw.name : undefined;
+    if (name === undefined || name.length === 0) {
+      continue;
+    }
+    const capabilities = Array.isArray(raw.capabilities)
+      ? raw.capabilities.filter((item): item is string => typeof item === "string")
+      : [];
+    const details = typeof raw.details === "object" && raw.details !== null ? raw.details : {};
+    const dimensions = positiveInteger(details.embedding_length);
+    const contextLength = positiveInteger(details.context_length);
+    const isEmbedding = capabilities.includes("embedding") || dimensions !== undefined;
+    if (!isEmbedding) {
+      continue;
+    }
+    found.push({
+      name,
+      ...(dimensions === undefined ? {} : { dimensions }),
+      ...(contextLength === undefined ? {} : { contextLength }),
+    });
+  }
+  found.sort((a, b) => compareCodepoints(a.name, b.name));
+  return found;
+}
+
+export interface ModelChoiceV1 {
+  model: EmbeddingModelInfoV1;
+  reason: string;
+}
+
+/**
+ * Chooses one embedding model out of what the provider actually has.
+ *
+ * The rule, in order: the widest context window first, because a code chunk
+ * that does not fit is a chunk that gets truncated; then the widest vector,
+ * because that is the model with more to say; then codepoint order, so the
+ * decision never depends on the order the provider happened to list its
+ * models in. `preferred` (an explicit user choice) short-circuits all of it.
+ *
+ * Deliberately NOT "whatever the product default is called": on a machine
+ * that has three embedding models and not the default one, silently falling
+ * back to a name that is not installed produces a configuration that fails on
+ * first use.
+ */
+export function chooseEmbeddingModel(
+  models: readonly EmbeddingModelInfoV1[],
+  preferred?: string,
+): ModelChoiceV1 | undefined {
+  if (models.length === 0) {
+    return undefined;
+  }
+  if (preferred !== undefined) {
+    const exact = models.find((model) => model.name === preferred);
+    if (exact !== undefined) {
+      return { model: exact, reason: "explicitly requested (--embedding-model)" };
+    }
+  }
+  const ranked = [...models].sort((a, b) => {
+    const contextDelta = (b.contextLength ?? 0) - (a.contextLength ?? 0);
+    if (contextDelta !== 0) {
+      return contextDelta;
+    }
+    const dimensionDelta = (b.dimensions ?? 0) - (a.dimensions ?? 0);
+    if (dimensionDelta !== 0) {
+      return dimensionDelta;
+    }
+    return compareCodepoints(a.name, b.name);
+  });
+  const model = ranked[0] as EmbeddingModelInfoV1;
+  return {
+    model,
+    reason:
+      models.length === 1
+        ? "the provider's only reported embedding model"
+        : `largest context window among ${String(models.length)} reported embedding models`,
+  };
+}
+
+export interface ProbeInput {
+  http: HttpProbePort;
+  limits: InitLimitsV1;
+  /** `UNTANGLE_OLLAMA_URL` when set. */
+  ollamaUrl: string;
+  /** `UNTANGLE_QDRANT_URL` when set — probed BEFORE the managed default. */
+  qdrantUrls: readonly string[];
+  /** Model name from `--embedding-model`, when given. */
+  preferredModel?: string;
+}
+
+/**
+ * WP-228: Erreichbarkeit heisst NICHT „benutzbar".
+ *
+ * Bis hierher fragte `init` nur `/api/version` und `/api/tags` — zwei
+ * Endpunkte, die der Indexlauf nie anfasst. Auf dem Rechner, an dem das
+ * gemessen wurde, antworteten beide sofort, waehrend `/api/embed` haengte:
+ * die geschriebene Konfiguration versprach einen Dienst, der keinen einzigen
+ * Chunk einbetten konnte. Deshalb wird jetzt der Endpunkt geprueft, den das
+ * Produkt WIRKLICH benutzt — mit kurzer Frist, und ein Fristablauf ist ein
+ * Befund, keine bestandene Pruefung.
+ */
+export async function probeOllama(input: ProbeInput): Promise<OllamaFindingsV1> {
+  const { http, limits, ollamaUrl } = input;
+  const notProbed = (detail: string): OllamaFindingsV1["embedProbe"] => ({
+    outcome: "not-probed",
+    timeoutMs: limits.embedProbeTimeoutMs,
+    detail,
+  });
+
+  const version = await http.getJson(`${trimSlash(ollamaUrl)}/api/version`, limits.probeTimeoutMs);
+  if (version === null) {
+    return {
+      outcome: "unreachable",
+      url: ollamaUrl,
+      embeddingModels: [],
+      embedProbe: notProbed("the provider was already unreachable"),
+    };
+  }
+  const versionString = readString(version, "version");
+  const tags = await http.getJson(`${trimSlash(ollamaUrl)}/api/tags`, limits.probeTimeoutMs);
+  const embeddingModels = readEmbeddingModels(tags);
+  const choice = chooseEmbeddingModel(embeddingModels, input.preferredModel);
+
+  const embedProbe =
+    choice === undefined
+      ? notProbed("the provider reports no embedding model")
+      : await runEmbedProbe(http, ollamaUrl, choice.model.name, limits.embedProbeTimeoutMs);
+
+  return {
+    outcome: "reachable",
+    url: ollamaUrl,
+    ...(versionString === undefined ? {} : { version: versionString }),
+    embeddingModels,
+    ...(choice === undefined ? {} : { chosenModel: choice.model, chosenReason: choice.reason }),
+    embedProbe,
+  };
+}
+
+/** Ein winziges Probe-Embedding, mit Frist. Wirft nie — es antwortet. */
+async function runEmbedProbe(
+  http: HttpProbePort,
+  ollamaUrl: string,
+  model: string,
+  timeoutMs: number,
+): Promise<OllamaFindingsV1["embedProbe"]> {
+  const result = await http.probeEmbedding(trimSlash(ollamaUrl), model, timeoutMs);
+  if (result.ok) {
+    return { outcome: "ok", timeoutMs, dimensions: result.dimensions };
+  }
+  return {
+    outcome: result.timedOut ? "timeout" : "failed",
+    timeoutMs,
+    detail: result.detail,
+  };
+}
+
+export async function probeQdrant(input: ProbeInput): Promise<QdrantFindingsV1> {
+  const { http, limits, qdrantUrls } = input;
+  const probedUrls = [...qdrantUrls];
+  // The skipped list — with its reason — comes from the shared policy, so it
+  // can never claim something other than what the policy actually enforces.
+  const skippedUrls = qdrantProbePlan({}).skipped.filter(
+    (candidate) => !probedUrls.includes(candidate.url),
+  );
+
+  for (const url of probedUrls) {
+    const payload = await http.getJson(`${trimSlash(url)}/`, limits.probeTimeoutMs);
+    if (payload === null) {
+      continue;
+    }
+    const version = readString(payload, "version");
+    return {
+      outcome: "reachable",
+      probedUrls,
+      url,
+      ...(version === undefined ? {} : { version }),
+      skippedUrls,
+    };
+  }
+  return { outcome: "unreachable", probedUrls, skippedUrls };
+}
+
+function trimSlash(value: string): string {
+  return value.endsWith("/") ? value.slice(0, -1) : value;
+}
+
+function readString(payload: unknown, key: string): string | undefined {
+  if (typeof payload !== "object" || payload === null) {
+    return undefined;
+  }
+  const value = (payload as Record<string, unknown>)[key];
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+
+function positiveInteger(value: unknown): number | undefined {
+  return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : undefined;
+}

package/src/commands/init/storage.ts

@@ -0,0 +1,136 @@
+import path from "node:path";
+
+import {
+  DEFAULT_STORAGE_CONFIG,
+  buildEnvConfigLayer,
+  mergeConfigLayers,
+  parseConfigText,
+  resolveUserConfigDir,
+  type RawConfigLayer,
+} from "@untangle-code/config";
+import type { FileSystemPort } from "@untangle-code/domain";
+
+import { OperationalError } from "../../cli/exit.js";
+
+import type { StorageFindingsV1 } from "./types.js";
+
+/** Explicit selection wins; an explicitly configured Qdrant remains opt-in. */
+export function initStorageBackend(
+  explicit: StorageFindingsV1["backend"] | undefined,
+  env: Readonly<Record<string, string | undefined>>,
+  existing: RawConfigLayer = {},
+): StorageFindingsV1["backend"] {
+  const selected = explicit ?? env["UNTANGLE_STORAGE_BACKEND"] ?? existing.storage?.backend;
+  if (selected !== undefined) {
+    if (selected === "lancedb" || selected === "qdrant") return selected;
+    throw new OperationalError(
+      `Unknown storage backend ${JSON.stringify(selected)}.`,
+      "Select --storage=lancedb or --storage=qdrant.",
+    );
+  }
+  return existing.qdrant !== undefined ||
+    buildEnvConfigLayer(env as NodeJS.ProcessEnv).layer.qdrant !== undefined
+    ? "qdrant"
+    : "lancedb";
+}
+
+/** Inspect a local directory without opening a database or creating files. */
+export async function inspectInitStorage(
+  fs: FileSystemPort,
+  root: string,
+  explicit: StorageFindingsV1["backend"] | undefined,
+  env: Readonly<Record<string, string | undefined>>,
+  signal?: AbortSignal,
+): Promise<StorageFindingsV1> {
+  signal?.throwIfAborted();
+  const { layer: existing, warnings } = await existingStorageConfig(fs, root, env, signal);
+  const backend = initStorageBackend(explicit, env, existing);
+  if (backend === "qdrant") {
+    const mode = env["UNTANGLE_QDRANT_MODE"] ?? existing.qdrant?.mode;
+    const url = env["UNTANGLE_QDRANT_URL"] ?? existing.qdrant?.url;
+    return {
+      backend,
+      ...(mode === "managed" || mode === "external" ? { qdrantMode: mode } : {}),
+      ...(url === undefined ? {} : { qdrantUrl: url }),
+      ...(warnings.length === 0 ? {} : { configurationWarnings: warnings }),
+    };
+  }
+  const localPath =
+    env["UNTANGLE_LANCEDB_PATH"] ?? existing.storage?.path ?? DEFAULT_STORAGE_CONFIG.path;
+  if (!localPath)
+    throw new OperationalError("The local storage path is empty.", "Set UNTANGLE_LANCEDB_PATH.");
+  const directory = path.resolve(root, localPath);
+  signal?.throwIfAborted();
+  try {
+    const entry = await fs.stat(directory);
+    signal?.throwIfAborted();
+    return {
+      backend,
+      path: localPath,
+      directory,
+      directoryStatus:
+        entry === null ? "missing" : entry.kind === "directory" ? "present" : "not-directory",
+      ...(warnings.length === 0 ? {} : { configurationWarnings: warnings }),
+    };
+  } catch (error) {
+    signal?.throwIfAborted();
+    return {
+      backend,
+      path: localPath,
+      directory,
+      directoryStatus: "unreadable",
+      detail: error instanceof Error ? error.message : String(error),
+      ...(warnings.length === 0 ? {} : { configurationWarnings: warnings }),
+    };
+  }
+}
+
+/** Read the same project layer formats/order as config loading, through init's filesystem port. */
+async function existingStorageConfig(
+  fs: FileSystemPort,
+  root: string,
+  env: Readonly<Record<string, string | undefined>>,
+  signal?: AbortSignal,
+): Promise<{ layer: RawConfigLayer; warnings: string[] }> {
+  signal?.throwIfAborted();
+  const json = path.join(root, ".untangle.json");
+  const toml = path.join(root, "untangle.toml");
+  const legacyExists = (await fs.stat(json)) !== null || (await fs.stat(toml)) !== null;
+  const user = path.join(
+    resolveUserConfigDir(env as NodeJS.ProcessEnv, process.platform),
+    "config.json",
+  );
+  const paths = [
+    { path: user, format: "json" as const },
+    { path: toml, format: "toml" as const },
+    {
+      path: legacyExists ? json : path.join(root, ".untangle/config.json"),
+      format: "json" as const,
+    },
+    { path: path.join(root, ".untangle.local.json"), format: "json" as const },
+  ];
+  const layers: RawConfigLayer[] = [];
+  const warnings: string[] = [];
+  let legacyQdrant = false;
+  for (const file of paths) {
+    signal?.throwIfAborted();
+    try {
+      if ((await fs.stat(file.path))?.kind !== "file") continue;
+      const layer = parseConfigText(await fs.readTextFile(file.path), file.format);
+      if (layer.qdrant !== undefined) legacyQdrant = true;
+      layers.push(layer);
+    } catch {
+      signal?.throwIfAborted();
+      warnings.push(
+        `Storage selection could not be inspected in ${file.path}; existing configuration is preserved unless --force is selected.`,
+      );
+    }
+  }
+  return {
+    layer: mergeConfigLayers<RawConfigLayer>([
+      ...(legacyQdrant ? [{ storage: { backend: "qdrant" } }] : []),
+      ...layers,
+    ]),
+    warnings,
+  };
+}

package/src/commands/init/toml.ts

@@ -0,0 +1,174 @@
+/**
+ * A small, purpose-built TOML emitter.
+ *
+ * WHY NOT A LIBRARY: the point of the file `init` writes is the COMMENTS —
+ * the user must be able to see what was decided and why. Every general TOML
+ * serializer takes a value and drops the prose, because prose is not part of
+ * the value. So the emitter here is a line builder that happens to know how
+ * to spell TOML scalars, not a serializer of an object graph.
+ *
+ * It is deliberately narrow: tables, array-of-tables, scalars and homogeneous
+ * scalar arrays. Anything outside that throws rather than guessing — a config
+ * file that silently loses a value would be worse than no config file at all.
+ */
+
+export type TomlScalar = string | number | boolean;
+
+const BARE_KEY = /^[A-Za-z0-9_-]+$/;
+
+/**
+ * TOML basic-string escaping. `\` and `"` plus the control characters TOML
+ * names explicitly; everything else below U+0020 goes to `\uXXXX` because a
+ * raw control byte in a config file is unreadable and, in some cases,
+ * unparseable.
+ */
+export function escapeTomlString(value: string): string {
+  let out = "";
+  for (const char of value) {
+    switch (char) {
+      case "\\":
+        out += "\\\\";
+        break;
+      case '"':
+        out += '\\"';
+        break;
+      case "\b":
+        out += "\\b";
+        break;
+      case "\t":
+        out += "\\t";
+        break;
+      case "\n":
+        out += "\\n";
+        break;
+      case "\f":
+        out += "\\f";
+        break;
+      case "\r":
+        out += "\\r";
+        break;
+      default: {
+        const code = char.codePointAt(0) ?? 0;
+        if (code < 0x20 || code === 0x7f) {
+          out += `\\u${code.toString(16).padStart(4, "0").toUpperCase()}`;
+        } else {
+          out += char;
+        }
+      }
+    }
+  }
+  return out;
+}
+
+/** A key is written bare when it can be, quoted when it must be. */
+export function formatTomlKey(key: string): string {
+  return BARE_KEY.test(key) ? key : `"${escapeTomlString(key)}"`;
+}
+
+/**
+ * Formats a number the way TOML distinguishes integers from floats: an
+ * integral value stays integral, a fractional one keeps its decimal point.
+ * Non-finite numbers are rejected — TOML has `inf`/`nan` literals, but a
+ * config value that is either is a bug upstream, not something to serialize.
+ */
+export function formatTomlNumber(value: number): string {
+  if (!Number.isFinite(value)) {
+    throw new Error(`cannot serialize non-finite number: ${String(value)}`);
+  }
+  if (Number.isInteger(value)) {
+    return value.toFixed(0);
+  }
+  return String(value);
+}
+
+export function formatTomlScalar(value: TomlScalar): string {
+  if (typeof value === "string") {
+    return `"${escapeTomlString(value)}"`;
+  }
+  if (typeof value === "boolean") {
+    return value ? "true" : "false";
+  }
+  return formatTomlNumber(value);
+}
+
+/**
+ * A line builder for commented TOML.
+ *
+ * Order is the order of the calls: this class never reorders anything, which
+ * is what lets the caller interleave prose and values so that each decision
+ * sits directly under its justification.
+ */
+export class TomlWriter {
+  private readonly lines: string[] = [];
+
+  /** A `#` comment. An empty string yields a bare `#`, which reads as a spacer. */
+  public comment(text: string): this {
+    this.lines.push(text.length === 0 ? "#" : `# ${text}`);
+    return this;
+  }
+
+  /** Several comment lines at once; convenience for wrapped prose. */
+  public comments(texts: readonly string[]): this {
+    for (const text of texts) {
+      this.comment(text);
+    }
+    return this;
+  }
+
+  public blank(): this {
+    this.lines.push("");
+    return this;
+  }
+
+  /** A `[table.path]` header. */
+  public table(path: readonly string[]): this {
+    this.lines.push(`[${path.map(formatTomlKey).join(".")}]`);
+    return this;
+  }
+
+  /** A `[[table.path]]` array-of-tables header. */
+  public arrayTable(path: readonly string[]): this {
+    this.lines.push(`[[${path.map(formatTomlKey).join(".")}]]`);
+    return this;
+  }
+
+  public scalar(key: string, value: TomlScalar): this {
+    this.lines.push(`${formatTomlKey(key)} = ${formatTomlScalar(value)}`);
+    return this;
+  }
+
+  /**
+   * A single-line array. Written multi-line once it would be long, because a
+   * 200-column exclude list is not something a human edits.
+   */
+  public array(key: string, values: readonly TomlScalar[]): this {
+    const rendered = values.map(formatTomlScalar);
+    const inline = `${formatTomlKey(key)} = [${rendered.join(", ")}]`;
+    if (rendered.length === 0 || inline.length <= 96) {
+      this.lines.push(inline);
+      return this;
+    }
+    this.lines.push(`${formatTomlKey(key)} = [`);
+    for (const item of rendered) {
+      this.lines.push(`  ${item},`);
+    }
+    this.lines.push("]");
+    return this;
+  }
+
+  /** An empty inline table — the honest way to write "nothing configured here". */
+  public emptyInlineTable(key: string): this {
+    this.lines.push(`${formatTomlKey(key)} = {}`);
+    return this;
+  }
+
+  /**
+   * The finished document. Always ends in exactly one newline: a trailing
+   * newline is what every line-oriented tool expects, and "exactly one" is
+   * what keeps the idempotency comparison from reporting phantom changes.
+   */
+  public render(): string {
+    const body = this.lines.join("\n").replace(/\n+$/u, "");
+    return `${body}\n`;
+  }
+}