@lukoweb/apitogo 0.1.147 → 0.2.1

package/src/config/v2/layout.ts

@@ -0,0 +1,237 @@
+import type { Dirent } from "node:fs";
+import { mkdir, readdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import type { StatePaths } from "./paths.js";
+import type { Layout } from "./schema.js";
+
+export type LayoutDetection = { layout: Layout; reason: string };
+
+/** Folders never searched for source files or docs: dependencies, build output, VCS and our own folders. */
+const IGNORED_DIRS = new Set([
+  "node_modules",
+  ".git",
+  "dist",
+  "build",
+  "bin",
+  "obj",
+  "out",
+  "coverage",
+  ".next",
+  ".turbo",
+  ".cache",
+  ".apitogo",
+  "apitogo",
+  "vendor",
+  "__pycache__",
+  ".venv",
+  "venv",
+]);
+
+/** Manifests that identify a project (Reference, section 3.5). */
+const PROJECT_MANIFESTS = new Set([
+  "package.json",
+  "go.mod",
+  "pyproject.toml",
+  "requirements.txt",
+  "Pipfile",
+  "pom.xml",
+  "build.gradle",
+  "build.gradle.kts",
+  "Gemfile",
+  "composer.json",
+  "Cargo.toml",
+  "mix.exs",
+  "deno.json",
+]);
+const PROJECT_MANIFEST_SUFFIXES = [".csproj", ".fsproj", ".sln"];
+
+const SOURCE_EXTENSIONS = new Set([
+  ".ts",
+  ".tsx",
+  ".js",
+  ".jsx",
+  ".mjs",
+  ".cjs",
+  ".py",
+  ".cs",
+  ".go",
+  ".java",
+  ".kt",
+  ".rb",
+  ".php",
+  ".rs",
+  ".ex",
+  ".swift",
+]);
+
+/** How deep, and how many entries, the source-file search may look. Detection is a hint, not a crawl. */
+const SOURCE_SEARCH_DEPTH = 3;
+const SOURCE_SEARCH_BUDGET = 3000;
+
+async function readEntries(dir: string): Promise<Dirent[]> {
+  try {
+    return await readdir(dir, { withFileTypes: true });
+  } catch {
+    return [];
+  }
+}
+
+const toPosix = (value: string) => value.split(path.sep).join("/");
+
+async function findSourceFile(root: string): Promise<string | undefined> {
+  let budget = SOURCE_SEARCH_BUDGET;
+
+  async function search(
+    dir: string,
+    depth: number,
+  ): Promise<string | undefined> {
+    const entries = await readEntries(dir);
+    for (const entry of entries) {
+      if (budget-- <= 0) return undefined;
+      if (entry.isFile() && SOURCE_EXTENSIONS.has(path.extname(entry.name))) {
+        return toPosix(path.relative(root, path.join(dir, entry.name)));
+      }
+    }
+    if (depth >= SOURCE_SEARCH_DEPTH) return undefined;
+    for (const entry of entries) {
+      if (!entry.isDirectory() || IGNORED_DIRS.has(entry.name)) continue;
+      const found = await search(path.join(dir, entry.name), depth + 1);
+      if (found) return found;
+    }
+    return undefined;
+  }
+
+  return search(root, 0);
+}
+
+/**
+ * Chooses the layout for `init` (Reference, section 3.5): an empty directory, or one with only VCS files, trivial
+ * files and documentation, is `standalone`; a recognisable project (a manifest or source files) is `embedded`.
+ */
+export async function detectLayout(dir: string): Promise<LayoutDetection> {
+  const entries = await readEntries(dir);
+  if (entries.length === 0) {
+    return { layout: "standalone", reason: "the directory is empty" };
+  }
+
+  const manifest = entries.find(
+    (entry) =>
+      entry.isFile() &&
+      (PROJECT_MANIFESTS.has(entry.name) ||
+        PROJECT_MANIFEST_SUFFIXES.some((suffix) =>
+          entry.name.endsWith(suffix),
+        )),
+  );
+  if (manifest) {
+    return {
+      layout: "embedded",
+      reason: `it contains a project (${manifest.name})`,
+    };
+  }
+
+  const source = await findSourceFile(dir);
+  if (source) {
+    return {
+      layout: "embedded",
+      reason: `it contains source code (${source})`,
+    };
+  }
+
+  return {
+    layout: "standalone",
+    reason: "it holds only documentation and trivial files, no project",
+  };
+}
+
+export type DocsCandidate = {
+  kind: "folder" | "file";
+  /** Path relative to the project directory, POSIX separators. */
+  path: string;
+  /** What goes into `portal.docs.content`: the docs are referenced in place, never moved or copied. */
+  glob: string;
+  fileCount: number;
+};
+
+const DOCS_FOLDER_NAMES = new Set(["docs", "doc", "documentation", "pages"]);
+const DOCS_EXTENSIONS = new Set([".md", ".mdx"]);
+/** Root markdown files that are project metadata, not documentation. */
+const NOT_DOCS = /^(readme|license|licence|changelog)(\.|$)/i;
+const DOCS_SEARCH_DEPTH = 2;
+const OWN_PAGES_FOLDER = "portal/pages";
+
+async function countDocs(dir: string): Promise<number> {
+  let count = 0;
+  for (const entry of await readEntries(dir)) {
+    if (entry.isDirectory()) {
+      if (!IGNORED_DIRS.has(entry.name))
+        count += await countDocs(path.join(dir, entry.name));
+    } else if (
+      entry.isFile() &&
+      DOCS_EXTENSIONS.has(path.extname(entry.name))
+    ) {
+      count += 1;
+    }
+  }
+  return count;
+}
+
+/**
+ * Finds existing documentation: folders named `docs`, `doc`, `documentation` or `pages` (at the top or one level
+ * down) that hold `.md` or `.mdx` files, and loose markdown files at the top that are not README, LICENSE or
+ * CHANGELOG. Nothing is moved: each candidate becomes a glob.
+ */
+export async function detectExistingDocs(
+  dir: string,
+): Promise<DocsCandidate[]> {
+  const candidates: DocsCandidate[] = [];
+
+  async function scan(current: string, depth: number) {
+    for (const entry of await readEntries(current)) {
+      const relative = toPosix(
+        path.relative(dir, path.join(current, entry.name)),
+      );
+      if (entry.isDirectory()) {
+        if (IGNORED_DIRS.has(entry.name)) continue;
+        if (relative === OWN_PAGES_FOLDER) continue; // the portal's own default pages folder is already covered by the defaults
+        if (DOCS_FOLDER_NAMES.has(entry.name.toLowerCase())) {
+          const fileCount = await countDocs(path.join(current, entry.name));
+          if (fileCount > 0) {
+            candidates.push({
+              kind: "folder",
+              path: relative,
+              glob: `${relative}/**/*.{md,mdx}`,
+              fileCount,
+            });
+          }
+        } else if (depth < DOCS_SEARCH_DEPTH - 1) {
+          await scan(path.join(current, entry.name), depth + 1);
+        }
+      } else if (
+        depth === 0 &&
+        entry.isFile() &&
+        DOCS_EXTENSIONS.has(path.extname(entry.name)) &&
+        !NOT_DOCS.test(entry.name)
+      ) {
+        candidates.push({
+          kind: "file",
+          path: relative,
+          glob: relative,
+          fileCount: 1,
+        });
+      }
+    }
+  }
+
+  await scan(dir, 0);
+  return candidates.sort((a, b) => a.path.localeCompare(b.path));
+}
+
+/**
+ * Creates the generated-state directory: itself, its own `.gitignore` containing `*`, and the `generated/` and
+ * `build/` folders. Never reads or edits the tenant's root `.gitignore`. Safe to call again.
+ */
+export async function ensureStateDir(state: StatePaths): Promise<void> {
+  await mkdir(state.generatedDir, { recursive: true });
+  await mkdir(state.buildDir, { recursive: true });
+  await writeFile(state.gitignore, "*\n");
+}

package/src/config/v2/ledger.ts

@@ -0,0 +1,219 @@
+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { ApitogoError } from "../../lib/util/invariant.js";
+import type { StatePaths } from "./paths.js";
+
+/** Version of the `state.json` format this build reads and writes. A newer file is refused, never overwritten. */
+export const LEDGER_VERSION = 1;
+
+export type StepStatus = "running" | "succeeded" | "failed";
+
+/** One step of one service, for example `portal` / `build`. */
+export type StepRecord = {
+  status: StepStatus;
+  /** Hash of the inputs the step ran with. An equal hash means the result is still valid. */
+  hash?: string;
+  startedAt: string;
+  finishedAt?: string;
+  /** Why the step failed. */
+  error?: string;
+  /** Public facts the step produced, such as a deployed URL. */
+  output?: Record<string, string>;
+};
+
+export type Ledger = {
+  version: typeof LEDGER_VERSION;
+  services: Record<string, { steps: Record<string, StepRecord> }>;
+};
+
+export class LedgerError extends ApitogoError {
+  constructor(
+    public readonly code: "LEDGER_CORRUPT" | "LEDGER_VERSION_UNSUPPORTED",
+    message: string,
+    developerHint?: string,
+  ) {
+    super(message, { developerHint, title: "Cannot read the project state" });
+    this.name = "LedgerError";
+  }
+}
+
+export const emptyLedger = (): Ledger => ({
+  version: LEDGER_VERSION,
+  services: {},
+});
+
+/** Reads `state.json`. A missing file is an empty ledger. A corrupt or newer file raises a {@link LedgerError}. */
+export async function readLedger(
+  state: Pick<StatePaths, "file">,
+): Promise<Ledger> {
+  let text: string;
+  try {
+    text = await readFile(state.file, "utf8");
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT")
+      return emptyLedger();
+    throw error;
+  }
+
+  let raw: unknown;
+  try {
+    raw = JSON.parse(text);
+  } catch {
+    throw corrupt(state.file, "is not valid JSON");
+  }
+  if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
+    throw corrupt(state.file, "is not an object");
+  }
+  const { version, services } = raw as Record<string, unknown>;
+  if (typeof version !== "number") throw corrupt(state.file, "has no version");
+  if (version > LEDGER_VERSION) {
+    throw new LedgerError(
+      "LEDGER_VERSION_UNSUPPORTED",
+      `${state.file} has version ${version} but this CLI supports ${LEDGER_VERSION}.`,
+      "Upgrade @lukoweb/apitogo.",
+    );
+  }
+  if (version !== LEDGER_VERSION)
+    throw corrupt(state.file, `has unknown version ${version}`);
+  if (
+    typeof services !== "object" ||
+    services === null ||
+    Array.isArray(services)
+  ) {
+    throw corrupt(state.file, "has no services object");
+  }
+  return { version: LEDGER_VERSION, services: services as Ledger["services"] };
+}
+
+function corrupt(file: string, why: string): LedgerError {
+  return new LedgerError(
+    "LEDGER_CORRUPT",
+    `${file} ${why}.`,
+    "It only holds regenerable state: delete it and run the command again.",
+  );
+}
+
+/** Replaces `state.json` atomically, so a crash never leaves a half-written file. */
+export async function writeLedger(
+  state: Pick<StatePaths, "file">,
+  ledger: Ledger,
+): Promise<void> {
+  await mkdir(path.dirname(state.file), { recursive: true });
+  const temporary = `${state.file}.${process.pid}.tmp`;
+  try {
+    await writeFile(temporary, `${JSON.stringify(ledger, null, 2)}\n`, "utf8");
+    await rename(temporary, state.file);
+  } catch (error) {
+    await rm(temporary, { force: true });
+    throw error;
+  }
+}
+
+export function stepOf(
+  ledger: Ledger,
+  service: string,
+  step: string,
+): StepRecord | undefined {
+  return ledger.services[service]?.steps[step];
+}
+
+function withStep(
+  ledger: Ledger,
+  service: string,
+  step: string,
+  record: StepRecord,
+): Ledger {
+  return {
+    ...ledger,
+    services: {
+      ...ledger.services,
+      [service]: {
+        steps: { ...ledger.services[service]?.steps, [step]: record },
+      },
+    },
+  };
+}
+
+export type PipelineStep = {
+  name: string;
+  /** Hash of this step's inputs. */
+  hash: () => string | Promise<string>;
+  run: () => Promise<Record<string, string> | undefined | void>;
+  /** Extra proof the previous result still exists (for example the build folder). Defaults to true. */
+  stillValid?: () => Promise<boolean>;
+};
+
+export type StepOutcome = {
+  step: string;
+  skipped: boolean;
+  output?: Record<string, string>;
+};
+
+/**
+ * Runs a service's steps in order against the ledger. A step is skipped when it already succeeded with the same
+ * hash and nothing before it ran again. A failure is recorded with its message and rethrown; the next run starts
+ * at that step, because the steps before it succeeded and are skipped.
+ */
+export async function runPipeline(
+  state: Pick<StatePaths, "file">,
+  service: string,
+  steps: readonly PipelineStep[],
+  options: { force?: boolean; now?: () => Date } = {},
+): Promise<StepOutcome[]> {
+  const now = options.now ?? (() => new Date());
+  let ledger = await readLedger(state);
+  const outcomes: StepOutcome[] = [];
+  let upstreamRan = false;
+
+  for (const step of steps) {
+    const hash = await step.hash();
+    const previous = stepOf(ledger, service, step.name);
+    const reusable =
+      !options.force &&
+      !upstreamRan &&
+      previous?.status === "succeeded" &&
+      previous.hash === hash &&
+      (step.stillValid ? await step.stillValid() : true);
+
+    if (reusable) {
+      outcomes.push({
+        step: step.name,
+        skipped: true,
+        output: previous?.output,
+      });
+      continue;
+    }
+
+    upstreamRan = true;
+    const startedAt = now().toISOString();
+    ledger = withStep(ledger, service, step.name, {
+      status: "running",
+      hash,
+      startedAt,
+    });
+    await writeLedger(state, ledger);
+    try {
+      const output = (await step.run()) || undefined;
+      ledger = withStep(ledger, service, step.name, {
+        status: "succeeded",
+        hash,
+        startedAt,
+        finishedAt: now().toISOString(),
+        ...(output ? { output } : {}),
+      });
+      await writeLedger(state, ledger);
+      outcomes.push({ step: step.name, skipped: false, output });
+    } catch (error) {
+      ledger = withStep(ledger, service, step.name, {
+        status: "failed",
+        hash,
+        startedAt,
+        finishedAt: now().toISOString(),
+        error: error instanceof Error ? error.message : String(error),
+      });
+      await writeLedger(state, ledger);
+      throw error;
+    }
+  }
+  return outcomes;
+}

package/src/config/v2/load.ts

@@ -0,0 +1,70 @@
+import { existsSync, readFileSync, statSync } from "node:fs";
+import path from "node:path";
+import { APITOGO_CONFIG_FILENAME } from "../apitogo-config-core.js";
+import { ConfigError } from "./errors.js";
+import { parseConfigV2 } from "./parse.js";
+import { type ResolvedConfigV2, resolveConfigV2 } from "./resolve.js";
+import type { ConfigV2 } from "./schema.js";
+
+export type LoadedConfigV2 = {
+  config: ConfigV2;
+  resolved: ResolvedConfigV2;
+};
+
+/** `JSON.parse` failures carry a position in different shapes across Node versions; turn it into line and column. */
+function describeJsonError(error: unknown, text: string): string {
+  const message = error instanceof Error ? error.message : String(error);
+  const lineColumn = /line (\d+) column (\d+)/.exec(message);
+  if (lineColumn) return `${message}`;
+
+  const position = /position (\d+)/.exec(message);
+  if (!position) return message;
+
+  const offset = Number(position[1]);
+  const before = text.slice(0, offset);
+  const line = before.split("\n").length;
+  const column = offset - before.lastIndexOf("\n");
+  return `${message} (line ${line} column ${column})`;
+}
+
+/**
+ * Reads, validates and resolves a v2 config. `pathOrDir` is either the config file or the directory that
+ * contains `apitogo.config.json`. Invalid JSON, unknown keys, inline secrets and every other schema problem
+ * raise a {@link ConfigError}; nothing is silently dropped.
+ */
+export function loadConfigV2(pathOrDir: string): LoadedConfigV2 {
+  const resolvedPath = path.resolve(pathOrDir);
+  const configPath =
+    existsSync(resolvedPath) && statSync(resolvedPath).isDirectory()
+      ? path.join(resolvedPath, APITOGO_CONFIG_FILENAME)
+      : resolvedPath;
+
+  if (!existsSync(configPath)) {
+    throw new ConfigError(path.basename(configPath), [
+      {
+        code: "CONFIG_NOT_FOUND",
+        path: "",
+        message: `No config file at ${configPath}.`,
+        hint: "Run `apitogo init` to create one.",
+      },
+    ]);
+  }
+
+  const text = readFileSync(configPath, "utf8");
+  let raw: unknown;
+  try {
+    raw = JSON.parse(text);
+  } catch (error) {
+    throw new ConfigError(path.basename(configPath), [
+      {
+        code: "CONFIG_INVALID_JSON",
+        path: "",
+        message: describeJsonError(error, text),
+        hint: "Fix the JSON syntax; comments and trailing commas are not allowed.",
+      },
+    ]);
+  }
+
+  const config = parseConfigV2(raw, path.basename(configPath));
+  return { config, resolved: resolveConfigV2(config, configPath) };
+}