create-ailk 0.30.0 → 0.32.0

package/dist/export-content.js

@@ -0,0 +1,303 @@
+/**
+ * `create-ailk export-content [<dir>] [--dry-run]` — write every published
+ * page back to its content file, sections recombined (#10648, spec
+ * docs/specs/10648-the-cutover-kit-frozen-inventory-spec.md D4–D6).
+ *
+ * The rollback that keeps published edits (roadmap D15): `git revert` would
+ * lose every edit made in the Admin since the cutover; this writes the
+ * workspace's published pages back as the files the importer reads, so a
+ * re-run of `migrate-content` reports every page `unchanged`.
+ *
+ * Reads, cross-checked (D4): only rows the workspace list reports `published`
+ * are exported, so the draft tip IS the published version and its raw content
+ * is exactly what the importer compares against; the public read supplies the
+ * published `sections` and a `publishedVersionId` that must equal the row's.
+ * A page with a draft ahead (`published_with_draft`) fails by name: its
+ * published version cannot be read with an agent key, and writing the draft
+ * would put unpublished words on the live site at rollback.
+ *
+ * Recombination (D5) is the exact inverse of the importer's conversion: the
+ * words go back into the sections' data through `applyWords`, and the words
+ * run the importer appended (`sectionsToMarkdown`, after a blank line) comes
+ * off the body. The file is `key: JSON.stringify(value)` frontmatter (JSON is
+ * YAML flow; create-ailk has no YAML dependency) and the body unchanged.
+ *
+ * Pure module: no `import.meta.url`, so jest imports it directly.
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { parseArgs } from "node:util";
+import { listWorkspacePages, loadSiteWordsCodec, readDraftPage, readPublishedPage, readSiteLocales, resolveWorkspaceAccess, } from "./inventory.js";
+import { inlinePins, letterOf, pinnedKind } from "./language-import.js";
+import { ParseError } from "./parse-args.js";
+/** A refusal the CLI exits 2 on, thrown before any request is sent. */
+export class ExportContentRefusedError extends Error {
+    exitCode = 2;
+    constructor(message) {
+        super(message);
+        this.name = "ExportContentRefusedError";
+    }
+}
+/**
+ * The editorial tree's directories and the stored-to-fumadocs key renames —
+ * the inverse of `import-mdx.ts`'s `EDITORIAL_FIELDS` (runtime-source D3,
+ * restated: create-ailk cannot import `apps/web`).
+ */
+const EDITORIAL_DIRS = {
+    BlogList: { dir: "blog", index: true },
+    BlogPosting: { dir: "blog", index: false },
+    DocsList: { dir: "docs", index: true },
+    DocsDetail: { dir: "docs", index: false },
+};
+const EDITORIAL_RENAMES = { publishedAt: "date", updatedAt: "updated" };
+/** Keys an editorial file does not carry: its path decides both. */
+const EDITORIAL_DROPPED = new Set(["type", "slug"]);
+const INDEX_SLUG = "index";
+/** The store's address shapes, restated (create-ailk has no runtime dep): a path built from them stays in the site. */
+const TYPE_SHAPE = /^[A-Z][A-Za-z]*$/;
+const SLUG_SHAPE = /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/;
+const LOCALE_SHAPE = /^[a-z]{2}(-[A-Z]{2})?$/;
+/** A pinned item's read times out as the kit's other reads do (`inventory.ts`). */
+const ITEM_READ_TIMEOUT_MS = 15_000;
+export const SPOKE_HINT = "the site's read does not serve this workspace's published version — is the spoke at --store database?";
+// ── File addressing (D6) ────────────────────────────────────────────────────
+/** The site-relative file a stored page is written to, and whether it is editorial. */
+export function fileFor(targetDir, type, slug, locale) {
+    const exists = (path) => existsSync(join(targetDir, path));
+    const editorial = EDITORIAL_DIRS[type];
+    if (editorial !== undefined && editorial.index === (slug === INDEX_SLUG)) {
+        const base = `apps/web/content/${locale}/${editorial.dir}`;
+        if (editorial.index)
+            return { path: `${base}/index.mdx`, editorial: true };
+        // The importer reads `<slug>.mdx` first and `<slug>/index.mdx` (#10701);
+        // a page is written back to the form it was read from.
+        const flat = `${base}/${slug}.mdx`;
+        const nested = `${base}/${slug}/index.mdx`;
+        return { path: !exists(flat) && exists(nested) ? nested : flat, editorial: true };
+    }
+    const tree = `content/${locale}/${type}/${slug}.mdx`;
+    const legacy = `content/${type}/${slug}.mdx`;
+    return { path: !exists(tree) && exists(legacy) ? legacy : tree, editorial: false };
+}
+// ── Recombination (D5) ──────────────────────────────────────────────────────
+/**
+ * The file whose import yields `content` again with `sections` as its layout,
+ * or why there is none.
+ */
+export function recombine(content, sections, codec, editorial, pinned) {
+    const { body: rawBody, ...rest } = content;
+    let body = typeof rawBody === "string" ? rawBody : "";
+    const frontmatter = {};
+    for (const [key, value] of Object.entries(rest)) {
+        if (editorial && EDITORIAL_DROPPED.has(key))
+            continue;
+        frontmatter[editorial ? (EDITORIAL_RENAMES[key] ?? key) : key] = value;
+    }
+    if (sections !== undefined && sections.length > 0) {
+        const applied = codec.applyWords(body, sections);
+        if (applied.issues.length > 0) {
+            const codes = [...new Set(applied.issues.map((i) => `${i.code} (${i.sectionId})`))].join(", ");
+            return { reason: `the words blocks do not match the published sections: ${codes}` };
+        }
+        let words;
+        try {
+            words = codec.sectionsToMarkdown(applied.sections);
+        }
+        catch (err) {
+            return { reason: `codec: ${err.message}` };
+        }
+        // The importer writes `<lead>\n\n<words>`, or `<words>` alone.
+        const lead = body === words ? "" : body.endsWith(`\n\n${words}`) ? body.slice(0, -words.length - 2) : undefined;
+        if (lead === undefined || lead !== lead.trimEnd()) {
+            return { reason: "words blocks are not where import writes them" };
+        }
+        body = lead;
+        // #10735 D9 — a pinned section is written with its Live items inline.
+        frontmatter["sections"] = pinned === undefined ? applied.sections : inlinePins(applied.sections, pinned);
+    }
+    const lines = Object.entries(frontmatter)
+        .filter(([, value]) => value !== undefined)
+        .map(([key, value]) => `${key}: ${JSON.stringify(value)}`);
+    return { text: `---\n${lines.join("\n")}\n---\n${body}` };
+}
+export async function exportContent(opts) {
+    const targetDir = resolve(opts.targetDir);
+    const env = opts.env ?? process.env;
+    const onLine = opts.onLine ?? (() => undefined);
+    const fetch = opts.fetch ?? globalThis.fetch;
+    const dryRun = opts.dryRun === true;
+    const refuse = (message) => {
+        throw new ExportContentRefusedError(`export-content: ${message} Nothing was written.`);
+    };
+    const access = resolveWorkspaceAccess(targetDir, { apiUrl: opts.apiUrl, workspaceId: opts.workspaceId, env }, refuse);
+    const { defaultLocale: locale } = readSiteLocales(targetDir);
+    const codec = opts.codec ?? (await loadSiteWordsCodec(targetDir, refuse));
+    onLine(dryRun
+        ? `Planning the export of workspace ${access.workspaceId} into ${opts.targetDir}. Nothing is written with --dry-run.`
+        : `Exporting workspace ${access.workspaceId}'s published pages into ${opts.targetDir}.`);
+    const rows = (await listWorkspacePages(access, fetch, locale, refuse))
+        .filter((r) => r.locale === locale && r.status !== "draft")
+        .sort((a, b) => (`${a.type}/${a.slug}` < `${b.type}/${b.slug}` ? -1 : 1));
+    const pages = [];
+    for (const row of rows) {
+        const page = await exportPage(row);
+        pages.push(page);
+        onLine(page.result === "failed"
+            ? `${page.type}/${page.slug}: failed — ${page.reason}`
+            : `${page.file}: ${page.result}`);
+    }
+    const count = (result) => pages.filter((p) => p.result === result).length;
+    onLine(`created ${count("created")} · written ${count("written")} · unchanged ${count("unchanged")} · failed ${count("failed")}`);
+    return { exitCode: count("failed") > 0 ? 1 : 0, pages };
+    async function exportPage(row) {
+        const { type, slug } = row;
+        const failed = (reason, file = null) => ({
+            type,
+            slug,
+            file,
+            result: "failed",
+            reason,
+        });
+        if (row.status === "published_with_draft") {
+            return failed("a draft is ahead of the published version, so the published words cannot be read. Publish or revert the draft in Admin, then re-run.");
+        }
+        if (!TYPE_SHAPE.test(type) || !SLUG_SHAPE.test(slug) || !LOCALE_SHAPE.test(locale)) {
+            return failed("not a page address a file can be written at");
+        }
+        const draft = await readDraftPage(access, fetch, type, slug, locale);
+        if (draft.content === undefined)
+            return failed(`the draft read answered ${draft.status}`);
+        if (draft.version !== row.publishedVersionId) {
+            return failed("the draft read is not the published version; re-run");
+        }
+        const published = await readPublishedPage(access, fetch, type, slug, locale);
+        if (!published.found || published.publishedVersionId !== row.publishedVersionId) {
+            return failed(SPOKE_HINT);
+        }
+        const pinned = await pinnedItems(published.sections ?? [], locale);
+        if (typeof pinned === "string")
+            return failed(pinned);
+        const { path, editorial } = fileFor(targetDir, type, slug, locale);
+        const file = recombine(draft.content, published.sections, codec, editorial, pinned);
+        if ("reason" in file)
+            return failed(file.reason, path);
+        const absolute = join(targetDir, path);
+        const current = existsSync(absolute) ? readFileSync(absolute, "utf8") : undefined;
+        const result = current === undefined ? "created" : current === file.text ? "unchanged" : "written";
+        if (!dryRun && result !== "unchanged") {
+            mkdirSync(dirname(absolute), { recursive: true });
+            writeFileSync(absolute, file.text);
+        }
+        return { type, slug, file: path, result };
+    }
+    /**
+     * The Live words of every item the page's faq / glossary-index sections pin
+     * (#10735 D9), by kind and slug — `undefined` when none pins — or why the
+     * page cannot be written. A missing, off or disabled item has no entry.
+     */
+    async function pinnedItems(sections, locale) {
+        const items = { faq: new Map(), term: new Map() };
+        let any = false;
+        for (const section of sections) {
+            const data = section.data;
+            if (["faq", "glossary-index"].includes(String(section.type)) && data?.["tags"] !== undefined) {
+                return "a tag pull cannot be written to a file; pin its items or remove it, then re-run";
+            }
+            const kind = pinnedKind(section);
+            if (kind === undefined)
+                continue;
+            any = true;
+            for (const pin of data["pinned"]) {
+                const prefix = `${kind}:`;
+                if (typeof pin !== "string" || !pin.startsWith(prefix) || !SLUG_SHAPE.test(pin.slice(prefix.length)))
+                    continue;
+                const itemSlug = pin.slice(prefix.length);
+                if (items[kind].has(itemSlug))
+                    continue;
+                let status = 0;
+                let body;
+                try {
+                    const response = await fetch(`${access.origin}/v1/workspaces/${encodeURIComponent(access.workspaceId)}/language/${kind}/${itemSlug}?locale=${encodeURIComponent(locale)}`, {
+                        headers: { authorization: `Bearer ${access.apiKey}` },
+                        // A redirect would re-send the key to another URL.
+                        redirect: "error",
+                        signal: AbortSignal.timeout(ITEM_READ_TIMEOUT_MS),
+                    });
+                    status = response.status;
+                    body = (await response.json().catch(() => undefined));
+                }
+                catch {
+                    // A timeout, a network failure or a redirect: status 0.
+                }
+                if (status === 404)
+                    continue;
+                if (status !== 200)
+                    return `the ${kind} item read answered ${status}`;
+                const item = body?.data?.item;
+                const words = item?.publishedWords;
+                if (item?.live !== true || words === null || words === undefined)
+                    continue;
+                items[kind].set(itemSlug, kind === "term" ? { ...words, letter: letterOf(words["term"] ?? "", locale) } : words);
+            }
+        }
+        return any ? items : undefined;
+    }
+}
+export const EXPORT_CONTENT_HELP = `create-ailk export-content — write every published page back to its content file.
+
+Usage:
+  AILK_AGENT_KEY=<key> create-ailk export-content [<dir>] [--dry-run] [--api-url <origin>] [--workspace <wksp_id>]
+
+Arguments:
+  <dir>                    The site. Defaults to the current dir.
+
+Options:
+  --dry-run                Print what would be written, and write nothing.
+  --api-url <origin>       The API origin that holds the workspace — on a
+                           spoke, its hub's. Else AILK_API_URL.
+  --workspace <wksp_id>    The workspace. Else AILK_WORKSPACE_ID.
+  -h, --help               Show this help.
+
+The rollback for a cutover: every page the workspace list reports published
+in the default locale is written to the file migrate-content reads it from
+— content/en/<Type>/<slug>.mdx (or the legacy content/<Type>/<slug>.mdx that
+exists), apps/web/content/en/blog/<slug>.mdx for a post and the same under
+docs for a docs page — with its words moved back into its sections, so a
+re-run of migrate-content reports every page unchanged. Published edits are
+kept. A page with a draft ahead of its published version fails by name:
+publish or revert it in Admin, then re-run. The site's own API read must
+serve the workspace's published version (on a spoke, the hub's row at
+--store database). Each page prints created, written, unchanged or failed.
+The agent key needs the content-edit grant; on a spoke, AILK_SPOKE_KEY is
+needed too. Exit 0 when no page failed, 1 when any did, 2 on a refusal.
+`;
+export function parseExportContentArgs(argv) {
+    let parsed;
+    try {
+        parsed = parseArgs({
+            args: argv,
+            strict: true,
+            allowPositionals: true,
+            options: {
+                "dry-run": { type: "boolean", default: false },
+                "api-url": { type: "string" },
+                workspace: { type: "string" },
+                help: { type: "boolean", default: false, short: "h" },
+            },
+        });
+    }
+    catch (err) {
+        throw new ParseError(`export-content: ${err.message}`);
+    }
+    const [target, ...rest] = parsed.positionals;
+    if (rest.length > 0) {
+        throw new ParseError(`export-content takes at most one target directory, got ${parsed.positionals.length} arguments`);
+    }
+    return {
+        target,
+        dryRun: parsed.values["dry-run"] === true,
+        apiUrl: parsed.values["api-url"],
+        workspaceId: parsed.values.workspace,
+        help: parsed.values.help === true,
+    };
+}

package/dist/inventory.d.ts

@@ -0,0 +1,216 @@
+/** The kit's directory in the site: committed, unlike `.ailk/`. */
+export declare const CUTOVER_DIR = ".ailk-cutover";
+export declare const INVENTORY_FILE = "inventory.json";
+export declare const AUDIT_FILE = "audit.json";
+/** The spoke's key for the hub's public read (`x-ailk-spoke-key`). */
+export declare const SPOKE_KEY_ENV_VAR = "AILK_SPOKE_KEY";
+/** A refusal the CLI exits 2 on. */
+export declare class InventoryRefusedError extends Error {
+    readonly exitCode = 2;
+    constructor(message: string);
+}
+export type InventorySource = "sitemap" | "crawl" | "roster" | "probe";
+export interface FrozenEntry {
+    path: string;
+    /** `page` for `text/html`, `file` for anything else. */
+    kind: "page" | "file";
+    locale: string;
+    from: InventorySource[];
+    prose: boolean;
+    images: string[];
+}
+export interface ProseExclusion {
+    path: string;
+    reason: string;
+    by: "inventory" | "operator";
+}
+export interface InventoryFile {
+    version: 1;
+    origin: string;
+    frozenAt: string;
+    defaultLocale: string;
+    docsPath: string;
+    sources: {
+        sitemap: number;
+        crawl: number;
+        roster: number;
+    };
+    frozen: FrozenEntry[];
+    notFrozen: {
+        path: string;
+        status: number;
+        location?: string;
+        from: InventorySource[];
+    }[];
+    images: {
+        url: string;
+        status: number;
+        pages: string[];
+    }[];
+    proseExclusions: ProseExclusion[];
+    /** URL path → `Type/slug`, written by the operator for code-built pages. */
+    joins: Record<string, string>;
+}
+export declare const AUDIT_VERDICTS: readonly ["pass", "excluded", "read-failed", "unjoined", "no-page", "ambiguous", "locale-not-imported", "not-published", "words-not-in-markdown"];
+export type AuditVerdict = (typeof AUDIT_VERDICTS)[number];
+export interface AuditPage {
+    path: string;
+    join: string | null;
+    verdict: AuditVerdict;
+    prose: boolean;
+    published: boolean | null;
+    wordsInMarkdown: boolean | null;
+}
+export interface AuditFile {
+    version: 1;
+    inventoryFrozenAt: string;
+    pages: AuditPage[];
+    /** Workspace pages no frozen URL joins. */
+    orphans: string[];
+    counts: Record<AuditVerdict, number>;
+}
+/** What the kit reads the workspace with. */
+export interface WorkspaceAccess {
+    origin: string;
+    workspaceId: string;
+    apiKey: string;
+    /** Set on a spoke: the hub's public read answers the spoke's workspace with it. */
+    spokeKey?: string;
+}
+/** The words codec's two functions the kit uses, typed structurally. */
+export interface WordsCodec {
+    applyWords(markdown: string, sections: readonly unknown[]): {
+        sections: unknown[];
+        issues: {
+            code: string;
+            sectionId: string;
+        }[];
+    };
+    sectionsToMarkdown(sections: readonly unknown[]): string;
+}
+/** One row of the workspace list, as the editor's view gives it. */
+export interface WorkspaceRow {
+    type: string;
+    slug: string;
+    kind: "page" | "post" | "file";
+    locale: string;
+    status: "draft" | "published" | "published_with_draft";
+    version: string;
+    publishedVersionId: string | null;
+}
+/** A value given, or `undefined` for an absent or empty one. */
+export declare function given(value: string | undefined): string | undefined;
+/**
+ * The key, workspace and API origin, as `migrate-content` reads them, plus the
+ * spoke key on a spoke. Every miss is refused through `refuse`, before any
+ * request.
+ */
+export declare function resolveWorkspaceAccess(targetDir: string, o: {
+    apiUrl?: string | undefined;
+    workspaceId?: string | undefined;
+    env: NodeJS.ProcessEnv;
+}, refuse: (message: string) => never): WorkspaceAccess;
+/** A site origin: http(s), reduced to its origin. */
+export declare function siteOrigin(value: string, flag: string, refuse: (message: string) => never): string;
+/**
+ * The workspace's pages in `locale`, through the editor's view. A row with no
+ * `version` is the member projection — a key without `content-edit` — which
+ * reports every live row `published` and would hide a draft ahead: refused.
+ * @throws Error on a read that does not answer 200.
+ */
+export declare function listWorkspacePages(access: WorkspaceAccess, fetch: typeof globalThis.fetch, locale: string, refuse: (message: string) => never): Promise<WorkspaceRow[]>;
+export type PublishedRead = {
+    found: false;
+    status: number;
+} | {
+    found: true;
+    sections: unknown[] | undefined;
+    body: string;
+    publishedVersionId: string | undefined;
+};
+/** The public read of one page with its body: what the site serves. */
+export declare function readPublishedPage(access: WorkspaceAccess, fetch: typeof globalThis.fetch, type: string, slug: string, locale: string): Promise<PublishedRead>;
+/** The draft tip's raw stored content — the value the importer compares against. */
+export declare function readDraftPage(access: WorkspaceAccess, fetch: typeof globalThis.fetch, type: string, slug: string, locale: string): Promise<{
+    status: number;
+    content?: Record<string, unknown>;
+    version?: string;
+}>;
+/**
+ * The site's words codec: `@working-theory/validation` resolved from its
+ * `apps/web`, the name held in a constant so no import specifier names it.
+ */
+export declare function loadSiteWordsCodec(targetDir: string, refuse: (message: string) => never): Promise<WordsCodec>;
+/** Equal as JSON, keys sorted at every depth. */
+export declare function sameJson(a: unknown, b: unknown): boolean;
+/** The site's locales from `project.yaml` `product`, read without a YAML dependency. */
+export declare function readSiteLocales(targetDir: string): {
+    defaultLocale: string;
+    locales: string[];
+};
+/**
+ * Whether a page carries prose (D3): inside `<main>` when it has one, with its
+ * script, style, noscript, nav, header, footer and form elements removed, a
+ * `<p>`, heading, `<li>` or `<blockquote>` with non-whitespace text.
+ */
+export declare function isProseBearing(html: string): boolean;
+interface RosterRoute {
+    route: string;
+    pageType: string | null;
+}
+type Join = {
+    join: string;
+} | {
+    verdict: "unjoined" | "no-page" | "ambiguous";
+    join?: string;
+};
+/** A URL's workspace page: the operator's join, then the roster, then the editorial prefixes. */
+export declare function joinUrl(rawPath: string, ctx: {
+    joins: Record<string, string>;
+    roster: RosterRoute[];
+    rows: WorkspaceRow[];
+    docsPath: string;
+    defaultLocale: string;
+}): Join;
+export interface InventoryOptions {
+    targetDir: string;
+    /** `--origin`: the site as served today. Needed only to freeze. */
+    origin?: string | undefined;
+    /** `--refreeze`: crawl again, keeping the operator's joins and exclusions. */
+    refreeze?: boolean;
+    /** `--docs-path`; defaults to `/docs`. */
+    docsPath?: string | undefined;
+    apiUrl?: string | undefined;
+    workspaceId?: string | undefined;
+    env?: NodeJS.ProcessEnv;
+    fetch?: typeof globalThis.fetch;
+    /** Injectable for tests; defaults to the site's installed codec. */
+    codec?: WordsCodec;
+    /** Injectable for tests: the freeze's one wall-clock value. */
+    now?: () => Date;
+    onLine?: (line: string) => void;
+}
+export interface InventoryResult {
+    exitCode: 0 | 1;
+    inventory: InventoryFile;
+    audit: AuditFile;
+}
+/**
+ * The inventory in the site, or `undefined` when none is frozen.
+ * @throws Error on a file that is not JSON or not a version-1 inventory.
+ */
+export declare function readInventory(targetDir: string): InventoryFile | undefined;
+export declare function formatAuditCounts(counts: Record<AuditVerdict, number>): string;
+export declare function inventory(opts: InventoryOptions): Promise<InventoryResult>;
+export interface InventoryArgs {
+    target: string | undefined;
+    origin: string | undefined;
+    refreeze: boolean;
+    docsPath: string | undefined;
+    apiUrl: string | undefined;
+    workspaceId: string | undefined;
+    help: boolean;
+}
+export declare const INVENTORY_HELP = "create-ailk inventory \u2014 freeze what a site serves today, and audit it against its workspace.\n\nUsage:\n  AILK_AGENT_KEY=<key> create-ailk inventory [<dir>] --origin <url> [--refreeze] [--docs-path <path>] [--api-url <origin>] [--workspace <wksp_id>]\n\nArguments:\n  <dir>                    The site. Defaults to the current dir.\n\nOptions:\n  --origin <url>           The site as served today. Needed to freeze; a\n                           frozen inventory keeps its own.\n  --refreeze               Crawl again. The operator's joins and exclusions\n                           are kept; the inventory's own are recomputed.\n  --docs-path <path>       Where the site serves its docs. Default /docs.\n  --api-url <origin>       The API origin that holds the workspace \u2014 on a\n                           spoke, its hub's. Else AILK_API_URL.\n  --workspace <wksp_id>    The workspace. Else AILK_WORKSPACE_ID.\n  -h, --help               Show this help.\n\nThe first run writes .ailk-cutover/inventory.json: every URL the sitemap,\na crawl of the site and the route roster name, each probed with no\nredirect followed. A direct 200 is frozen with its prose status and its\nimages; anything else is listed as not frozen with its status. A later\nrun reads that file and crawls nothing. Commit it.\n\nEvery run writes .ailk-cutover/audit.json: each frozen page joined to its\nworkspace page \u2014 by the inventory's \"joins\", then the roster, then /blog and\nthe docs path \u2014 and whether it is published and whether its words are in\nits Markdown. The agent key needs the content-edit grant; on a spoke,\nAILK_SPOKE_KEY is needed too. Exit 0 when every prose page passes, 1 when\nany does not (one line each), 2 on a refusal.\n";
+export declare function parseInventoryArgs(argv: string[]): InventoryArgs;
+export {};