create-ailk 0.28.1 → 0.29.0

package/dist/lib/import-mdx.js

@@ -0,0 +1,452 @@
+/**
+ * `importMdx` — one MDX→Artifact importer, idempotent on re-run (#10203).
+ *
+ * Reads a site's default-locale MDX pages through the site's own MDX adapter
+ * and writes each one as a `text/page` Artifact through the tenant-guarded
+ * workspace content routes. Per page (D3): read the draft; create on 404;
+ * PATCH under `If-Match` when the content differs; write nothing when it is
+ * the same — so a second run over the same files creates nothing and reports
+ * every page `unchanged`. A file the adapter rejects is `failed` with its path
+ * and reason and the run goes on. The seed step (U6b) and `migrate-content`
+ * (U6c) both call it; this module adds no command.
+ *
+ * create-ailk ships one runtime dependency, so the adapter is never imported:
+ * the caller injects it, and `loadSiteMdxAdapter` resolves the SITE's
+ * installed copy from its `apps/web` at run time (the #6937 precedent).
+ *
+ * The key travels only as `authorization: Bearer <key>`, never across a
+ * redirect (`redirect: "error"`), and is never printed. Of an API answer only
+ * a status and a `[a-z0-9_]` error code reach output (SEC-7876-S1).
+ *
+ * Spec: docs/specs/10203-import-mdx-one-mdx-artifact-spec.md.
+ */
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { createRequire } from "node:module";
+import { dirname, join, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { TopologyError, normalizeHubUrl } from "../topology.js";
+/**
+ * The package the adapter comes from. Interpolated, not written inline, so no
+ * import specifier of a package create-ailk does not depend on appears here.
+ */
+const CONTENT_ADAPTERS_PACKAGE = "@working-theory/content-adapters";
+/** The adapter's locale shape (`BCP47_RE` in its `mdx/adapter.ts`). */
+const LOCALE = /^[a-z]{2}(-[A-Z]{2})?$/;
+/** `wksp_<uuidv7>` — the `@working-theory/ids` shape, restated: no runtime dep. */
+const WORKSPACE_ID = /^wksp_[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
+/** A key is visible ASCII only: what an HTTP header can carry, with no space. */
+const KEY_SHAPE = /^[\x21-\x7e]+$/;
+/** The only API-supplied text printed: a code of this shape, else `unknown`. */
+const SAFE_CODE = /^[a-z0-9_]{1,64}$/;
+const REQUEST_TIMEOUT_MS = 15_000;
+const ADAPTER_REASON_MAX = 300;
+/** A 429 is retried this many times; the next one fails the page. */
+const RATE_LIMIT_RETRIES = 3;
+const RATE_LIMIT_WAIT_MAX_S = 60;
+/** A refusal the CLI exits 2 on, thrown before any file is read or request sent. */
+export class ImportMdxRefusedError extends Error {
+    exitCode = 2;
+    constructor(message) {
+        super(message);
+        this.name = "ImportMdxRefusedError";
+    }
+}
+// ── Refusals ────────────────────────────────────────────────────────────────
+function refuse(message) {
+    throw new ImportMdxRefusedError(`import-mdx: ${message} Nothing was sent.`);
+}
+function checkOptions(opts, defaultLocale) {
+    let isDir;
+    try {
+        isDir = statSync(opts.contentRoot).isDirectory();
+    }
+    catch {
+        isDir = false;
+    }
+    if (!isDir)
+        refuse(`the content root ${opts.contentRoot} is not a directory.`);
+    if (!KEY_SHAPE.test(opts.apiKey)) {
+        refuse("the API key is empty or carries a space, a control character or a non-ASCII character.");
+    }
+    if (!WORKSPACE_ID.test(opts.workspaceId)) {
+        refuse("the workspace id is not a workspace id (wksp_<uuidv7>).");
+    }
+    if (!LOCALE.test(defaultLocale)) {
+        refuse("the default locale is not a locale such as en or pt-BR.");
+    }
+    try {
+        return normalizeHubUrl(opts.apiOrigin);
+    }
+    catch (err) {
+        if (!(err instanceof TopologyError))
+            throw err;
+        refuse("the API origin must be an https: origin (http: only on a loopback host) with no path, query, fragment or credentials.");
+    }
+}
+/** Every `.mdx` file and every link to one or to a directory below `root`; `_` entries unseen. */
+function mdxFiles(root, prefix = []) {
+    return readdirSync(join(root, ...prefix), { withFileTypes: true }).flatMap((entry) => {
+        if (entry.name.startsWith("_"))
+            return [];
+        const segments = [...prefix, entry.name];
+        if (entry.isDirectory())
+            return mdxFiles(root, segments);
+        if (entry.isSymbolicLink()) {
+            let dir = false;
+            try {
+                dir = statSync(join(root, ...segments)).isDirectory();
+            }
+            catch {
+                // A dangling link: reported only when it is named like a page.
+            }
+            return dir || entry.name.endsWith(".mdx")
+                ? [{ segments, link: true }]
+                : [];
+        }
+        return entry.isFile() && entry.name.endsWith(".mdx")
+            ? [{ segments, link: false }]
+            : [];
+    });
+}
+function byTypeThenSlug(a, b) {
+    if (a.type !== b.type)
+        return a.type < b.type ? -1 : 1;
+    return a.slug < b.slug ? -1 : a.slug > b.slug ? 1 : 0;
+}
+function plan(contentRoot, types, defaultLocale) {
+    const localePages = [];
+    const legacyPages = [];
+    const skipped = [];
+    for (const { segments, link } of mdxFiles(contentRoot)) {
+        const path = segments.join("/");
+        const slugOf = (file) => file.slice(0, -".mdx".length);
+        const [first, second, third] = segments;
+        if (link) {
+            skipped.push({
+                path,
+                result: "skipped",
+                reason: "a symbolic link is not followed",
+            });
+        }
+        else if (segments.length === 3 &&
+            first === defaultLocale &&
+            types.includes(second)) {
+            localePages.push({
+                path,
+                type: second,
+                slug: slugOf(third),
+                localeTree: true,
+            });
+        }
+        else if (segments.length === 2 && types.includes(first)) {
+            legacyPages.push({
+                path,
+                type: first,
+                slug: slugOf(second),
+                localeTree: false,
+            });
+        }
+        else if (segments.length > 1 &&
+            LOCALE.test(first) &&
+            first !== defaultLocale) {
+            skipped.push({
+                path,
+                result: "skipped",
+                reason: `locale ${first} is not imported (only ${defaultLocale})`,
+            });
+        }
+        else {
+            skipped.push({
+                path,
+                result: "skipped",
+                reason: "not a page: the adapter reads <locale>/<Type>/<slug>.mdx",
+            });
+        }
+    }
+    skipped.sort((a, b) => (a.path < b.path ? -1 : 1));
+    return {
+        pages: [
+            ...localePages.sort(byTypeThenSlug),
+            ...legacyPages.sort(byTypeThenSlug),
+        ],
+        skipped,
+    };
+}
+// ── "Unchanged" (D4) ────────────────────────────────────────────────────────
+function sortKeys(value) {
+    if (Array.isArray(value))
+        return value.map(sortKeys);
+    if (value === null || typeof value !== "object")
+        return value;
+    return Object.fromEntries(Object.keys(value)
+        .sort()
+        .map((key) => [key, sortKeys(value[key])]));
+}
+/** JSON with keys sorted at every depth; `undefined` members and Date objects as JSON has them. */
+function canonical(value) {
+    return JSON.stringify(sortKeys(JSON.parse(JSON.stringify(value ?? null))));
+}
+function safeCode(value) {
+    return typeof value === "string" && SAFE_CODE.test(value) ? value : "unknown";
+}
+function retryAfterSeconds(header) {
+    const seconds = header !== null && /^\d+$/.test(header) ? Number(header) : 0;
+    return seconds >= 1 && seconds <= RATE_LIMIT_WAIT_MAX_S
+        ? seconds
+        : RATE_LIMIT_WAIT_MAX_S;
+}
+async function call(t, method, url, body, ifMatch) {
+    for (let retry = 0;; retry++) {
+        let response;
+        try {
+            response = await t.fetch(url, {
+                method,
+                headers: {
+                    authorization: `Bearer ${t.key}`,
+                    ...(body !== undefined && { "content-type": "application/json" }),
+                    ...(ifMatch !== undefined && { "if-match": ifMatch }),
+                },
+                ...(body !== undefined && { body: JSON.stringify(body) }),
+                // A redirect would re-send the key and the body to another URL.
+                redirect: "error",
+                signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+            });
+        }
+        catch (err) {
+            const name = err instanceof Error ? err.name.toLowerCase() : undefined;
+            return { network: `network ${safeCode(name)}` };
+        }
+        if (response.status === 429 && retry < RATE_LIMIT_RETRIES) {
+            const seconds = retryAfterSeconds(response.headers.get("retry-after"));
+            t.onLine(`rate limited — waiting ${seconds}s`);
+            await t.sleep(seconds * 1000);
+            continue;
+        }
+        let envelope = {};
+        try {
+            envelope = ((await response.json()) ?? {});
+        }
+        catch {
+            // No JSON body: the status alone is reported.
+        }
+        const data = envelope.data;
+        return {
+            status: response.status,
+            code: safeCode(envelope.error?.code),
+            data: data !== null && typeof data === "object"
+                ? data
+                : {},
+        };
+    }
+}
+function failed(path, reason, halt) {
+    return {
+        outcome: { path, result: "failed", reason },
+        ...(halt !== undefined && { halt }),
+    };
+}
+/** A non-success answer: 401/403 stops the run, anything else fails the page. */
+function refusedAnswer(path, answer) {
+    if ("network" in answer)
+        return failed(path, answer.network);
+    const reason = `${answer.status} ${answer.code}`;
+    return answer.status === 401 || answer.status === 403
+        ? failed(path, reason, `not sent: the API refused the key (${reason})`)
+        : failed(path, reason);
+}
+/**
+ * A success answered for another locale: the API does not serve `locale`. A
+ * success with no locale at all is not the API's answer shape: that page fails.
+ */
+function wrongLocale(path, answer, locale) {
+    if ("network" in answer || answer.data["locale"] === locale)
+        return undefined;
+    if (typeof answer.data["locale"] !== "string") {
+        return failed(path, `${answer.status} unknown`);
+    }
+    return failed(path, `the API does not serve locale ${locale}`, `not sent: the API does not serve locale ${locale}`);
+}
+async function writePage(t, page, locale, content) {
+    const { path, type, slug } = page;
+    const address = `${t.base}/content/${encodeURIComponent(type)}/${encodeURIComponent(slug)}?locale=${encodeURIComponent(locale)}`;
+    const done = (result) => ({
+        outcome: { path, result, type, slug },
+    });
+    for (let reread = false;; reread = true) {
+        const read = await call(t, "GET", address);
+        if ("network" in read)
+            return refusedAnswer(path, read);
+        if (read.status === 200) {
+            const mismatch = wrongLocale(path, read, locale);
+            if (mismatch !== undefined)
+                return mismatch;
+            const version = read.data["version"];
+            if (typeof version !== "string")
+                return failed(path, "200 unknown");
+            if (canonical(read.data["content"]) === canonical(content)) {
+                return done("unchanged");
+            }
+            // The token is the version just compared against, so a draft saved
+            // since the read is a 409, never a silent overwrite.
+            const saved = await call(t, "PATCH", address, content, version);
+            if ("network" in saved || saved.status !== 200)
+                return refusedAnswer(path, saved);
+            return wrongLocale(path, saved, locale) ?? done("updated");
+        }
+        if (read.status !== 404 || reread)
+            return refusedAnswer(path, read);
+        const created = await call(t, "POST", `${t.base}/content`, {
+            ...content,
+            locale,
+        });
+        if ("network" in created)
+            return refusedAnswer(path, created);
+        if (created.status === 201) {
+            return wrongLocale(path, created, locale) ?? done("created");
+        }
+        // 409: created by someone else since the read — read it once more.
+        if (created.status !== 409)
+            return refusedAnswer(path, created);
+    }
+}
+// ── The report (D6) ─────────────────────────────────────────────────────────
+function outcomeLine(outcome) {
+    return "reason" in outcome
+        ? `${outcome.path}: ${outcome.result} — ${outcome.reason}`
+        : `${outcome.path}: ${outcome.result}`;
+}
+/** The whole report, one line per page and a final count line. */
+export function formatImportMdxReport(result) {
+    const { created, updated, unchanged, skipped, failed: failures, } = result.counts;
+    return `${[
+        ...result.pages.map(outcomeLine),
+        `created ${created} · updated ${updated} · unchanged ${unchanged} · skipped ${skipped} · failed ${failures}`,
+    ].join("\n")}\n`;
+}
+// ── The importer ────────────────────────────────────────────────────────────
+/** One whitespace-collapsed line of at most `ADAPTER_REASON_MAX` characters. */
+function adapterReason(err) {
+    const message = err instanceof Error ? err.message : String(err);
+    return `adapter: ${message.replace(/\s+/g, " ").trim().slice(0, ADAPTER_REASON_MAX)}`;
+}
+export async function importMdx(opts) {
+    const defaultLocale = opts.defaultLocale ?? "en";
+    const origin = checkOptions(opts, defaultLocale);
+    const onLine = opts.onLine ?? (() => undefined);
+    const t = {
+        fetch: opts.fetch ?? globalThis.fetch,
+        base: `${origin}/v1/workspaces/${encodeURIComponent(opts.workspaceId)}`,
+        key: opts.apiKey,
+        sleep: opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms))),
+        onLine,
+    };
+    const contentRoot = resolve(opts.contentRoot);
+    const { pages, skipped } = plan(contentRoot, opts.adapters.SCHEMA_ORG_TYPES, defaultLocale);
+    const reader = opts.adapters.createMdxAdapter({ contentRoot, defaultLocale });
+    const outcomes = [];
+    const record = (outcome) => {
+        outcomes.push(outcome);
+        onLine(outcomeLine(outcome));
+    };
+    // Two pages at one (locale, slug) address would overwrite each other on
+    // every run, so the first in processing order wins and the rest fail.
+    const addresses = new Map();
+    let halt;
+    for (const page of pages) {
+        const holder = addresses.get(page.slug);
+        if (holder !== undefined) {
+            record({
+                path: page.path,
+                result: "failed",
+                reason: `same address as ${holder} (a page is addressed by locale and slug)`,
+            });
+            continue;
+        }
+        addresses.set(page.slug, page.path);
+        if (halt !== undefined) {
+            record({ path: page.path, result: "failed", reason: halt });
+            continue;
+        }
+        let source;
+        try {
+            source = page.localeTree
+                ? await reader.getPageSource(page.type, page.slug, defaultLocale)
+                : await reader.getPageSource(page.type, page.slug);
+        }
+        catch (err) {
+            record({ path: page.path, result: "failed", reason: adapterReason(err) });
+            continue;
+        }
+        if (source === null) {
+            record({
+                path: page.path,
+                result: "failed",
+                reason: "adapter: not read",
+            });
+            continue;
+        }
+        const result = await writePage(t, page, defaultLocale, {
+            ...source.frontmatter,
+            body: source.body,
+        });
+        halt = result.halt;
+        record(result.outcome);
+    }
+    skipped.forEach(record);
+    const counts = {
+        created: 0,
+        updated: 0,
+        unchanged: 0,
+        skipped: 0,
+        failed: 0,
+    };
+    for (const outcome of outcomes)
+        counts[outcome.result] += 1;
+    return { pages: outcomes, counts };
+}
+// ── The site's own adapter (D1) ─────────────────────────────────────────────
+/** Whether a `node_modules` at or above `dir` holds the adapter package. */
+function installedAbove(dir) {
+    const pkg = join("node_modules", ...CONTENT_ADAPTERS_PACKAGE.split("/"));
+    for (let at = dir;; at = dirname(at)) {
+        if (existsSync(join(at, pkg, "package.json")))
+            return true;
+        if (dirname(at) === at)
+            return false;
+    }
+}
+/**
+ * The site's installed adapter package, resolved from its `apps/web` — never
+ * create-ailk's own `node_modules`.
+ */
+export async function loadSiteMdxAdapter(siteRoot) {
+    const web = join(resolve(siteRoot), "apps", "web");
+    let entry;
+    try {
+        // `resolve` falls back to NODE_PATH (which `pnpm exec` and `npx` set) once
+        // the site's own node_modules come up empty, so the package must first be
+        // found in one of those.
+        if (!installedAbove(web))
+            throw new Error("not installed");
+        entry = createRequire(join(web, "package.json")).resolve(CONTENT_ADAPTERS_PACKAGE);
+    }
+    catch {
+        throw new ImportMdxRefusedError(`import-mdx: ${CONTENT_ADAPTERS_PACKAGE} is not installed in ${web}. Install the site's dependencies (pnpm install) first. Nothing was sent.`);
+    }
+    let mod;
+    try {
+        mod = (await import(pathToFileURL(entry).href));
+    }
+    catch (err) {
+        const code = err?.code;
+        const name = err instanceof Error ? err.name : "error";
+        throw new ImportMdxRefusedError(`import-mdx: ${CONTENT_ADAPTERS_PACKAGE} in ${web} is installed but failed to load (${typeof code === "string" ? code : name}). Nothing was sent.`);
+    }
+    const types = mod.SCHEMA_ORG_TYPES;
+    if (typeof mod.createMdxAdapter !== "function" ||
+        !Array.isArray(types) ||
+        !types.every((type) => typeof type === "string")) {
+        throw new ImportMdxRefusedError(`import-mdx: ${CONTENT_ADAPTERS_PACKAGE} in ${web} does not export createMdxAdapter and SCHEMA_ORG_TYPES. Install the site's dependencies (pnpm install) first. Nothing was sent.`);
+    }
+    return mod;
+}

package/dist/migrate-content.d.ts

@@ -0,0 +1,45 @@
+import { type ImportMdxResult, type MdxAdapterModule } from "./lib/import-mdx.js";
+/** A refusal the CLI exits 2 on, thrown before any request is sent. */
+export declare class MigrateContentRefusedError extends Error {
+    readonly exitCode = 2;
+    constructor(message: string);
+}
+export interface MigrateContentOptions {
+    /** The site. */
+    targetDir: string;
+    /** `--apply`: write. Without it nothing is written. */
+    apply?: boolean;
+    /** `--api-url`. */
+    apiUrl?: string | undefined;
+    /** `--workspace`. */
+    workspaceId?: string | undefined;
+    /** Injectable for tests; defaults to `process.env`. */
+    env?: NodeJS.ProcessEnv;
+    /** Injectable for tests; defaults to the site's installed adapter. */
+    adapters?: MdxAdapterModule;
+    /** Injectable for tests; defaults to `globalThis.fetch`. */
+    fetch?: typeof globalThis.fetch;
+    /** Injectable for tests; the wait before a rate-limited retry. */
+    sleep?: (ms: number) => Promise<void>;
+    /** Each report line as it happens; defaults to nothing. */
+    onLine?: (line: string) => void;
+}
+/**
+ * A fetch that sends `GET`s through `inner` and never sends a write: a `POST`
+ * is answered `201` and a `PATCH` `200` locally, with the one member of a
+ * write's answer `importMdx` reads (`data.locale`). Any other method throws.
+ */
+export declare function readOnlyFetch(inner: typeof globalThis.fetch, locale: string): typeof globalThis.fetch;
+export declare function migrateContent(opts: MigrateContentOptions): Promise<{
+    result: ImportMdxResult;
+    exitCode: 0 | 1;
+}>;
+export interface MigrateContentArgs {
+    target: string | undefined;
+    apply: boolean;
+    apiUrl: string | undefined;
+    workspaceId: string | undefined;
+    help: boolean;
+}
+export declare const MIGRATE_CONTENT_HELP = "create-ailk migrate-content \u2014 import an existing site's pages into its workspace.\n\nUsage:\n  AILK_AGENT_KEY=<key> create-ailk migrate-content [<dir>] [--apply] [--api-url <origin>] [--workspace <wksp_id>]\n\nArguments:\n  <dir>                    The site. Defaults to the current dir.\n\nOptions:\n  --apply                  Write the pages. Without it the command reads the\n                           workspace, prints what it would do, and writes\n                           nothing.\n  --api-url <origin>       The API origin that holds the workspace \u2014 on a\n                           spoke, its hub's. Else AILK_API_URL. https: only\n                           (http: on localhost or 127.0.0.1).\n  --workspace <wksp_id>    The workspace the pages are written to. Else\n                           AILK_WORKSPACE_ID.\n  -h, --help               Show this help.\n\nEach content/en/<Type>/<slug>.mdx page (and each legacy\ncontent/<Type>/<slug>.mdx) is written to the workspace as a draft through\nthe API's content routes, with the agent key, which is read from\nAILK_AGENT_KEY only and needs the content-edit grant. Nothing is published,\nso the live site does not change. The site needs no .ailk-site-manifest.json\nand may be a spoke; its dependencies must be installed.\n\nA dry run prints create, update, unchanged, skipped or failed per page. It\ncannot see a write the API would refuse at --apply (a conflict or a server\nerror), and a page whose content fails its page type is planned and saved\nwith errors the editor shows, not failed. --apply prints created, updated,\nunchanged, skipped or failed per page, so a re-run is safe. Exit 0 when no\npage failed, 1 when any did, 2 on a refusal (nothing is sent).\n";
+export declare function parseMigrateContentArgs(argv: string[]): MigrateContentArgs;