@gtkx/cli 1.6.0 → 2.0.0-beta.1

package/src/i18n/source-messages.ts

@@ -1,5 +1,6 @@
-import type { ExtractedKey, I18nextToolkitConfig, Logger } from "i18next-cli";
-import { findKeys } from "i18next-cli";
+import type { ExtractedKey, ExtractedKeysMap, Logger, Plugin } from "i18next-cli";
+import { isPathInside, isPathWithin, toPosixPath } from "@gtkx/utils";
+import { runExtractor } from "i18next-cli";
 import {
     existsSync,
     mkdirSync,
@@ -8,12 +9,12 @@ import {
     rmSync,
     writeFileSync,
 } from "node:fs";
-import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { dirname, isAbsolute, join, relative, resolve } from "node:path";
 import type { CatalogProject } from "./catalogs.js";
 import { runCliTool } from "../internal/run-cli-tool.js";
 import { replaceCatalogTemplate } from "./catalog-template.js";
-import { gtkxExtractorPlugin } from "./extractor-plugin.js";
 import { metadataTemplateFiles } from "./metadata-templates.js";
+import { clearI18nResources, i18nToolkitConfig } from "./types.js";
 
 type SourceLocation = {
     file: string;
@@ -23,17 +24,11 @@ type SourceLocation = {
 
 type SourceMessage = {
     context: string | null;
-    kind: "default" | "plural" | "pluralDefaults" | "point";
     locations: SourceLocation[];
-    msgid: string;
     namespace: string;
     plural: string | null;
     singular: string;
-};
-
-type SourceCatalog = {
-    messages: SourceMessage[];
-    sourceFiles: string[];
+    sourceKey: string;
 };
 
 type SyntheticEntry = {
@@ -45,7 +40,6 @@ type SourceExtraction = {
     messages: SourceMessage[];
     output: string;
     project: CatalogProject;
-    sourceFiles: string[];
     workDir: string;
 };
 
@@ -54,7 +48,6 @@ type CatalogTemplateExtraction = {
     output?: string | undefined;
     project: CatalogProject;
     shouldPreserveMetadataMessages: boolean;
-    sourceFiles: string[];
 };
 
 type PluralVariant = {
@@ -94,31 +87,6 @@ const quietLogger = (): { logger: Logger; reports: string[] } => {
     };
 };
 
-const extractorConfig = (sourceFiles: string[]): I18nextToolkitConfig => {
-    const integration = gtkxExtractorPlugin();
-
-    return {
-        locales: ["en"],
-        plugins: [integration.plugin],
-        extract: {
-            input: sourceFiles,
-            output: "unused/{{language}}/{{namespace}}.json",
-            contextSeparator: CONTEXT_SEPARATOR,
-            defaultNS: DEFAULT_NAMESPACE,
-            disablePlurals: false,
-            extractFromComments: false,
-            functions: [],
-            generateBasePluralForms: false,
-            keySeparator: ".",
-            nsSeparator: false,
-            pluralSeparator: "_",
-            transComponents: integration.transComponents,
-            useTranslationNames: [],
-            warnOnConflicts: "error",
-        },
-    };
-};
-
 const splitContext = (key: string): { context: string | null; msgid: string } => {
     const index = key.lastIndexOf(CONTEXT_SEPARATOR);
 
@@ -155,17 +123,22 @@ const normalizedLocations = (locations: SourceLocation[]): SourceLocation[] => {
 };
 
 const pointSource = (entry: ExtractedKey, msgid: string): string => {
-    if (entry.explicitDefault !== true || typeof entry.defaultValue !== "string") {
+    if (typeof entry.defaultValue !== "string") {
         return msgid;
     }
 
-    return entry.defaultValue;
+    if (entry.explicitDefault === true) {
+        return entry.defaultValue;
+    }
+
+    return entry.defaultValue === msgid || msgid.endsWith(`.${entry.defaultValue}`)
+        ? msgid
+        : entry.defaultValue;
 };
 
 const pointMessage = (entry: ExtractedKey): SourceMessage => {
     const { context, msgid } = splitContext(entry.key);
     validateIdentity(context, msgid);
-    const isDefaultExplicit = entry.explicitDefault === true;
     const singular = pointSource(entry, msgid);
 
     if (singular.length === 0) {
@@ -174,12 +147,11 @@ const pointMessage = (entry: ExtractedKey): SourceMessage => {
 
     return {
         context,
-        kind: isDefaultExplicit ? "default" : "point",
         locations: normalizedLocations(entry.locations ?? []),
-        msgid,
         namespace: namespaceFor(entry),
         plural: null,
         singular,
+        sourceKey: msgid,
     };
 };
 
@@ -242,8 +214,7 @@ const pluralMessage = ({ baseKey, namespace, variants }: PluralGroup): SourceMes
     const { one, other } = pairedVariants(variants);
     const { context, msgid } = splitContext(baseKey);
     validateIdentity(context, msgid);
-    const hasStableKey = one.defaultValue !== other.defaultValue;
-    const singular = hasStableKey ? one.defaultValue : msgid;
+    const singular = one.defaultValue;
     const plural = other.defaultValue;
 
     if (singular === plural || singular.length === 0 || plural.length === 0) {
@@ -252,12 +223,11 @@ const pluralMessage = ({ baseKey, namespace, variants }: PluralGroup): SourceMes
 
     return {
         context,
-        kind: hasStableKey ? "pluralDefaults" : "plural",
         locations: normalizedLocations(variants.flatMap((variant) => variant.locations)),
-        msgid,
         namespace,
         plural,
         singular,
+        sourceKey: msgid,
     };
 };
 
@@ -295,52 +265,79 @@ const sourceMessages = (entries: ExtractedKey[]): SourceMessage[] => {
 const compareMessages = (left: SourceMessage, right: SourceMessage): number =>
     left.namespace.localeCompare(right.namespace) ||
     (left.context ?? "").localeCompare(right.context ?? "") ||
-    left.msgid.localeCompare(right.msgid) ||
     left.singular.localeCompare(right.singular) ||
     (left.plural ?? "").localeCompare(right.plural ?? "");
 
+const inferLocations = (message: SourceMessage, sources: Map<string, string>): SourceMessage => {
+    if (message.locations.length > 0) {
+        return message;
+    }
+
+    const locations: SourceLocation[] = [];
+
+    for (const [file, source] of sources) {
+        const index = source.indexOf(message.sourceKey);
+
+        if (index !== -1) {
+            locations.push({ file, line: source.slice(0, index).split("\n").length });
+        }
+    }
+
+    return { ...message, locations };
+};
+
 const normalizedSourceFiles = (root: string, paths: string[]): string[] => {
     const files = paths
         .map((path) => resolve(root, path))
-        .filter((path) => {
-            const projectPath = relative(root, path);
-
-            return projectPath !== ".." && !projectPath.startsWith(`..${sep}`) && !isAbsolute(projectPath);
-        });
+        .filter((path) => isPathWithin(root, path));
 
     return new Set(files).values().toArray().toSorted((left, right) => left.localeCompare(right));
 };
 
-const findSourceMessages = async (sourceFiles: string[]): Promise<SourceMessage[]> => {
+const findSourceMessages = async (root: string, sourceFiles: string[]): Promise<SourceMessage[]> => {
     if (sourceFiles.length === 0) {
+        clearI18nResources(root);
+
         return [];
     }
 
-    const fileErrors: string[] = [];
+    let extracted: ExtractedKeysMap | undefined;
     const { logger, reports } = quietLogger();
-    const { allKeys } = await findKeys(extractorConfig(sourceFiles), logger, fileErrors);
 
-    if (fileErrors.length > 0 || reports.length > 0) {
-        throw new Error([...fileErrors, ...reports].join("\n"));
+    const capture: Plugin = {
+        name: "gtkx-gettext",
+        onEnd(keys) {
+            extracted = keys;
+        },
+    };
+
+    const result = await runExtractor(i18nToolkitConfig(root, sourceFiles, [capture]), {
+        logger,
+        quiet: true,
+        syncPrimaryWithDefaults: true,
+        trustDerivedDefaults: true,
+    });
+
+    if (extracted === undefined || reports.length > 0 || result.hasErrors) {
+        throw new Error(reports.join("\n") || "i18next extraction failed");
     }
 
-    return sourceMessages(allKeys.values().toArray()).toSorted(compareMessages);
+    const sources = new Map(sourceFiles.map((file) => [file, readFileSync(file, "utf8")]));
+
+    return sourceMessages(extracted.values().toArray())
+        .map((message) => inferLocations(message, sources))
+        .toSorted(compareMessages);
 };
 
 const projectPath = (root: string, path: string): string | null => {
     const absolute = isAbsolute(path) ? resolve(path) : resolve(root, path);
     const projectRelative = relative(root, absolute);
 
-    if (
-        projectRelative === "" ||
-        projectRelative === ".." ||
-        projectRelative.startsWith(`..${sep}`) ||
-        isAbsolute(projectRelative)
-    ) {
+    if (!isPathInside(root, absolute)) {
         return null;
     }
 
-    return projectRelative.replaceAll("\\", "/");
+    return toPosixPath(projectRelative);
 };
 
 const writePotfiles = (project: CatalogProject, sourceFiles: string[]): string => {
@@ -371,9 +368,6 @@ const renderCatalogCall = (message: SourceMessage): string => {
         : `pgettext(${JSON.stringify(message.context)}, ${msgid});`;
 };
 
-const readSources = (sourceFiles: string[]): Map<string, string> =>
-    new Map(sourceFiles.map((path) => [path, readFileSync(path, "utf8")]));
-
 const locatedOwners = (
     project: CatalogProject,
     message: SourceMessage,
@@ -393,49 +387,23 @@ const locatedOwners = (
     return unique.values().toArray();
 };
 
-const inferredOwner = (
-    project: CatalogProject,
-    message: SourceMessage,
-    sources: Map<string, string>,
-): { path: string; line: number } | null => {
-    for (const [path, source] of sources) {
-        const index = source.indexOf(message.msgid);
-
-        if (index !== -1) {
-            return {
-                path: projectPath(project.root, path) ?? SYNTHETIC_FILENAME,
-                line: source.slice(0, index).split("\n").length,
-            };
-        }
-    }
-
-    return null;
-};
-
 const sourceOwners = (
     project: CatalogProject,
     message: SourceMessage,
-    sources: Map<string, string>,
 ): { path: string; line: number | undefined }[] => {
     const located = locatedOwners(project, message);
 
-    if (located.length > 0) {
-        return located;
-    }
-
-    return [inferredOwner(project, message, sources) ?? { path: SYNTHETIC_FILENAME, line: undefined }];
+    return located.length > 0 ? located : [{ path: SYNTHETIC_FILENAME, line: undefined }];
 };
 
 const syntheticEntries = (
     project: CatalogProject,
     messages: SourceMessage[],
-    sourceFiles: string[],
 ): Map<string, SyntheticEntry[]> => {
-    const sources = readSources(sourceFiles);
     const entries: Map<string, SyntheticEntry[]> = new Map();
 
     for (const message of messages) {
-        for (const owner of sourceOwners(project, message, sources)) {
+        for (const owner of sourceOwners(project, message)) {
             const owned = entries.get(owner.path) ?? [];
             owned.push({ call: renderCatalogCall(message), line: owner.line });
             entries.set(owner.path, owned);
@@ -492,9 +460,8 @@ const writeSyntheticSources = (
     workDir: string,
     project: CatalogProject,
     messages: SourceMessage[],
-    sourceFiles: string[],
 ): string => {
-    const entries = syntheticEntries(project, messages, sourceFiles);
+    const entries = syntheticEntries(project, messages);
     const paths: string[] = [];
 
     for (const [path, owned] of entries) {
@@ -510,8 +477,8 @@ const writeSyntheticSources = (
     return potfiles;
 };
 
-const extractSourceMessages = ({ project, messages, sourceFiles, output, workDir }: SourceExtraction): void => {
-    const potfilesPath = writeSyntheticSources(workDir, project, messages, sourceFiles);
+const extractSourceMessages = ({ project, messages, output, workDir }: SourceExtraction): void => {
+    const potfilesPath = writeSyntheticSources(workDir, project, messages);
 
     runCliTool({
         tool: "xgettext",
@@ -555,7 +522,6 @@ const joinMetadataFragment = (output: string, fragment: string): void => {
 const extractCatalogTemplate = ({
     project,
     messages,
-    sourceFiles,
     shouldPreserveMetadataMessages,
     output = resolve(project.poDir, `${project.domain}.pot`),
 }: CatalogTemplateExtraction): void => {
@@ -567,10 +533,10 @@ const extractCatalogTemplate = ({
         if (shouldPreserveMetadataMessages && hasPreviousTemplate) {
             const fragment = join(workDir, "metadata.pot");
             extractMetadataFragment(project, output, fragment);
-            extractSourceMessages({ project, messages, sourceFiles, output: source, workDir });
+            extractSourceMessages({ project, messages, output: source, workDir });
             joinMetadataFragment(source, fragment);
         } else {
-            extractSourceMessages({ project, messages, sourceFiles, output: source, workDir });
+            extractSourceMessages({ project, messages, output: source, workDir });
         }
 
         replaceCatalogTemplate(source, output);
@@ -583,26 +549,22 @@ const extractSourceCatalogTo = async (
     project: CatalogProject,
     paths: string[],
     output: string,
-): Promise<SourceCatalog> => {
+): Promise<void> => {
     const sourceFiles = normalizedSourceFiles(project.root, paths);
-    const messages = await findSourceMessages(sourceFiles);
+    const messages = await findSourceMessages(project.root, sourceFiles);
     writePotfiles(project, sourceFiles);
-    extractCatalogTemplate({ project, messages, sourceFiles, shouldPreserveMetadataMessages: false, output });
-
-    return { messages, sourceFiles };
+    extractCatalogTemplate({ project, messages, shouldPreserveMetadataMessages: false, output });
 };
 
 const extractSourceCatalog = async (
     project: CatalogProject,
     paths: string[],
     shouldPreserveMetadataMessages = true,
-): Promise<SourceCatalog> => {
+): Promise<void> => {
     const sourceFiles = normalizedSourceFiles(project.root, paths);
-    const messages = await findSourceMessages(sourceFiles);
+    const messages = await findSourceMessages(project.root, sourceFiles);
     writePotfiles(project, sourceFiles);
-    extractCatalogTemplate({ project, messages, sourceFiles, shouldPreserveMetadataMessages });
-
-    return { messages, sourceFiles };
+    extractCatalogTemplate({ project, messages, shouldPreserveMetadataMessages });
 };
 
-export { type SourceMessage, extractSourceCatalog, extractSourceCatalogTo };
+export { extractSourceCatalog, extractSourceCatalogTo };

package/src/i18n/types.ts

@@ -1,154 +1,120 @@
-import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import type { I18nextToolkitConfig, Logger, Plugin } from "i18next-cli";
+import { runTypesGenerator } from "i18next-cli";
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
-import type { SourceMessage } from "./source-messages.js";
-
-type I18nTypesResult = {
-    isWritten: boolean;
-    path: string;
-};
 
+const CONTEXT_SEPARATOR = "\u{4}";
+const DEFAULT_NAMESPACE = "translation";
 const I18N_TYPES_FILENAME = "i18n.d.ts";
-const INTERPOLATION_PATTERN = /\{\{[^{}]+\}\}/u;
-
-const stringType = (values: string[]): string =>
-    [...new Set(values)].map((value) => JSON.stringify(value)).join(" | ");
-
-const contextField = (context: string | null): string =>
-    context === null ? "" : ` context: ${JSON.stringify(context)};`;
-
-const hasInterpolation = (values: string[]): boolean => values.some((value) => INTERPOLATION_PATTERN.test(value));
+const I18N_RESOURCES_FILENAME = "i18n-resources.d.ts";
 
-const pointVariant = (message: SourceMessage): string => {
-    const isRequired = message.context !== null || hasInterpolation([message.msgid]);
-    const isInterpolated = hasInterpolation([message.msgid]);
+const generatedDir = (root: string): string => join(root, "node_modules", ".gtkx");
+const i18nResourceDir = (root: string): string => join(generatedDir(root), "i18n");
+const i18nTypesPath = (root: string): string => join(generatedDir(root), I18N_TYPES_FILENAME);
+const i18nResourcesPath = (root: string): string => join(generatedDir(root), I18N_RESOURCES_FILENAME);
 
-    return (
-        `{${contextField(message.context)} interpolated: ${String(isInterpolated)}; kind: "point"; ` +
-        `required: ${String(isRequired)}; value: ${JSON.stringify(message.msgid)} }`
-    );
+const clearI18nResources = (root: string): void => {
+    rmSync(i18nResourceDir(root), { force: true, recursive: true });
 };
 
-const defaultVariant = (message: SourceMessage): string => {
-    const values = [message.singular];
-    const isInterpolated = hasInterpolation(values);
-    const isRequired = message.context !== null || isInterpolated;
+const i18nToolkitConfig = (
+    root: string,
+    sourceFiles: string[],
+    plugins: Plugin[] = [],
+): I18nextToolkitConfig => ({
+    locales: ["en"],
+    plugins,
+    extract: {
+        contextSeparator: CONTEXT_SEPARATOR,
+        defaultNS: DEFAULT_NAMESPACE,
+        disablePlurals: false,
+        extractFromComments: false,
+        functions: ["t"],
+        generateBasePluralForms: false,
+        input: sourceFiles,
+        keySeparator: ".",
+        nsSeparator: false,
+        output: join(i18nResourceDir(root), "{{language}}", "{{namespace}}.json"),
+        pluralSeparator: "_",
+        primaryLanguage: "en",
+        removeUnusedKeys: true,
+        transComponents: ["Trans", "TransWithoutContext"],
+        useTranslationNames: ["useTranslation"],
+        warnOnConflicts: "error",
+    },
+    types: {
+        input: join(i18nResourceDir(root), "en", "*.json"),
+        output: i18nTypesPath(root),
+        resourcesFile: i18nResourcesPath(root),
+    },
+});
+
+const declarations = (): string => `import type Resources from "./i18n-resources.js";
+
+declare module "@gtkx/i18n" {
+    interface I18nResources extends Resources {}
+}
+`;
+
+const writeChanged = (path: string, content: string): void => {
+    let current: string | null;
 
-    return (
-        `{${contextField(message.context)} defaultValue: ${JSON.stringify(message.singular)}; ` +
-        `interpolated: ${String(isInterpolated)}; kind: "default"; required: ${String(isRequired)}; ` +
-        `value: ${stringType(values)} }`
-    );
-};
-
-const requirePlural = (message: SourceMessage): string => {
-    if (message.plural === null) {
-        throw new Error("Plural translation metadata requires a plural source string");
-    }
-
-    return message.plural;
-};
-
-const pluralVariant = (message: SourceMessage): string => {
-    const plural = requirePlural(message);
-    const values = [message.singular, plural];
-    const isInterpolated = hasInterpolation(values);
-
-    return (
-        `{${contextField(message.context)} defaultValue: ${JSON.stringify(plural)}; ` +
-        `interpolated: ${String(isInterpolated)}; kind: "plural"; required: true; ` +
-        `value: ${stringType(values)} }`
-    );
-};
-
-const pluralDefaultsVariant = (message: SourceMessage): string => {
-    const plural = requirePlural(message);
-    const values = [message.singular, plural];
-    const isInterpolated = hasInterpolation(values);
-
-    return (
-        `{${contextField(message.context)} defaultValueOne: ${JSON.stringify(message.singular)}; ` +
-        `defaultValueOther: ${JSON.stringify(plural)}; interpolated: ${String(isInterpolated)}; ` +
-        `kind: "pluralDefaults"; required: true; value: ${stringType(values)} }`
-    );
-};
-
-const variantsFor = (message: SourceMessage): string[] => {
-    if (message.kind === "plural") {
-        return [pluralVariant(message)];
+    try {
+        current = readFileSync(path, "utf8");
+    } catch {
+        current = null;
     }
 
-    if (message.kind === "pluralDefaults") {
-        return [pluralDefaultsVariant(message)];
+    if (current === content) {
+        return;
     }
 
-    return message.kind === "default" ? [defaultVariant(message)] : [pointVariant(message)];
+    mkdirSync(dirname(path), { recursive: true });
+    writeFileSync(path, content);
 };
 
-const renderRegistry = (messages: SourceMessage[]): string[] => {
-    const entries: Map<string, Set<string>> = new Map();
-
-    for (const message of messages) {
-        const variants = entries.get(message.msgid) ?? new Set<string>();
-
-        for (const variant of variantsFor(message)) {
-            variants.add(variant);
-        }
-
-        entries.set(message.msgid, variants);
-    }
-
-    return [...entries]
-        .toSorted(([left], [right]) => left.localeCompare(right))
-        .map(([msgid, variants]) => `        ${JSON.stringify(msgid)}: ${[...variants].join(" | ")};`);
+const typegenLogger = (): { errors: string[]; logger: Logger } => {
+    const errors: string[] = [];
+
+    return {
+        errors,
+        logger: {
+            error(value) {
+                errors.push(String(value));
+            },
+            info() {
+                return;
+            },
+            warn() {
+                return;
+            },
+        },
+    };
 };
 
-const generatedHeader = (): string[] => [
-    "/**",
-    " * Translation declarations generated by GTKX. Regenerated by",
-    " * `gtkx codegen`, `gtkx dev`, and `gtkx build`; do not edit.",
-    " */",
-];
+const emitI18nTypes = async (root: string): Promise<void> => {
+    const typesPath = i18nTypesPath(root);
+    const resourcesPath = i18nResourcesPath(root);
+    writeChanged(typesPath, declarations());
+    const { errors, logger } = typegenLogger();
+    await runTypesGenerator(i18nToolkitConfig(root, []), { logger, quiet: true });
 
-const renderI18nTypes = (messages: SourceMessage[], isEnabled = true): string => {
-    if (!isEnabled) {
-        return [...generatedHeader(), "export {};", ""].join("\n");
+    if (errors.length > 0 || !existsSync(typesPath) || !existsSync(resourcesPath)) {
+        throw new Error(errors.join("\n") || "i18next type generation did not produce its outputs");
     }
-
-    return [
-        ...generatedHeader(),
-        'import "@gtkx/i18n";',
-        "",
-        'declare module "@gtkx/i18n" {',
-        "    interface TranslationRegistry {",
-        ...renderRegistry(messages),
-        "    }",
-        "}",
-        "",
-    ].join("\n");
 };
 
-const i18nTypesPath = (root: string): string => join(root, "node_modules", ".gtkx", I18N_TYPES_FILENAME);
-
-const readFileOrNull = (path: string): string | null => {
-    try {
-        return readFileSync(path, "utf8");
-    } catch {
-        return null;
-    }
+const clearI18nTypes = (root: string): void => {
+    rmSync(i18nTypesPath(root), { force: true });
+    rmSync(i18nResourcesPath(root), { force: true });
+    clearI18nResources(root);
 };
 
-const emitI18nTypes = (root: string, messages: SourceMessage[], isEnabled = true): I18nTypesResult => {
-    const path = i18nTypesPath(root);
-    const content = renderI18nTypes(messages, isEnabled);
-
-    if (readFileOrNull(path) === content) {
-        return { isWritten: false, path };
-    }
-
-    mkdirSync(dirname(path), { recursive: true });
-    writeFileSync(path, content);
-
-    return { isWritten: true, path };
+export {
+    I18N_TYPES_FILENAME,
+    clearI18nResources,
+    clearI18nTypes,
+    emitI18nTypes,
+    i18nToolkitConfig,
+    i18nTypesPath,
 };
-
-export { I18N_TYPES_FILENAME, emitI18nTypes, i18nTypesPath };

package/src/internal/docs-elements.ts

@@ -7,13 +7,13 @@ type DocsElements = {
 };
 
 const resolveDocsElements = async (cwd: string): Promise<DocsElements> => {
-    const { gi, reactSubexports } = resolveStore(cwd);
+    const { gi } = resolveStore(cwd);
 
     if (!existsSync(gi.storeDir)) {
         return { props: {}, omittedProps: {} };
     }
 
-    const { props, omittedProps } = await readBuiltinElements(reactSubexports, gi.storeDir);
+    const { props, omittedProps } = await readBuiltinElements();
 
     return { props, omittedProps };
 };