@atcute/lex-cli 2.5.3 → 2.6.0

package/src/codegen.ts

@@ -16,8 +16,6 @@ import type {
 } from '@atcute/lexicon-doc';
 import { formatLexiconRef, parseLexiconRef, type ParsedLexiconRef } from '@atcute/lexicon-doc';
 
-import * as prettier from 'prettier';
-
 export interface SourceFile {
 	filename: string;
 	code: string;
@@ -34,13 +32,6 @@ export interface LexiconApiOptions {
 	modules: {
 		importSuffix: string;
 	};
-	prettier: {
-		cwd: string;
-	};
-}
-
-export interface LexiconApiResult {
-	files: SourceFile[];
 }
 
 type DocumentMap = Map<string, LexiconDoc>;
@@ -68,7 +59,7 @@ const resolveExternalImport = (nsid: string, mappings: ImportMapping[]): ImportM
 
 const PURE = `/*#__PURE__*/`;
 
-export const generateLexiconApi = async (opts: LexiconApiOptions): Promise<LexiconApiResult> => {
+export function* generateLexiconApi(opts: LexiconApiOptions): Generator<SourceFile> {
 	const importExt = opts.modules?.importSuffix;
 
 	const documents = opts.documents.toSorted((a, b) => {
@@ -83,7 +74,6 @@ export const generateLexiconApi = async (opts: LexiconApiOptions): Promise<Lexic
 	});
 
 	const map: DocumentMap = new Map(documents.map((doc) => [doc.id, doc]));
-	const files: SourceFile[] = [];
 	const generatedIds = new Set<string>();
 
 	for (const doc of documents) {
@@ -336,7 +326,7 @@ export const generateLexiconApi = async (opts: LexiconApiOptions): Promise<Lexic
 		if (file.exports) {
 			generatedIds.add(doc.id);
 
-			files.push({
+			yield {
 				filename: filename,
 				code:
 					file.imports +
@@ -354,7 +344,7 @@ export const generateLexiconApi = async (opts: LexiconApiOptions): Promise<Lexic
 					file.sinterfaces +
 					`\n\n` +
 					file.ambients,
-			});
+			};
 		}
 	}
 
@@ -369,23 +359,12 @@ export const generateLexiconApi = async (opts: LexiconApiOptions): Promise<Lexic
 			code += `export * as ${toTitleCase(doc.id)} from ${lit(`./types/${doc.id.replaceAll('.', '/')}${importExt}`)};\n`;
 		}
 
-		files.push({
+		yield {
 			filename: 'index.ts',
 			code: code,
-		});
-	}
-
-	if (opts.prettier) {
-		const config = await prettier.resolveConfig(opts.prettier.cwd, { editorconfig: true });
-
-		for (const file of files) {
-			const formatted = await prettier.format(file.code, { ...config, parser: 'typescript' });
-			file.code = formatted;
-		}
+		};
 	}
-
-	return { files };
-};
+}
 
 const generateXrpcQuery = (imports: ImportSet, path: ParsedLexiconRef, spec: LexXrpcQuery): string => {
 	const params = generateXrpcParameters(imports, path, spec.parameters);
@@ -729,6 +708,7 @@ const generateType = (
 					const refPath = resolvePath(path, ref);
 					return { path: refPath, uri: formatLexiconRef(refPath) };
 				})
+				// oxlint-disable-next-line unicorn/no-array-sort -- map already clones
 				.sort((a, b) => {
 					if (a.uri < b.uri) {
 						return -1;

package/src/commands/export.ts

@@ -8,11 +8,11 @@ import { message } from '@optique/core/message';
 import { type InferValue } from '@optique/core/parser';
 import { command, constant } from '@optique/core/primitives';
 import pc from 'picocolors';
-import prettier from 'prettier';
 
-import { loadConfig, type ExportConfig, type NormalizedConfig } from '../config.js';
-import { loadLexicons } from '../lexicon-loader.js';
-import { sharedOptions } from '../shared-options.js';
+import { loadConfig, type ExportConfig, type NormalizedConfig } from '../config.ts';
+import { createFormatter, type Formatter } from '../formatter.ts';
+import { loadLexicons } from '../lexicon-loader.ts';
+import { sharedOptions } from '../shared-options.ts';
 
 export const exportCommandSchema = command(
 	'export',
@@ -44,27 +44,17 @@ const ensureExportConfig = (config: NormalizedConfig): ExportConfig => {
 	return config.export;
 };
 
-/**
- * writes a lexicon document to disk as formatted JSON
- * @param outdir output directory
- * @param nsid the NSID of the lexicon
- * @param doc the lexicon document
- * @param prettierConfig prettier configuration
- */
 const writeLexicon = async (
 	outdir: string,
 	nsid: string,
 	doc: LexiconDoc,
-	prettierConfig: prettier.Options | null,
+	formatter: Formatter,
 ): Promise<void> => {
 	const nsidPath = nsid.replaceAll('.', '/');
 	const target = path.join(outdir, `${nsidPath}.json`);
 	const dirname = path.dirname(target);
 
-	const code = await prettier.format(JSON.stringify(doc, null, 2), {
-		...prettierConfig,
-		parser: 'json',
-	});
+	const code = await formatter.format(JSON.stringify(doc, null, 2), target);
 
 	await fs.mkdir(dirname, { recursive: true });
 	await fs.writeFile(target, code);
@@ -81,7 +71,7 @@ export const runExport = async (args: ExportCommand): Promise<void> => {
 	// use export.files if specified, otherwise fall back to root files config
 	const files = exportConfig.files ?? config.files;
 	const outdir = path.resolve(config.root, exportConfig.outdir);
-	const prettierConfig = await prettier.resolveConfig(config.root, { editorconfig: true });
+	const formatter = await createFormatter(config.formatter, config.root);
 
 	// load lexicons from files
 	const loaded = await loadLexicons(files, config.root);
@@ -99,9 +89,7 @@ export const runExport = async (args: ExportCommand): Promise<void> => {
 	await fs.mkdir(outdir, { recursive: true });
 
 	// write each lexicon as JSON
-	for (const { nsid, doc } of loaded) {
-		await writeLexicon(outdir, nsid, doc, prettierConfig);
-	}
+	await Promise.all(loaded.map(({ nsid, doc }) => writeLexicon(outdir, nsid, doc, formatter)));
 
 	console.log(pc.green(`exported ${loaded.length} lexicon(s) to ${outdir}`));
 };

package/src/commands/generate.ts

@@ -7,11 +7,12 @@ import { type InferValue } from '@optique/core/parser';
 import { command, constant } from '@optique/core/primitives';
 import pc from 'picocolors';
 
-import { generateLexiconApi, type ImportMapping } from '../codegen.js';
-import { loadConfig } from '../config.js';
-import { loadLexicons } from '../lexicon-loader.js';
-import { packageJsonSchema } from '../lexicon-metadata.js';
-import { sharedOptions } from '../shared-options.js';
+import { generateLexiconApi, type ImportMapping } from '../codegen.ts';
+import { loadConfig } from '../config.ts';
+import { createFormatter } from '../formatter.ts';
+import { loadLexicons } from '../lexicon-loader.ts';
+import { packageJsonSchema } from '../lexicon-metadata.ts';
+import { sharedOptions } from '../shared-options.ts';
 
 /**
  * resolves package imports to ImportMapping[]
@@ -147,24 +148,27 @@ export const runGenerate = async (args: GenerateCommand): Promise<void> => {
 	const loaded = await loadLexicons(config.files, config.root);
 	const documents = loaded.map((l) => l.doc);
 
-	const generationResult = await generateLexiconApi({
+	const outdir = path.join(config.root, config.outdir);
+	const formatter = await createFormatter(config.formatter, config.root);
+	const pending: Promise<void>[] = [];
+
+	for (const file of generateLexiconApi({
 		documents: documents,
 		mappings: allMappings,
 		modules: {
 			importSuffix: config.modules?.importSuffix ?? '.js',
 		},
-		prettier: {
-			cwd: process.cwd(),
-		},
-	});
-
-	const outdir = path.join(config.root, config.outdir);
-
-	for (const file of generationResult.files) {
+	})) {
 		const filename = path.join(outdir, file.filename);
-		const dirname = path.dirname(filename);
 
-		await fs.mkdir(dirname, { recursive: true });
-		await fs.writeFile(filename, file.code);
+		pending.push(
+			(async () => {
+				const formatted = await formatter.format(file.code, filename);
+				await fs.mkdir(path.dirname(filename), { recursive: true });
+				await fs.writeFile(filename, formatted);
+			})(),
+		);
 	}
+
+	await Promise.all(pending);
 };

package/src/commands/pull.ts

@@ -8,13 +8,13 @@ import { message } from '@optique/core/message';
 import { type InferValue } from '@optique/core/parser';
 import { command, constant } from '@optique/core/primitives';
 import pc from 'picocolors';
-import prettier from 'prettier';
 
-import { loadConfig, type NormalizedConfig, type PullConfig, type SourceConfig } from '../config.js';
-import { pullAtprotoSource } from '../pull-sources/atproto.js';
-import { pullGitSource } from '../pull-sources/git.js';
-import type { PullResult, PulledLexicon, SourceLocation } from '../pull-sources/types.js';
-import { sharedOptions } from '../shared-options.js';
+import { loadConfig, type NormalizedConfig, type PullConfig, type SourceConfig } from '../config.ts';
+import { createFormatter, type Formatter } from '../formatter.ts';
+import { pullAtprotoSource } from '../pull-sources/atproto.ts';
+import { pullGitSource } from '../pull-sources/git.ts';
+import type { PullResult, PulledLexicon, SourceLocation } from '../pull-sources/types.ts';
+import { sharedOptions } from '../shared-options.ts';
 
 export const pullCommandSchema = command(
 	'pull',
@@ -110,16 +110,13 @@ const writeLexicon = async (
 	outdir: string,
 	nsid: string,
 	doc: LexiconDoc,
-	prettierConfig: prettier.Options | null,
+	formatter: Formatter,
 ): Promise<void> => {
 	const nsidPath = nsid.replaceAll('.', '/');
 	const target = path.join(outdir, `${nsidPath}.json`);
 	const dirname = path.dirname(target);
 
-	const code = await prettier.format(JSON.stringify(doc, null, 2), {
-		...prettierConfig,
-		parser: 'json',
-	});
+	const code = await formatter.format(JSON.stringify(doc, null, 2), target);
 
 	await fs.mkdir(dirname, { recursive: true });
 	await fs.writeFile(target, code);
@@ -139,7 +136,7 @@ const pullSource = async (source: SourceConfig): Promise<PullResult> => {
 const writeSourceReadme = async (
 	outdir: string,
 	revisions: SourceRevision[],
-	prettierConfig: prettier.Options | null,
+	formatter: Formatter,
 ): Promise<void> => {
 	const lines = [
 		'# lexicon sources',
@@ -173,12 +170,10 @@ const writeSourceReadme = async (
 	lines.push('');
 
 	const content = lines.join('\n');
-	const formatted = await prettier.format(content, {
-		...prettierConfig,
-		parser: 'markdown',
-	});
+	const target = path.join(outdir, 'README.md');
+	const formatted = await formatter.format(content, target);
 
-	await fs.writeFile(path.join(outdir, 'README.md'), formatted);
+	await fs.writeFile(target, formatted);
 };
 
 /**
@@ -190,7 +185,7 @@ export const runPull = async (args: PullCommand): Promise<void> => {
 	const pullConfig = ensurePullConfig(config);
 
 	const outdir = path.resolve(config.root, pullConfig.outdir);
-	const prettierConfig = await prettier.resolveConfig(config.root, { editorconfig: true });
+	const formatter = await createFormatter(config.formatter, config.root);
 
 	const seen = new Map<string, SourceLocation>();
 	const collected: PulledLexicon[] = [];
@@ -224,9 +219,8 @@ export const runPull = async (args: PullCommand): Promise<void> => {
 
 	await fs.mkdir(outdir, { recursive: true });
 
-	for (const entry of collected) {
-		await writeLexicon(outdir, entry.nsid, entry.doc, prettierConfig);
-	}
-
-	await writeSourceReadme(outdir, sourceRevisions, prettierConfig);
+	await Promise.all([
+		...collected.map((entry) => writeLexicon(outdir, entry.nsid, entry.doc, formatter)),
+		writeSourceReadme(outdir, sourceRevisions, formatter),
+	]);
 };

package/src/config.ts

@@ -1,4 +1,5 @@
 import * as fs from 'node:fs/promises';
+import { availableParallelism } from 'node:os';
 import * as path from 'node:path';
 import * as url from 'node:url';
 
@@ -8,7 +9,7 @@ import { isHandle, isNsid } from '@atcute/lexicons/syntax';
 import * as v from '@badrap/valita';
 import pc from 'picocolors';
 
-import type { ImportMapping } from './codegen.js';
+import type { ImportMapping } from './codegen.ts';
 
 const gitSourceConfigSchema = v.object({
 	type: v.literal('git'),
@@ -63,6 +64,18 @@ const exportConfigSchema = v.object({
 	clean: v.boolean().optional(),
 });
 
+const formatterConfigSchema = v.union(
+	v.object({ type: v.literal('prettier') }),
+	v.object({
+		type: v.literal('command'),
+		command: v.string().assert((value) => value.length > 0, `must not be empty`),
+		concurrency: v
+			.number()
+			.assert((value) => Number.isInteger(value) && value > 0, `must be a positive integer`)
+			.optional(() => availableParallelism()),
+	}),
+);
+
 export type GitSourceConfig = v.Infer<typeof gitSourceConfigSchema>;
 export type AtprotoNsidsSourceConfig = v.Infer<typeof atprotoNsidsSourceConfigSchema>;
 export type AtprotoAuthoritySourceConfig = v.Infer<typeof atprotoAuthoritySourceConfigSchema>;
@@ -70,6 +83,7 @@ export type AtprotoSourceConfig = v.Infer<typeof atprotoSourceConfigSchema>;
 export type SourceConfig = v.Infer<typeof sourceConfigSchema>;
 export type PullConfig = v.Infer<typeof pullConfigSchema>;
 export type ExportConfig = v.Infer<typeof exportConfigSchema>;
+export type FormatterConfig = v.Infer<typeof formatterConfigSchema>;
 
 const isValidLexiconPattern = (pattern: string): boolean => {
 	if (pattern.endsWith('.*')) {
@@ -126,6 +140,7 @@ export const lexiconConfigSchema = v.object({
 		})
 		.partial()
 		.optional(),
+	formatter: formatterConfigSchema.optional((): FormatterConfig => ({ type: 'prettier' })),
 	pull: pullConfigSchema.optional(),
 	export: exportConfigSchema.optional(),
 });

package/src/formatter.ts

@@ -0,0 +1,145 @@
+import { spawn } from 'node:child_process';
+import { availableParallelism } from 'node:os';
+
+import type { FormatterConfig } from './config.ts';
+
+/** formats source code */
+export interface Formatter {
+	/**
+	 * formats the given code
+	 * @param code source code to format
+	 * @param filepath filepath hint for language detection and config resolution
+	 * @returns formatted code
+	 */
+	format(code: string, filepath: string): Promise<string>;
+}
+
+const inferPrettierParser = (filepath: string): string => {
+	if (filepath.endsWith('.ts') || filepath.endsWith('.tsx')) {
+		return 'typescript';
+	}
+	if (filepath.endsWith('.json')) {
+		return 'json';
+	}
+	if (filepath.endsWith('.md') || filepath.endsWith('.markdown')) {
+		return 'markdown';
+	}
+	return 'typescript';
+};
+
+// #region semaphore
+
+interface Lock {
+	release(): void;
+}
+
+class Semaphore {
+	private waiting: (() => void)[] = [];
+	private active = 0;
+	private max: number;
+
+	constructor(max: number) {
+		this.max = max;
+	}
+
+	acquire(): Promise<Lock> {
+		const lock: Lock = {
+			release: () => {
+				this.active--;
+				const next = this.waiting.shift();
+				if (next) {
+					this.active++;
+					next();
+				}
+			},
+		};
+
+		if (this.active < this.max) {
+			this.active++;
+			return Promise.resolve(lock);
+		}
+
+		const { promise, resolve } = Promise.withResolvers<Lock>();
+		this.waiting.push(() => resolve(lock));
+		return promise;
+	}
+}
+
+// #endregion
+
+/**
+ * creates a formatter from the given configuration
+ * @param config formatter configuration
+ * @param root project root for config resolution
+ * @returns a formatter instance
+ */
+export const createFormatter = async (config: FormatterConfig, root: string): Promise<Formatter> => {
+	let inner: Formatter;
+	let concurrency: number;
+
+	if (config.type === 'prettier') {
+		const prettier = await import('prettier');
+		const prettierConfig = await prettier.resolveConfig(root, { editorconfig: true });
+
+		// prettier is in-process and CPU-bound, so concurrency only helps
+		// avoid buffering all files in memory at once
+		concurrency = availableParallelism();
+		inner = {
+			async format(code, filepath) {
+				return prettier.format(code, { ...prettierConfig, parser: inferPrettierParser(filepath) });
+			},
+		};
+	} else {
+		// the template uses {filepath} as a placeholder, which is passed as a
+		// positional argument to sh to avoid shell injection via filenames
+		const shellCmd = config.command.replaceAll('{filepath}', '"$1"');
+
+		concurrency = config.concurrency;
+		inner = {
+			format(code, filepath) {
+				return new Promise<string>((resolve, reject) => {
+					const child = spawn('sh', ['-c', shellCmd, 'sh', filepath], {
+						stdio: ['pipe', 'pipe', 'pipe'],
+					});
+
+					const stdoutChunks: Buffer[] = [];
+					const stderrChunks: Buffer[] = [];
+
+					child.stdout.on('data', (chunk: Buffer) => {
+						stdoutChunks.push(chunk);
+					});
+
+					child.stderr.on('data', (chunk: Buffer) => {
+						stderrChunks.push(chunk);
+					});
+
+					child.on('error', reject);
+
+					child.on('close', (exitCode: number | null) => {
+						if (exitCode !== 0) {
+							const stderr = Buffer.concat(stderrChunks).toString();
+							reject(new Error(`formatter exited with code ${exitCode}:\n${stderr}`));
+						} else {
+							resolve(Buffer.concat(stdoutChunks).toString());
+						}
+					});
+
+					child.stdin.end(code);
+				});
+			},
+		};
+	}
+
+	const semaphore = new Semaphore(concurrency);
+
+	return {
+		async format(code, filepath) {
+			const lock = await semaphore.acquire();
+			try {
+				return await inner.format(code, filepath);
+			} finally {
+				lock.release();
+			}
+		},
+	};
+};

package/src/index.ts

@@ -1,4 +1,4 @@
-import { lexiconConfigSchema, type LexiconConfig } from './config.js';
+import { lexiconConfigSchema, type LexiconConfig } from './config.ts';
 
 export type { LexiconConfig };
 

package/src/pull-sources/atproto.ts

@@ -20,9 +20,9 @@ import {
 
 import pc from 'picocolors';
 
-import type { AtprotoSourceConfig } from '../config.js';
+import type { AtprotoSourceConfig } from '../config.ts';
 
-import type { PullResult, SourceLocation } from './types.js';
+import type { PullResult, SourceLocation } from './types.ts';
 
 /**
  * discovers all published lexicons for an authority by listing records in the

package/src/pull-sources/git.ts

@@ -6,10 +6,10 @@ import type { LexiconDoc } from '@atcute/lexicon-doc';
 
 import pc from 'picocolors';
 
-import type { GitSourceConfig } from '../config.js';
-import { runGit, GitError } from '../git.js';
+import type { GitSourceConfig } from '../config.ts';
+import { runGit, GitError } from '../git.ts';
 
-import type { PullResult, PulledLexicon, SourceLocation } from './types.js';
+import type { PullResult, PulledLexicon, SourceLocation } from './types.ts';
 
 /**
  * pulls lexicon documents from a git repository source