@atcute/lex-cli 2.3.3 → 2.5.0

package/src/commands/pull.ts

@@ -0,0 +1,231 @@
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+
+import { lexiconDoc, refineLexiconDoc, type LexiconDoc } from '@atcute/lexicon-doc';
+import { merge, object } from '@optique/core/constructs';
+import { message } from '@optique/core/message';
+import { type InferValue } from '@optique/core/parser';
+import { command, constant } from '@optique/core/primitives';
+import pc from 'picocolors';
+import prettier from 'prettier';
+
+import { loadConfig, type NormalizedConfig, type PullConfig, type SourceConfig } from '../config.js';
+import { pullAtprotoSource } from '../pull-sources/atproto.js';
+import { pullGitSource } from '../pull-sources/git.js';
+import type { PullResult, PulledLexicon, SourceLocation } from '../pull-sources/types.js';
+import { sharedOptions } from '../shared-options.js';
+
+export const pullCommandSchema = command(
+	'pull',
+	merge(
+		object({
+			type: constant('pull'),
+		}),
+		sharedOptions,
+	),
+	{
+		brief: message`pull lexicon documents from configured sources`,
+		description: message`fetches lexicon documents from configured git repositories and writes them to the output directory.`,
+	},
+);
+
+export type PullCommand = InferValue<typeof pullCommandSchema>;
+
+interface SourceRevision {
+	source: SourceConfig;
+	rev?: string;
+}
+
+const ensurePullConfig = (config: NormalizedConfig): PullConfig => {
+	if (!config.pull) {
+		console.error(pc.bold(pc.red(`pull configuration missing`)));
+		process.exit(1);
+	}
+
+	return config.pull;
+};
+
+const parseLexiconFile = async (loc: SourceLocation): Promise<LexiconDoc> => {
+	let source: string;
+
+	try {
+		source = await fs.readFile(loc.absolutePath, 'utf8');
+	} catch (err) {
+		console.error(
+			pc.bold(pc.red(`file read error for ${loc.relativePath} when pulling ${loc.sourceDescription}`)),
+		);
+		console.error(`found in ${loc.absolutePath}`);
+		console.error(err);
+		process.exit(1);
+	}
+
+	let json: unknown;
+	try {
+		json = JSON.parse(source);
+	} catch (err) {
+		console.error(
+			pc.bold(pc.red(`json parse error in ${loc.relativePath} when pulling ${loc.sourceDescription}`)),
+		);
+		console.error(`found in ${loc.absolutePath}`);
+		console.error(err);
+		process.exit(1);
+	}
+
+	const result = lexiconDoc.try(json, { mode: 'passthrough' });
+	if (!result.ok) {
+		console.error(
+			pc.bold(
+				pc.red(`schema validation failed for ${loc.relativePath} when pulling ${loc.sourceDescription}`),
+			),
+		);
+		console.error(`found in ${loc.absolutePath}`);
+		console.error(result.message);
+
+		for (const issue of result.issues) {
+			console.log(`- ${issue.code} at .${issue.path.join('.')}`);
+		}
+
+		process.exit(1);
+	}
+
+	const issues = refineLexiconDoc(result.value, true);
+	if (issues.length > 0) {
+		console.error(
+			pc.bold(pc.red(`lint validation failed for ${loc.relativePath} when pulling ${loc.sourceDescription}`)),
+		);
+		console.error(`found in ${loc.absolutePath}`);
+
+		for (const issue of issues) {
+			console.log(`- ${issue.message} at .${issue.path.join('.')}`);
+		}
+
+		process.exit(1);
+	}
+
+	return result.value;
+};
+
+const writeLexicon = async (
+	outdir: string,
+	nsid: string,
+	doc: LexiconDoc,
+	prettierConfig: prettier.Options | null,
+): Promise<void> => {
+	const nsidPath = nsid.replaceAll('.', '/');
+	const target = path.join(outdir, `${nsidPath}.json`);
+	const dirname = path.dirname(target);
+
+	const code = await prettier.format(JSON.stringify(doc, null, 2), {
+		...(prettierConfig ?? {}),
+		parser: 'json',
+	});
+
+	await fs.mkdir(dirname, { recursive: true });
+	await fs.writeFile(target, code);
+};
+
+const pullSource = async (source: SourceConfig): Promise<PullResult> => {
+	switch (source.type) {
+		case 'git': {
+			return pullGitSource(source, parseLexiconFile);
+		}
+		case 'atproto': {
+			return pullAtprotoSource(source);
+		}
+	}
+};
+
+const writeSourceReadme = async (
+	outdir: string,
+	revisions: SourceRevision[],
+	prettierConfig: prettier.Options | null,
+): Promise<void> => {
+	const lines = [
+		'# lexicon sources',
+		'',
+		'this directory contains lexicon documents pulled from the following sources:',
+		'',
+	];
+
+	for (const { source, rev } of revisions) {
+		switch (source.type) {
+			case 'git': {
+				lines.push(`- ${source.remote}${source.ref ? ` (ref: ${source.ref})` : ``}`);
+				if (rev) {
+					lines.push(`  - commit: ${rev}`);
+				}
+				break;
+			}
+			case 'atproto': {
+				if (source.mode === 'nsids') {
+					lines.push(`- atproto (nsids: ${source.nsids.join(', ')})`);
+				} else {
+					lines.push(
+						`- atproto (authority: ${source.authority}${source.pattern ? `, pattern: ${source.pattern.join(', ')}` : ''})`,
+					);
+				}
+				break;
+			}
+		}
+	}
+
+	lines.push('');
+
+	const content = lines.join('\n');
+	const formatted = await prettier.format(content, {
+		...(prettierConfig ?? {}),
+		parser: 'markdown',
+	});
+
+	await fs.writeFile(path.join(outdir, 'README.md'), formatted);
+};
+
+/**
+ * runs the pull command to fetch lexicon documents from configured sources
+ * @param args parsed command arguments
+ */
+export const runPull = async (args: PullCommand): Promise<void> => {
+	const config = await loadConfig(args.config);
+	const pullConfig = ensurePullConfig(config);
+
+	const outdir = path.resolve(config.root, pullConfig.outdir);
+	const prettierConfig = await prettier.resolveConfig(config.root, { editorconfig: true });
+
+	const seen = new Map<string, SourceLocation>();
+	const collected: PulledLexicon[] = [];
+	const sourceRevisions: SourceRevision[] = [];
+
+	for (const source of pullConfig.sources) {
+		const result = await pullSource(source);
+
+		sourceRevisions.push({ source, rev: result.rev });
+
+		for (const [nsid, entry] of result.pulled) {
+			const existing = seen.get(nsid);
+
+			if (existing) {
+				console.error(pc.bold(pc.red(`duplicate lexicon "${nsid}"`)));
+				console.error(`- found ${entry.location.relativePath} from ${entry.location.sourceDescription}`);
+				console.error(`  at ${entry.location.absolutePath}`);
+				console.error(`- already found ${existing.relativePath} from ${existing.sourceDescription}`);
+				console.error(`  at ${existing.absolutePath}`);
+				process.exit(1);
+			}
+
+			seen.set(nsid, entry.location);
+			collected.push(entry);
+		}
+	}
+
+	if (pullConfig.clean) {
+		await fs.rm(outdir, { recursive: true, force: true });
+	}
+
+	await fs.mkdir(outdir, { recursive: true });
+
+	for (const entry of collected) {
+		await writeLexicon(outdir, entry.nsid, entry.doc, prettierConfig);
+	}
+
+	await writeSourceReadme(outdir, sourceRevisions, prettierConfig);
+};

package/src/config.ts

@@ -1,13 +1,76 @@
+import * as fs from 'node:fs/promises';
 import * as path from 'node:path';
 import * as url from 'node:url';
 
 import * as v from '@badrap/valita';
 import pc from 'picocolors';
 
-import { isNsid } from '@atcute/lexicons/syntax';
+import { isAtprotoDid } from '@atcute/identity';
+import { isHandle, isNsid } from '@atcute/lexicons/syntax';
 
 import type { ImportMapping } from './codegen.js';
 
+const gitSourceConfigSchema = v.object({
+	type: v.literal('git'),
+	remote: v.string().assert((value) => value.length > 0, `must not be empty`),
+	ref: v
+		.string()
+		.assert((value) => value.length > 0, `must not be empty`)
+		.optional(),
+	pattern: v
+		.array(v.string().assert((value) => value.length > 0, `must not be empty`))
+		.assert((value) => value.length > 0, `must include at least one glob pattern`),
+});
+
+const atprotoNsidsSourceConfigSchema = v.object({
+	type: v.literal('atproto'),
+	mode: v.literal('nsids'),
+	nsids: v
+		.array(v.string().assert((value) => isNsid(value), `must be valid nsid`))
+		.assert((value) => value.length > 0, `must include at least one nsid`),
+});
+
+const atprotoAuthoritySourceConfigSchema = v.object({
+	type: v.literal('atproto'),
+	mode: v.literal('authority'),
+	authority: v
+		.string()
+		.assert((value) => isHandle(value) || isAtprotoDid(value), `must a valid at-identifier`),
+	pattern: v
+		.array(
+			v
+				.string()
+				.assert((value) => isValidLexiconPattern(value), `must be valid nsid or pattern ending with .*`),
+		)
+		.optional(),
+});
+
+const atprotoSourceConfigSchema = v.union(atprotoNsidsSourceConfigSchema, atprotoAuthoritySourceConfigSchema);
+
+const sourceConfigSchema = v.union(gitSourceConfigSchema, atprotoSourceConfigSchema);
+
+const pullConfigSchema = v.object({
+	outdir: v.string().assert((value) => value.length > 0, `must not be empty`),
+	clean: v.boolean().optional(),
+	sources: v
+		.array(sourceConfigSchema)
+		.assert((value) => value.length > 0, `must include at least one source`),
+});
+
+const exportConfigSchema = v.object({
+	outdir: v.string().assert((value) => value.length > 0, `must not be empty`),
+	files: v.array(v.string().assert((value) => value.length > 0, `must not be empty`)).optional(),
+	clean: v.boolean().optional(),
+});
+
+export type GitSourceConfig = v.Infer<typeof gitSourceConfigSchema>;
+export type AtprotoNsidsSourceConfig = v.Infer<typeof atprotoNsidsSourceConfigSchema>;
+export type AtprotoAuthoritySourceConfig = v.Infer<typeof atprotoAuthoritySourceConfigSchema>;
+export type AtprotoSourceConfig = v.Infer<typeof atprotoSourceConfigSchema>;
+export type SourceConfig = v.Infer<typeof sourceConfigSchema>;
+export type PullConfig = v.Infer<typeof pullConfigSchema>;
+export type ExportConfig = v.Infer<typeof exportConfigSchema>;
+
 const isValidLexiconPattern = (pattern: string): boolean => {
 	if (pattern.endsWith('.*')) {
 		return isNsid(`${pattern.slice(0, -2)}.x`);
@@ -19,7 +82,7 @@ const isValidLexiconPattern = (pattern: string): boolean => {
 const mappingImports: v.Type<ImportMapping['imports']> = v.unknown().chain((value) => {
 	if (typeof value === 'string') {
 		if (value.length === 0) {
-			return v.err({ message: 'imports must not be empty' });
+			return v.err('imports must not be empty');
 		}
 
 		return v.ok(value);
@@ -29,7 +92,7 @@ const mappingImports: v.Type<ImportMapping['imports']> = v.unknown().chain((valu
 		return v.ok(value as ImportMapping['imports']);
 	}
 
-	return v.err({ message: 'imports must be a string or function' });
+	return v.err('imports must be a string or function');
 });
 
 const importMappingSchema: v.Type<ImportMapping> = v.object({
@@ -37,50 +100,34 @@ const importMappingSchema: v.Type<ImportMapping> = v.object({
 		.array(
 			v.string().chain((value) => {
 				if (!isValidLexiconPattern(value)) {
-					return v.err({
-						message: 'invalid NSID pattern (must be valid NSID or end with .*)',
-					});
+					return v.err(`invalid NSID pattern (must be valid NSID or end with .*)`);
 				}
 
 				return v.ok(value);
 			}),
 		)
-		.assert((patterns) => patterns.length > 0, {
-			message: 'nsid requires at least one pattern',
-		}),
+		.assert((patterns) => patterns.length > 0, `nsid requires at least one pattern`),
 	imports: mappingImports,
 });
 
 export const lexiconConfigSchema = v.object({
-	outdir: v.string().assert((value) => value.length > 0, {
-		message: 'outdir must not be empty',
-	}),
+	outdir: v.string().assert((value) => value.length > 0, `must not be empty`),
 	files: v
-		.array(
-			v.string().assert((value) => value.length > 0, {
-				message: 'files entries must not be empty',
-			}),
-		)
-		.assert((value) => value.length > 0, {
-			message: 'files must include at least one pattern',
-		}),
-	imports: v
-		.array(
-			v.string().assert((value) => value.length > 0, {
-				message: 'imports entries must not be empty',
-			}),
-		)
-		.optional(),
+		.array(v.string().assert((value) => value.length > 0, `must not be empty`))
+		.assert((value) => value.length > 0, `must include at least one glob pattern`),
+	imports: v.array(v.string().assert((value) => value.length > 0, `must not be empty`)).optional(),
 	mappings: v.array(importMappingSchema).optional(),
 	modules: v
 		.object({
 			importSuffix: v
 				.string()
-				.assert((value) => value.length > 0, { message: 'importSuffix must not be empty' })
+				.assert((value) => value.length > 0, `must not be empty`)
 				.optional(),
 		})
 		.partial()
 		.optional(),
+	pull: pullConfigSchema.optional(),
+	export: exportConfigSchema.optional(),
 });
 
 export type LexiconConfig = v.Infer<typeof lexiconConfigSchema>;
@@ -89,8 +136,33 @@ export interface NormalizedConfig extends LexiconConfig {
 	root: string;
 }
 
-export const loadConfig = async (configPath: string): Promise<NormalizedConfig> => {
-	const configFilename = path.resolve(configPath);
+export const loadConfig = async (configPath?: string): Promise<NormalizedConfig> => {
+	let configFilename: string | undefined;
+
+	if (configPath) {
+		configFilename = path.resolve(configPath);
+	} else {
+		// try to find lex.config.js or lex.config.ts in the current directory
+		const candidates = ['lex.config.js', 'lex.config.ts'];
+
+		for (const candidate of candidates) {
+			const candidatePath = path.resolve(candidate);
+			try {
+				await fs.access(candidatePath);
+				configFilename = candidatePath;
+				break;
+			} catch {
+				// file doesn't exist, try next candidate
+			}
+		}
+
+		if (!configFilename) {
+			console.error(pc.bold(pc.red(`config file not found`)));
+			console.error(`looked for: ${candidates.join(', ')}`);
+			process.exit(1);
+		}
+	}
+
 	const configDirname = path.dirname(configFilename);
 
 	let rawConfig: unknown;

package/src/git.ts

@@ -0,0 +1,104 @@
+import { spawn } from 'node:child_process';
+
+export interface GitCommandOptions {
+	cwd?: string;
+	env?: NodeJS.ProcessEnv;
+	stdin?: string;
+	timeoutMs?: number;
+}
+
+export interface GitResult {
+	stdout: string;
+	stderr: string;
+}
+
+export class GitError extends Error {
+	readonly args: readonly string[];
+	readonly code: number | null;
+	readonly signal: NodeJS.Signals | null;
+	readonly stdout: string;
+	readonly stderr: string;
+
+	constructor(
+		args: readonly string[],
+		stdout: string,
+		stderr: string,
+		code: number | null,
+		signal: NodeJS.Signals | null,
+	) {
+		const reason = code !== null ? `code ${code}` : `signal ${signal ?? 'unknown'}`;
+		super(`git ${args.join(' ')} failed with ${reason}`);
+
+		this.name = 'GitError';
+		this.args = args;
+		this.code = code;
+		this.signal = signal;
+		this.stdout = stdout;
+		this.stderr = stderr;
+	}
+}
+
+/**
+ * runs git with the provided arguments and throws when the command fails.
+ * @param args positional arguments for git
+ * @param options execution options
+ * @returns stdout and stderr from git
+ * @throws GitError when git exits with a non-zero status or is terminated
+ */
+export const runGit = (args: readonly string[], options: GitCommandOptions = {}): Promise<GitResult> => {
+	return new Promise((resolve, reject) => {
+		const child = spawn('git', args, {
+			cwd: options.cwd,
+			env: options.env,
+			stdio: ['pipe', 'pipe', 'pipe'],
+		});
+
+		let stdout = '';
+		let stderr = '';
+
+		child.stdout?.setEncoding('utf8');
+		child.stderr?.setEncoding('utf8');
+
+		child.stdout?.on('data', (chunk) => {
+			stdout += chunk;
+		});
+
+		child.stderr?.on('data', (chunk) => {
+			stderr += chunk;
+		});
+
+		let timer: NodeJS.Timeout | undefined;
+		if (options.timeoutMs !== undefined) {
+			timer = setTimeout(() => {
+				child.kill('SIGKILL');
+			}, options.timeoutMs);
+		}
+
+		child.on('error', (err) => {
+			if (timer) {
+				clearTimeout(timer);
+			}
+
+			reject(err);
+		});
+
+		child.on('close', (code, signal) => {
+			if (timer) {
+				clearTimeout(timer);
+			}
+
+			if (code === 0) {
+				resolve({ stdout, stderr });
+				return;
+			}
+
+			reject(new GitError(args, stdout, stderr, code, signal));
+		});
+
+		if (options.stdin !== undefined) {
+			child.stdin?.end(options.stdin);
+		} else {
+			child.stdin?.end();
+		}
+	});
+};