@atcute/lex-cli 2.4.0 → 2.5.0

package/src/{pull.ts → commands/pull.ts}

@@ -1,34 +1,39 @@
 import * as fs from 'node:fs/promises';
-import * as os from 'node:os';
 import * as path from 'node:path';
 
 import { lexiconDoc, refineLexiconDoc, type LexiconDoc } from '@atcute/lexicon-doc';
-import prettier from 'prettier';
+import { merge, object } from '@optique/core/constructs';
+import { message } from '@optique/core/message';
+import { type InferValue } from '@optique/core/parser';
+import { command, constant } from '@optique/core/primitives';
 import pc from 'picocolors';
+import prettier from 'prettier';
 
-import { runGit, GitError } from './git.js';
-import type { NormalizedConfig, PullConfig, SourceConfig } from './config.js';
+import { loadConfig, type NormalizedConfig, type PullConfig, type SourceConfig } from '../config.js';
+import { pullAtprotoSource } from '../pull-sources/atproto.js';
+import { pullGitSource } from '../pull-sources/git.js';
+import type { PullResult, PulledLexicon, SourceLocation } from '../pull-sources/types.js';
+import { sharedOptions } from '../shared-options.js';
+
+export const pullCommandSchema = command(
+	'pull',
+	merge(
+		object({
+			type: constant('pull'),
+		}),
+		sharedOptions,
+	),
+	{
+		brief: message`pull lexicon documents from configured sources`,
+		description: message`fetches lexicon documents from configured git repositories and writes them to the output directory.`,
+	},
+);
+
+export type PullCommand = InferValue<typeof pullCommandSchema>;
 
 interface SourceRevision {
 	source: SourceConfig;
-	rev: string;
-}
-
-interface SourceLocation {
-	absolutePath: string;
-	relativePath: string;
-	sourceDescription: string;
-}
-
-interface PulledLexicon {
-	nsid: string;
-	doc: LexiconDoc;
-	location: SourceLocation;
-}
-
-interface PullResult {
-	pulled: Map<string, PulledLexicon>;
-	rev: string;
+	rev?: string;
 }
 
 const ensurePullConfig = (config: NormalizedConfig): PullConfig => {
@@ -119,99 +124,13 @@ const writeLexicon = async (
 	await fs.writeFile(target, code);
 };
 
-/**
- * pulls lexicon documents from a git repository source
- * @param source git source configuration
- * @returns pulled lexicons and commit hash
- */
-const pullGitSource = async (source: SourceConfig & { type: 'git' }): Promise<PullResult> => {
-	const tempParent = await fs.mkdtemp(path.join(os.tmpdir(), 'lex-cli-pull-'));
-
-	const cloneDir = path.join(tempParent, 'repo');
-
-	try {
-		await runGit(
-			[
-				'clone',
-				'--filter=blob:none',
-				'--depth',
-				'1',
-				'--sparse',
-				...(source.ref ? ['--branch', source.ref, '--single-branch'] : []),
-				source.remote,
-				cloneDir,
-			],
-			{ timeoutMs: 60_000 },
-		);
-	} catch (err) {
-		if (err instanceof GitError) {
-			console.error(pc.bold(pc.red(`git clone failed for ${source.remote}:`)));
-			console.error(err.stderr || err.message);
-			process.exit(1);
-		}
-
-		throw err;
-	}
-
-	try {
-		await runGit(['-C', cloneDir, 'sparse-checkout', 'set', '--no-cone', ...source.pattern], {
-			timeoutMs: 30_000,
-		});
-	} catch (err) {
-		if (err instanceof GitError) {
-			console.error(pc.bold(pc.red(`git sparse-checkout failed for ${source.remote}:`)));
-			console.error(err.stderr || err.message);
-			process.exit(1);
-		}
-
-		throw err;
-	}
-
-	const pulled = new Map<string, PulledLexicon>();
-
-	for await (const filename of fs.glob(source.pattern, { cwd: cloneDir })) {
-		const absolute = path.join(cloneDir, filename);
-		const stat = await fs.stat(absolute);
-
-		if (!stat.isFile()) {
-			continue;
-		}
-
-		const location: SourceLocation = {
-			absolutePath: absolute,
-			relativePath: filename,
-			sourceDescription: source.remote,
-		};
-
-		const doc = await parseLexiconFile(location);
-
-		pulled.set(doc.id, { nsid: doc.id, doc, location });
-	}
-
-	// get the commit hash
-	let rev: string;
-	try {
-		const result = await runGit(['-C', cloneDir, 'rev-parse', 'HEAD'], { timeoutMs: 10_000 });
-		rev = result.stdout.trim();
-	} catch (err) {
-		if (err instanceof GitError) {
-			console.error(pc.bold(pc.red(`git rev-parse failed for ${source.remote}:`)));
-			console.error(err.stderr || err.message);
-			process.exit(1);
-		}
-
-		throw err;
-	}
-
-	await fs.rm(tempParent, { recursive: true, force: true });
-
-	return { pulled, rev };
-};
-
 const pullSource = async (source: SourceConfig): Promise<PullResult> => {
 	switch (source.type) {
 		case 'git': {
-			return pullGitSource(source);
+			return pullGitSource(source, parseLexiconFile);
+		}
+		case 'atproto': {
+			return pullAtprotoSource(source);
 		}
 	}
 };
@@ -232,7 +151,19 @@ const writeSourceReadme = async (
 		switch (source.type) {
 			case 'git': {
 				lines.push(`- ${source.remote}${source.ref ? ` (ref: ${source.ref})` : ``}`);
-				lines.push(`  - commit: ${rev}`);
+				if (rev) {
+					lines.push(`  - commit: ${rev}`);
+				}
+				break;
+			}
+			case 'atproto': {
+				if (source.mode === 'nsids') {
+					lines.push(`- atproto (nsids: ${source.nsids.join(', ')})`);
+				} else {
+					lines.push(
+						`- atproto (authority: ${source.authority}${source.pattern ? `, pattern: ${source.pattern.join(', ')}` : ''})`,
+					);
+				}
 				break;
 			}
 		}
@@ -250,11 +181,13 @@ const writeSourceReadme = async (
 };
 
 /**
- * pulls lexicon documents from configured sources and writes them to disk using nsid-based paths.
- * @param config normalized lex-cli configuration
+ * runs the pull command to fetch lexicon documents from configured sources
+ * @param args parsed command arguments
  */
-export const runPull = async (config: NormalizedConfig): Promise<void> => {
+export const runPull = async (args: PullCommand): Promise<void> => {
+	const config = await loadConfig(args.config);
 	const pullConfig = ensurePullConfig(config);
+
 	const outdir = path.resolve(config.root, pullConfig.outdir);
 	const prettierConfig = await prettier.resolveConfig(config.root, { editorconfig: true });
 

package/src/config.ts

@@ -1,10 +1,12 @@
+import * as fs from 'node:fs/promises';
 import * as path from 'node:path';
 import * as url from 'node:url';
 
 import * as v from '@badrap/valita';
 import pc from 'picocolors';
 
-import { isNsid } from '@atcute/lexicons/syntax';
+import { isAtprotoDid } from '@atcute/identity';
+import { isHandle, isNsid } from '@atcute/lexicons/syntax';
 
 import type { ImportMapping } from './codegen.js';
 
@@ -20,7 +22,32 @@ const gitSourceConfigSchema = v.object({
 		.assert((value) => value.length > 0, `must include at least one glob pattern`),
 });
 
-const sourceConfigSchema = v.union(gitSourceConfigSchema);
+const atprotoNsidsSourceConfigSchema = v.object({
+	type: v.literal('atproto'),
+	mode: v.literal('nsids'),
+	nsids: v
+		.array(v.string().assert((value) => isNsid(value), `must be valid nsid`))
+		.assert((value) => value.length > 0, `must include at least one nsid`),
+});
+
+const atprotoAuthoritySourceConfigSchema = v.object({
+	type: v.literal('atproto'),
+	mode: v.literal('authority'),
+	authority: v
+		.string()
+		.assert((value) => isHandle(value) || isAtprotoDid(value), `must a valid at-identifier`),
+	pattern: v
+		.array(
+			v
+				.string()
+				.assert((value) => isValidLexiconPattern(value), `must be valid nsid or pattern ending with .*`),
+		)
+		.optional(),
+});
+
+const atprotoSourceConfigSchema = v.union(atprotoNsidsSourceConfigSchema, atprotoAuthoritySourceConfigSchema);
+
+const sourceConfigSchema = v.union(gitSourceConfigSchema, atprotoSourceConfigSchema);
 
 const pullConfigSchema = v.object({
 	outdir: v.string().assert((value) => value.length > 0, `must not be empty`),
@@ -30,9 +57,19 @@ const pullConfigSchema = v.object({
 		.assert((value) => value.length > 0, `must include at least one source`),
 });
 
+const exportConfigSchema = v.object({
+	outdir: v.string().assert((value) => value.length > 0, `must not be empty`),
+	files: v.array(v.string().assert((value) => value.length > 0, `must not be empty`)).optional(),
+	clean: v.boolean().optional(),
+});
+
 export type GitSourceConfig = v.Infer<typeof gitSourceConfigSchema>;
+export type AtprotoNsidsSourceConfig = v.Infer<typeof atprotoNsidsSourceConfigSchema>;
+export type AtprotoAuthoritySourceConfig = v.Infer<typeof atprotoAuthoritySourceConfigSchema>;
+export type AtprotoSourceConfig = v.Infer<typeof atprotoSourceConfigSchema>;
 export type SourceConfig = v.Infer<typeof sourceConfigSchema>;
 export type PullConfig = v.Infer<typeof pullConfigSchema>;
+export type ExportConfig = v.Infer<typeof exportConfigSchema>;
 
 const isValidLexiconPattern = (pattern: string): boolean => {
 	if (pattern.endsWith('.*')) {
@@ -90,6 +127,7 @@ export const lexiconConfigSchema = v.object({
 		.partial()
 		.optional(),
 	pull: pullConfigSchema.optional(),
+	export: exportConfigSchema.optional(),
 });
 
 export type LexiconConfig = v.Infer<typeof lexiconConfigSchema>;
@@ -98,8 +136,33 @@ export interface NormalizedConfig extends LexiconConfig {
 	root: string;
 }
 
-export const loadConfig = async (configPath: string): Promise<NormalizedConfig> => {
-	const configFilename = path.resolve(configPath);
+export const loadConfig = async (configPath?: string): Promise<NormalizedConfig> => {
+	let configFilename: string | undefined;
+
+	if (configPath) {
+		configFilename = path.resolve(configPath);
+	} else {
+		// try to find lex.config.js or lex.config.ts in the current directory
+		const candidates = ['lex.config.js', 'lex.config.ts'];
+
+		for (const candidate of candidates) {
+			const candidatePath = path.resolve(candidate);
+			try {
+				await fs.access(candidatePath);
+				configFilename = candidatePath;
+				break;
+			} catch {
+				// file doesn't exist, try next candidate
+			}
+		}
+
+		if (!configFilename) {
+			console.error(pc.bold(pc.red(`config file not found`)));
+			console.error(`looked for: ${candidates.join(', ')}`);
+			process.exit(1);
+		}
+	}
+
 	const configDirname = path.dirname(configFilename);
 
 	let rawConfig: unknown;

package/src/lexicon-loader.ts

@@ -0,0 +1,199 @@
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import * as url from 'node:url';
+
+import { lexiconDoc, refineLexiconDoc, type LexiconDoc } from '@atcute/lexicon-doc';
+import { build, type LexDocumentBuilder } from '@atcute/lexicon-doc/builder';
+
+import pc from 'picocolors';
+
+/** file extensions recognized as module files */
+const MODULE_EXTENSIONS = new Set(['.js', '.cjs', '.mjs', '.ts', '.cts', '.mts']);
+
+/**
+ * represents a loaded lexicon document with its source file
+ */
+export interface LoadedLexicon {
+	nsid: string;
+	doc: LexiconDoc;
+	filename: string;
+}
+
+/**
+ * checks if a filename is a module file based on extension
+ * @param filename the filename to check
+ * @returns true if it's a module file
+ */
+const isModuleFile = (filename: string): boolean => {
+	const ext = path.extname(filename);
+	return MODULE_EXTENSIONS.has(ext);
+};
+
+/**
+ * basic validation that a value looks like a LexDocumentBuilder
+ * @param value the value to check
+ * @returns true if it appears to be a LexDocumentBuilder
+ */
+const isLexDocumentBuilder = (value: unknown): value is LexDocumentBuilder => {
+	return (
+		typeof value === 'object' &&
+		value !== null &&
+		'id' in value &&
+		typeof (value as any).id === 'string' &&
+		'defs' in value &&
+		typeof (value as any).defs === 'object'
+	);
+};
+
+/**
+ * loads and validates a lexicon document from a JSON file
+ * @param absolutePath absolute path to the JSON file
+ * @param relativePath relative path for error messages
+ * @returns parsed and validated lexicon document
+ */
+const loadJsonFile = async (absolutePath: string, relativePath: string): Promise<LexiconDoc> => {
+	let source: string;
+	try {
+		source = await fs.readFile(absolutePath, 'utf8');
+	} catch (err) {
+		console.error(pc.bold(pc.red(`file read error with "${relativePath}"`)));
+		console.error(err);
+		process.exit(1);
+	}
+
+	let json: unknown;
+	try {
+		json = JSON.parse(source);
+	} catch (err) {
+		console.error(pc.bold(pc.red(`json parse error in "${relativePath}"`)));
+		console.error(err);
+		process.exit(1);
+	}
+
+	const result = lexiconDoc.try(json, { mode: 'strip' });
+	if (!result.ok) {
+		console.error(pc.bold(pc.red(`schema validation failed for "${relativePath}"`)));
+		console.error(result.message);
+
+		for (const issue of result.issues) {
+			console.log(`- ${issue.code} at .${issue.path.join('.')}`);
+		}
+
+		process.exit(1);
+	}
+
+	const issues = refineLexiconDoc(result.value, true);
+	if (issues.length > 0) {
+		console.error(pc.bold(pc.red(`lint validation failed for "${relativePath}"`)));
+
+		for (const issue of issues) {
+			console.log(`- ${issue.message} at .${issue.path.join('.')}`);
+		}
+
+		process.exit(1);
+	}
+
+	return result.value;
+};
+
+/**
+ * loads a LexDocumentBuilder from a module file
+ * @param absolutePath absolute path to the module file
+ * @param relativePath relative path for error messages
+ * @returns the LexDocumentBuilder from the module's default export
+ */
+const loadModuleBuilder = async (absolutePath: string, relativePath: string): Promise<LexDocumentBuilder> => {
+	let mod: unknown;
+	try {
+		const fileUrl = url.pathToFileURL(absolutePath);
+		mod = await import(fileUrl.href);
+	} catch (err) {
+		console.error(pc.bold(pc.red(`failed to import module "${relativePath}"`)));
+		console.error(err);
+		process.exit(1);
+	}
+
+	const defaultExport = (mod as any)?.default;
+	if (!isLexDocumentBuilder(defaultExport)) {
+		console.error(pc.bold(pc.red(`module "${relativePath}" default export is not a valid LexDocumentBuilder`)));
+		console.error(`expected default export to be a LexDocumentBuilder (object with 'id' and 'defs')`);
+		process.exit(1);
+	}
+
+	return defaultExport;
+};
+
+/**
+ * loads lexicon documents from glob patterns
+ * @param patterns glob patterns to match files
+ * @param root root directory for resolving paths
+ * @returns array of loaded lexicon documents
+ */
+export const loadLexicons = async (patterns: string[], root: string): Promise<LoadedLexicon[]> => {
+	const results: LoadedLexicon[] = [];
+	const seen = new Map<string, string>();
+
+	// collect JSON docs and module builders separately
+	const jsonDocs: Array<{ doc: LexiconDoc; filename: string }> = [];
+	const moduleBuilders: Array<{ builder: LexDocumentBuilder; filename: string }> = [];
+
+	for await (const filename of fs.glob(patterns, { cwd: root })) {
+		const absolutePath = path.join(root, filename);
+
+		if (isModuleFile(filename)) {
+			const builder = await loadModuleBuilder(absolutePath, filename);
+			moduleBuilders.push({ builder, filename });
+		} else {
+			// assume JSON for anything else (including .json)
+			const doc = await loadJsonFile(absolutePath, filename);
+			jsonDocs.push({ doc, filename });
+		}
+	}
+
+	// add JSON docs directly (already built)
+	for (const { doc, filename } of jsonDocs) {
+		const existing = seen.get(doc.id);
+		if (existing) {
+			console.error(pc.bold(pc.red(`duplicate lexicon "${doc.id}"`)));
+			console.error(`- found in ${filename}`);
+			console.error(`- already found in ${existing}`);
+			process.exit(1);
+		}
+
+		seen.set(doc.id, filename);
+		results.push({ nsid: doc.id, doc, filename });
+	}
+
+	// build all module builders together (for cross-references)
+	if (moduleBuilders.length > 0) {
+		// check for duplicates in builders
+		const buildersByNsid = new Map<string, string>();
+		for (const { builder, filename } of moduleBuilders) {
+			const existing = buildersByNsid.get(builder.id) ?? seen.get(builder.id);
+			if (existing) {
+				console.error(pc.bold(pc.red(`duplicate lexicon "${builder.id}"`)));
+				console.error(`- found in ${filename}`);
+				console.error(`- already found in ${existing}`);
+				process.exit(1);
+			}
+			buildersByNsid.set(builder.id, filename);
+		}
+
+		let built: Record<string, LexiconDoc>;
+		try {
+			built = build({ documents: moduleBuilders.map((m) => m.builder) });
+		} catch (err) {
+			console.error(pc.bold(pc.red(`build failed for module lexicons`)));
+			console.error(err);
+			process.exit(1);
+		}
+
+		for (const { builder, filename } of moduleBuilders) {
+			const doc = built[builder.id];
+			seen.set(builder.id, filename);
+			results.push({ nsid: builder.id, doc, filename });
+		}
+	}
+
+	return results;
+};