@atcute/lex-cli 2.3.3 → 2.5.0

package/dist/pull-sources/atproto.js

@@ -0,0 +1,192 @@
+import { getPdsEndpoint, isAtprotoDid } from '@atcute/identity';
+import { CompositeDidDocumentResolver, CompositeHandleResolver, DohJsonHandleResolver, PlcDidDocumentResolver, WebDidDocumentResolver, WellKnownHandleResolver, } from '@atcute/identity-resolver';
+import { refineLexiconDoc } from '@atcute/lexicon-doc';
+import { DohJsonLexiconAuthorityResolver, LexiconSchemaResolver } from '@atcute/lexicon-resolver';
+import { isHandle, isNsid, parseCanonicalResourceUri, } from '@atcute/lexicons/syntax';
+import pc from 'picocolors';
+/**
+ * discovers all published lexicons for an authority by listing records in the
+ * com.atproto.lexicon.schema collection
+ * @param authority the authority DID
+ * @param didResolver DID document resolver
+ * @returns array of NSID strings
+ */
+const discoverLexiconsForAuthority = async (authority, didResolver) => {
+    // resolve DID to get PDS endpoint
+    const didDocument = await didResolver.resolve(authority);
+    const pdsEndpoint = getPdsEndpoint(didDocument);
+    if (!pdsEndpoint) {
+        throw new Error(`no pds service in did document; did=${authority}`);
+    }
+    // call com.atproto.repo.listRecords to get all lexicon schema records
+    const nsids = [];
+    let cursor;
+    do {
+        const url = new URL('/xrpc/com.atproto.repo.listRecords', pdsEndpoint);
+        url.searchParams.set('repo', authority);
+        url.searchParams.set('collection', 'com.atproto.lexicon.schema');
+        url.searchParams.set('limit', '100');
+        if (cursor) {
+            url.searchParams.set('cursor', cursor);
+        }
+        const response = await fetch(url, {
+            headers: { accept: 'application/json' },
+        });
+        if (!response.ok) {
+            throw new Error(`http ${response.status} when listing records`);
+        }
+        const data = (await response.json());
+        // extract NSIDs from record keys (the rkey in at://did/collection/rkey)
+        for (const record of data.records) {
+            const r = parseCanonicalResourceUri(record.uri);
+            if (!r.ok) {
+                continue;
+            }
+            const nsid = r.value.rkey;
+            if (!isNsid(nsid)) {
+                continue;
+            }
+            nsids.push(nsid);
+        }
+        cursor = data.cursor;
+    } while (cursor);
+    return nsids;
+};
+/**
+ * pulls lexicon documents from AT Protocol network resolution
+ * @param source atproto source configuration
+ * @returns pulled lexicons and ISO timestamp
+ */
+export const pullAtprotoSource = async (source) => {
+    // create resolver instances (reusable across NSIDs)
+    const authorityResolver = new DohJsonLexiconAuthorityResolver({
+        dohUrl: 'https://cloudflare-dns.com/dns-query',
+    });
+    const didResolver = new CompositeDidDocumentResolver({
+        methods: {
+            plc: new PlcDidDocumentResolver(),
+            web: new WebDidDocumentResolver(),
+        },
+    });
+    const schemaResolver = new LexiconSchemaResolver({
+        didDocumentResolver: didResolver,
+    });
+    const handleResolver = new CompositeHandleResolver({
+        strategy: 'race',
+        methods: {
+            http: new WellKnownHandleResolver(),
+            dns: new DohJsonHandleResolver({
+                dohUrl: 'https://cloudflare-dns.com/dns-query',
+            }),
+        },
+    });
+    const pulled = new Map();
+    const errors = [];
+    let nsids;
+    let authorityDid = null;
+    let sourceDesc;
+    let sourceName = null;
+    if (source.mode === 'nsids') {
+        nsids = source.nsids;
+        sourceDesc = `atproto (${nsids.length} nsids)`;
+    }
+    else {
+        // mode 2: authority-based
+        // step 2a: resolve authority (handle -> DID if needed)
+        let resolvedDid;
+        const handle = isHandle(source.authority) ? source.authority : null;
+        try {
+            if (isAtprotoDid(source.authority)) {
+                resolvedDid = source.authority;
+            }
+            else if (handle) {
+                resolvedDid = await handleResolver.resolve(handle);
+            }
+            else {
+                console.error(pc.bold(pc.red(`invalid authority: ${source.authority}`)));
+                console.error(`must be a valid DID or handle`);
+                process.exit(1);
+            }
+            authorityDid = resolvedDid;
+            sourceDesc = `atproto (authority: ${authorityDid})`;
+            sourceName = handle ?? authorityDid;
+        }
+        catch (err) {
+            console.error(pc.bold(pc.red(`failed to resolve authority: ${source.authority}`)));
+            console.error(err);
+            process.exit(1);
+        }
+        // step 2b: discover all lexicons for this authority
+        try {
+            nsids = await discoverLexiconsForAuthority(authorityDid, didResolver);
+        }
+        catch (err) {
+            console.error(pc.bold(pc.red(`failed to discover lexicons for ${sourceName}`)));
+            console.error(err);
+            process.exit(1);
+        }
+        // step 2c: filter by pattern if specified
+        if (source.pattern) {
+            nsids = nsids.filter((nsid) => {
+                return source.pattern.some((pattern) => {
+                    if (pattern.endsWith('.*')) {
+                        const prefix = pattern.slice(0, -2);
+                        return nsid === prefix || nsid.startsWith(prefix + '.');
+                    }
+                    return nsid === pattern;
+                });
+            });
+        }
+        if (nsids.length === 0) {
+            console.warn(pc.yellow(`warning: no lexicons found for ${sourceName}`));
+        }
+    }
+    // fetch each NSID
+    let fetchedCount = 0;
+    for (const nsid of nsids) {
+        try {
+            // step 1: resolve authority from NSID (DNS)
+            const resolvedAuthority = await authorityResolver.resolve(nsid);
+            // step 2: cross-verify authority if in authority-based mode
+            if (authorityDid && resolvedAuthority !== authorityDid) {
+                throw new Error(`authority mismatch: NSID ${nsid} claims authority ${resolvedAuthority} but expected ${authorityDid}`);
+            }
+            // step 3: fetch schema from authority's PDS
+            const resolved = await schemaResolver.resolve(resolvedAuthority, nsid);
+            // step 4: lint the lexicon document
+            const issues = refineLexiconDoc(resolved.schema, true);
+            if (issues.length > 0) {
+                const messages = issues.map((i) => `  ${i.path}: ${i.message}`).join('\n');
+                throw new Error(`lint validation failed:\n${messages}`);
+            }
+            // create in-memory location (no file on disk yet)
+            const location = {
+                absolutePath: `<atproto:${nsid}>`,
+                relativePath: `${nsid}.json`,
+                sourceDescription: sourceDesc,
+            };
+            pulled.set(nsid, {
+                nsid,
+                doc: resolved.schema,
+                location,
+            });
+            fetchedCount++;
+            console.log(`${pc.green('+')} ${nsid}`);
+        }
+        catch (err) {
+            // best-effort: collect errors but continue
+            errors.push({ nsid, error: err });
+        }
+    }
+    // report all errors at end
+    if (errors.length > 0) {
+        console.warn(pc.yellow(`\nwarning: failed to fetch ${errors.length} lexicon(s):`));
+        for (const { nsid, error } of errors) {
+            console.warn(`  - ${nsid}: ${error.message}`);
+        }
+    }
+    const suffix = sourceName ? ` from ${pc.cyan(sourceName)}` : '';
+    console.log(`pulled ${pc.cyan(fetchedCount.toString())} lexicons${suffix}`);
+    return { pulled };
+};
+//# sourceMappingURL=atproto.js.map

package/dist/pull-sources/atproto.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"atproto.js","sourceRoot":"","sources":["../../src/pull-sources/atproto.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAEhE,OAAO,EACN,4BAA4B,EAC5B,uBAAuB,EACvB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,GACvB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,gBAAgB,EAAmB,MAAM,qBAAqB,CAAC;AACxE,OAAO,EAAE,+BAA+B,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AAClG,OAAO,EACN,QAAQ,EACR,MAAM,EACN,yBAAyB,GAGzB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,MAAM,YAAY,CAAC;AAK5B;;;;;;GAMG;AACH,MAAM,4BAA4B,GAAG,KAAK,EACzC,SAAqB,EACrB,WAAgC,EACd,EAAE;IACpB,kCAAkC;IAClC,MAAM,WAAW,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACzD,MAAM,WAAW,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;IAEhD,IAAI,CAAC,WAAW,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,uCAAuC,SAAS,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,sEAAsE;IACtE,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,IAAI,MAA0B,CAAC;IAE/B,GAAG,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,oCAAoC,EAAE,WAAW,CAAC,CAAC;QACvE,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACxC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,EAAE,4BAA4B,CAAC,CAAC;QACjE,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACrC,IAAI,MAAM,EAAE,CAAC;YACZ,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YACjC,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE;SACvC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,uBAAuB,CAAC,CAAC;QACjE,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAGlC,CAAC;QAEF,wEAAwE;QACxE,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,CAAC,GAAG,yBAAyB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAChD,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;gBACX,SAAS;YACV,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,SAAS;YACV,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC;QAED,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IACtB,CAAC,QAAQ,MAAM,EAAE;IAEjB,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,EAAE,MAA2B,EAAuB,EAAE;IAC3F,oDAAoD;IACpD,MAAM,iBAAiB,GAAG,IAAI,+BAA+B,CAAC;QAC7D,MAAM,EAAE,sCAAsC;KAC9C,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,IAAI,4BAA4B,CAAC;QACpD,OAAO,EAAE;YACR,GAAG,EAAE,IAAI,sBAAsB,EAAE;YACjC,GAAG,EAAE,IAAI,sBAAsB,EAAE;SACjC;KACD,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,IAAI,qBAAqB,CAAC;QAChD,mBAAmB,EAAE,WAAW;KAChC,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,IAAI,uBAAuB,CAAC;QAClD,QAAQ,EAAE,MAAM;QAChB,OAAO,EAAE;YACR,IAAI,EAAE,IAAI,uBAAuB,EAAE;YACnC,GAAG,EAAE,IAAI,qBAAqB,CAAC;gBAC9B,MAAM,EAAE,sCAAsC;aAC9C,CAAC;SACF;KACD,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,IAAI,GAAG,EAAuE,CAAC;IAC9F,MAAM,MAAM,GAA0C,EAAE,CAAC;IAEzD,IAAI,KAAa,CAAC;IAClB,IAAI,YAAY,GAAsB,IAAI,CAAC;IAC3C,IAAI,UAAkB,CAAC;IACvB,IAAI,UAAU,GAAkB,IAAI,CAAC;IAErC,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC7B,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACrB,UAAU,GAAG,YAAY,KAAK,CAAC,MAAM,SAAS,CAAC;IAChD,CAAC;SAAM,CAAC;QACP,0BAA0B;QAC1B,uDAAuD;QACvD,IAAI,WAAuB,CAAC;QAC5B,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;QAEpE,IAAI,CAAC;YACJ,IAAI,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;gBACpC,WAAW,GAAG,MAAM,CAAC,SAAS,CAAC;YAChC,CAAC;iBAAM,IAAI,MAAM,EAAE,CAAC;gBACnB,WAAW,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACP,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,sBAAsB,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;gBACzE,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;gBAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjB,CAAC;YAED,YAAY,GAAG,WAAW,CAAC;YAC3B,UAAU,GAAG,uBAAuB,YAAY,GAAG,CAAC;YACpD,UAAU,GAAG,MAAM,IAAI,YAAY,CAAC;QACrC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,gCAAgC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;YACnF,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;QAED,oDAAoD;QACpD,IAAI,CAAC;YACJ,KAAK,GAAG,MAAM,4BAA4B,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;QACvE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,mCAAmC,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC;YAChF,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;QAED,0CAA0C;QAC1C,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;gBAC7B,OAAO,MAAM,CAAC,OAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;oBACvC,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;wBAC5B,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;wBACpC,OAAO,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;oBACzD,CAAC;oBACD,OAAO,IAAI,KAAK,OAAO,CAAC;gBACzB,CAAC,CAAC,CAAC;YACJ,CAAC,CAAC,CAAC;QACJ,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,kCAAkC,UAAU,EAAE,CAAC,CAAC,CAAC;QACzE,CAAC;IACF,CAAC;IAED,kBAAkB;IAClB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC;YACJ,4CAA4C;YAC5C,MAAM,iBAAiB,GAAG,MAAM,iBAAiB,CAAC,OAAO,CAAC,IAAY,CAAC,CAAC;YAExE,4DAA4D;YAC5D,IAAI,YAAY,IAAI,iBAAiB,KAAK,YAAY,EAAE,CAAC;gBACxD,MAAM,IAAI,KAAK,CACd,4BAA4B,IAAI,qBAAqB,iBAAiB,iBAAiB,YAAY,EAAE,CACrG,CAAC;YACH,CAAC;YAED,4CAA4C;YAC5C,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,iBAAiB,EAAE,IAAY,CAAC,CAAC;YAE/E,oCAAoC;YACpC,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACvD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC3E,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,EAAE,CAAC,CAAC;YACzD,CAAC;YAED,kDAAkD;YAClD,MAAM,QAAQ,GAAmB;gBAChC,YAAY,EAAE,YAAY,IAAI,GAAG;gBACjC,YAAY,EAAE,GAAG,IAAI,OAAO;gBAC5B,iBAAiB,EAAE,UAAU;aAC7B,CAAC;YAEF,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE;gBAChB,IAAI;gBACJ,GAAG,EAAE,QAAQ,CAAC,MAAM;gBACpB,QAAQ;aACR,CAAC,CAAC;YACH,YAAY,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,2CAA2C;YAC3C,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAY,EAAE,CAAC,CAAC;QAC5C,CAAC;IACF,CAAC;IAED,2BAA2B;IAC3B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,8BAA8B,MAAM,CAAC,MAAM,cAAc,CAAC,CAAC,CAAC;QACnF,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,MAAM,EAAE,CAAC;YACtC,OAAO,CAAC,IAAI,CAAC,OAAO,IAAI,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/C,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAChE,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,YAAY,MAAM,EAAE,CAAC,CAAC;IAE5E,OAAO,EAAE,MAAM,EAAE,CAAC;AACnB,CAAC,CAAC"}

package/dist/pull-sources/git.d.ts

@@ -0,0 +1,11 @@
+import type { LexiconDoc } from '@atcute/lexicon-doc';
+import type { GitSourceConfig } from '../config.js';
+import type { PullResult, SourceLocation } from './types.js';
+/**
+ * pulls lexicon documents from a git repository source
+ * @param source git source configuration
+ * @param parseLexiconFile function to parse and validate lexicon files
+ * @returns pulled lexicons and commit hash
+ */
+export declare const pullGitSource: (source: GitSourceConfig, parseLexiconFile: (loc: SourceLocation) => Promise<LexiconDoc>) => Promise<PullResult>;
+//# sourceMappingURL=git.d.ts.map

package/dist/pull-sources/git.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"git.d.ts","sourceRoot":"","sources":["../../src/pull-sources/git.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAItD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,KAAK,EAAE,UAAU,EAAiB,cAAc,EAAE,MAAM,YAAY,CAAC;AAE5E;;;;;GAKG;AACH,eAAO,MAAM,aAAa,GACzB,QAAQ,eAAe,EACvB,kBAAkB,CAAC,GAAG,EAAE,cAAc,KAAK,OAAO,CAAC,UAAU,CAAC,KAC5D,OAAO,CAAC,UAAU,CAkFpB,CAAC"}

package/dist/pull-sources/git.js

@@ -0,0 +1,80 @@
+import * as fs from 'node:fs/promises';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import pc from 'picocolors';
+import { runGit, GitError } from '../git.js';
+/**
+ * pulls lexicon documents from a git repository source
+ * @param source git source configuration
+ * @param parseLexiconFile function to parse and validate lexicon files
+ * @returns pulled lexicons and commit hash
+ */
+export const pullGitSource = async (source, parseLexiconFile) => {
+    const tempParent = await fs.mkdtemp(path.join(os.tmpdir(), 'lex-cli-pull-'));
+    const cloneDir = path.join(tempParent, 'repo');
+    try {
+        await runGit([
+            'clone',
+            '--filter=blob:none',
+            '--depth',
+            '1',
+            '--sparse',
+            ...(source.ref ? ['--branch', source.ref, '--single-branch'] : []),
+            source.remote,
+            cloneDir,
+        ], { timeoutMs: 60_000 });
+    }
+    catch (err) {
+        if (err instanceof GitError) {
+            console.error(pc.bold(pc.red(`git clone failed for ${source.remote}:`)));
+            console.error(err.stderr || err.message);
+            process.exit(1);
+        }
+        throw err;
+    }
+    try {
+        await runGit(['-C', cloneDir, 'sparse-checkout', 'set', '--no-cone', ...source.pattern], {
+            timeoutMs: 30_000,
+        });
+    }
+    catch (err) {
+        if (err instanceof GitError) {
+            console.error(pc.bold(pc.red(`git sparse-checkout failed for ${source.remote}:`)));
+            console.error(err.stderr || err.message);
+            process.exit(1);
+        }
+        throw err;
+    }
+    const pulled = new Map();
+    for await (const filename of fs.glob(source.pattern, { cwd: cloneDir })) {
+        const absolute = path.join(cloneDir, filename);
+        const stat = await fs.stat(absolute);
+        if (!stat.isFile()) {
+            continue;
+        }
+        const location = {
+            absolutePath: absolute,
+            relativePath: filename,
+            sourceDescription: source.remote,
+        };
+        const doc = await parseLexiconFile(location);
+        pulled.set(doc.id, { nsid: doc.id, doc, location });
+    }
+    // get the commit hash
+    let rev;
+    try {
+        const result = await runGit(['-C', cloneDir, 'rev-parse', 'HEAD'], { timeoutMs: 10_000 });
+        rev = result.stdout.trim();
+    }
+    catch (err) {
+        if (err instanceof GitError) {
+            console.error(pc.bold(pc.red(`git rev-parse failed for ${source.remote}:`)));
+            console.error(err.stderr || err.message);
+            process.exit(1);
+        }
+        throw err;
+    }
+    await fs.rm(tempParent, { recursive: true, force: true });
+    return { pulled, rev };
+};
+//# sourceMappingURL=git.js.map

package/dist/pull-sources/git.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"git.js","sourceRoot":"","sources":["../../src/pull-sources/git.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAGlC,OAAO,EAAE,MAAM,YAAY,CAAC;AAE5B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAI7C;;;;;GAKG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EACjC,MAAuB,EACvB,gBAA8D,EACxC,EAAE;IACxB,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,eAAe,CAAC,CAAC,CAAC;IAE7E,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAE/C,IAAI,CAAC;QACJ,MAAM,MAAM,CACX;YACC,OAAO;YACP,oBAAoB;YACpB,SAAS;YACT,GAAG;YACH,UAAU;YACV,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,MAAM,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAClE,MAAM,CAAC,MAAM;YACb,QAAQ;SACR,EACD,EAAE,SAAS,EAAE,MAAM,EAAE,CACrB,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,IAAI,GAAG,YAAY,QAAQ,EAAE,CAAC;YAC7B,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,wBAAwB,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YACzE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;YACzC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;QAED,MAAM,GAAG,CAAC;IACX,CAAC;IAED,IAAI,CAAC;QACJ,MAAM,MAAM,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,iBAAiB,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,EAAE;YACxF,SAAS,EAAE,MAAM;SACjB,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,IAAI,GAAG,YAAY,QAAQ,EAAE,CAAC;YAC7B,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,kCAAkC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YACnF,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;YACzC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;QAED,MAAM,GAAG,CAAC;IACX,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAEhD,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAErC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;YACpB,SAAS;QACV,CAAC;QAED,MAAM,QAAQ,GAAmB;YAChC,YAAY,EAAE,QAAQ;YACtB,YAAY,EAAE,QAAQ;YACtB,iBAAiB,EAAE,MAAM,CAAC,MAAM;SAChC,CAAC;QAEF,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAE7C,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,sBAAsB;IACtB,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1F,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IAC5B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,IAAI,GAAG,YAAY,QAAQ,EAAE,CAAC;YAC7B,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,4BAA4B,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YAC7E,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;YACzC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;QAED,MAAM,GAAG,CAAC;IACX,CAAC;IAED,MAAM,EAAE,CAAC,EAAE,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAE1D,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;AACxB,CAAC,CAAC"}

package/dist/pull-sources/types.d.ts

@@ -0,0 +1,16 @@
+import type { LexiconDoc } from '@atcute/lexicon-doc';
+export interface SourceLocation {
+    absolutePath: string;
+    relativePath: string;
+    sourceDescription: string;
+}
+export interface PulledLexicon {
+    nsid: string;
+    doc: LexiconDoc;
+    location: SourceLocation;
+}
+export interface PullResult {
+    pulled: Map<string, PulledLexicon>;
+    rev?: string;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/pull-sources/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/pull-sources/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEtD,MAAM,WAAW,cAAc;IAC9B,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,aAAa;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,UAAU,CAAC;IAChB,QAAQ,EAAE,cAAc,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IAC1B,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IACnC,GAAG,CAAC,EAAE,MAAM,CAAC;CACb"}

package/dist/pull-sources/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/pull-sources/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/pull-sources/types.ts"],"names":[],"mappings":""}

package/dist/shared-options.d.ts

@@ -0,0 +1,6 @@
+export declare const sharedOptions: import("@optique/core/parser").Parser<{
+    readonly config: string | undefined;
+}, {
+    readonly config: [import("@optique/core/valueparser").ValueParserResult<string> | undefined] | undefined;
+}>;
+//# sourceMappingURL=shared-options.d.ts.map

package/dist/shared-options.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"shared-options.d.ts","sourceRoot":"","sources":["../src/shared-options.ts"],"names":[],"mappings":"AAMA,eAAO,MAAM,aAAa;;;;EAMxB,CAAC"}

package/dist/shared-options.js

@@ -0,0 +1,11 @@
+import { object } from '@optique/core/constructs';
+import { message } from '@optique/core/message';
+import { optional } from '@optique/core/modifiers';
+import { option } from '@optique/core/primitives';
+import { path as pathParser } from '@optique/run/valueparser';
+export const sharedOptions = object(`Global options`, {
+    config: optional(option('-c', '--config', pathParser({ metavar: 'CONFIG' }), {
+        description: message `path to the lexicon configuration file. defaults to searching for lex.config.js or lex.config.ts in the current directory.`,
+    })),
+});
+//# sourceMappingURL=shared-options.js.map

package/dist/shared-options.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"shared-options.js","sourceRoot":"","sources":["../src/shared-options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,EAAE,IAAI,IAAI,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAE9D,MAAM,CAAC,MAAM,aAAa,GAAG,MAAM,CAAC,gBAAgB,EAAE;IACrD,MAAM,EAAE,QAAQ,CACf,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,UAAU,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE;QAC3D,WAAW,EAAE,OAAO,CAAA,4HAA4H;KAChJ,CAAC,CACF;CACD,CAAC,CAAC"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "type": "module",
   "name": "@atcute/lex-cli",
-  "version": "2.3.3",
+  "version": "2.5.0",
   "description": "cli tool to generate type definitions for atcute",
   "license": "0BSD",
   "repository": {
@@ -22,16 +22,19 @@
   },
   "dependencies": {
     "@badrap/valita": "^0.4.6",
-    "@optique/core": "^0.6.2",
-    "@optique/run": "^0.6.2",
+    "@optique/core": "^0.6.3",
+    "@optique/run": "^0.6.3",
     "picocolors": "^1.1.1",
     "prettier": "^3.6.2",
-    "@atcute/lexicon-doc": "^2.0.0"
+    "@atcute/identity": "^1.1.3",
+    "@atcute/lexicon-doc": "^2.0.3",
+    "@atcute/identity-resolver": "^1.1.4",
+    "@atcute/lexicon-resolver": "^0.1.4",
+    "@atcute/lexicons": "^1.2.5"
   },
   "devDependencies": {
-    "@types/node": "^22.19.0",
-    "tschema": "^3.2.0",
-    "@atcute/lexicons": "^1.2.4"
+    "@types/node": "^22.19.1",
+    "tschema": "^3.2.0"
   },
   "scripts": {
     "build": "pnpm run generate:schema && tsc",

package/src/cli.ts

@@ -1,205 +1,18 @@
-import * as fs from 'node:fs/promises';
-import * as path from 'node:path';
-
-import { lexiconDoc, refineLexiconDoc, type LexiconDoc } from '@atcute/lexicon-doc';
-
-import { object } from '@optique/core/constructs';
-import { command, constant, option } from '@optique/core/primitives';
+import { or } from '@optique/core/constructs';
 import { run } from '@optique/run';
-import { path as pathParser } from '@optique/run/valueparser';
-import pc from 'picocolors';
-
-import { generateLexiconApi, type ImportMapping } from './codegen.js';
-import { loadConfig } from './config.js';
-import { packageJsonSchema } from './lexicon-metadata.js';
-
-/**
- * Resolves package imports to ImportMapping[]
- */
-const resolveImportsToMappings = async (
-	imports: string[],
-	configDirname: string,
-): Promise<ImportMapping[]> => {
-	const mappings: ImportMapping[] = [];
-
-	for (const packageName of imports) {
-		// Walk up from config directory to find package in node_modules
-		let packageJson: unknown;
-		let currentDir = configDirname;
-		let found = false;
-
-		while (currentDir !== path.dirname(currentDir)) {
-			const candidatePath = path.join(currentDir, 'node_modules', packageName, 'package.json');
-			try {
-				const content = await fs.readFile(candidatePath, 'utf8');
-				packageJson = JSON.parse(content);
-				found = true;
-				break;
-			} catch (err: any) {
-				// Only continue to parent if file not found
-				if (err.code !== 'ENOENT') {
-					console.error(pc.bold(pc.red(`failed to read package.json for "${packageName}":`)));
-					console.error(err);
-					process.exit(1);
-				}
-
-				// Not found, try parent directory
-				currentDir = path.dirname(currentDir);
-			}
-		}
-
-		if (!found) {
-			console.error(pc.bold(pc.red(`failed to resolve package "${packageName}"`)));
-			console.error(`Could not find package in node_modules starting from ${configDirname}`);
-			process.exit(1);
-		}
-
-		// Validate package.json
-		const result = packageJsonSchema.try(packageJson, { mode: 'passthrough' });
-		if (!result.ok) {
-			console.error(pc.bold(pc.red(`invalid atcute:lexicons in "${packageName}":`)));
-			console.error(result.message);
-
-			for (const issue of result.issues) {
-				console.log(`- ${issue.code} at .${issue.path.join('.')}`);
-			}
-
-			process.exit(1);
-		}
-
-		const lexicons = result.value['atcute:lexicons'];
-		if (!lexicons?.mappings) {
-			continue;
-		}
-
-		// Convert mapping to ImportMapping[]
-		for (const [pattern, entry] of Object.entries(lexicons.mappings)) {
-			const isWildcard = pattern.endsWith('.*');
-
-			mappings.push({
-				nsid: [pattern],
-				imports: (nsid: string) => {
-					// Check if pattern matches
-					if (isWildcard) {
-						if (!nsid.startsWith(pattern.slice(0, -1))) {
-							throw new Error(`NSID ${nsid} does not match pattern ${pattern}`);
-						}
-					} else {
-						if (nsid !== pattern) {
-							throw new Error(`NSID ${nsid} does not match pattern ${pattern}`);
-						}
-					}
-
-					const nsidPrefix = isWildcard ? pattern.slice(0, -2) : pattern;
-					const nsidRemainder = isWildcard ? nsid.slice(nsidPrefix.length + 1) : '';
-
-					let expandedPath = entry.path
-						.replaceAll('{{nsid}}', nsid.replaceAll('.', '/'))
-						.replaceAll('{{nsid_remainder}}', nsidRemainder.replaceAll('.', '/'))
-						.replaceAll('{{nsid_prefix}}', nsidPrefix.replaceAll('.', '/'));
-
-					if (expandedPath === '.') {
-						expandedPath = packageName;
-					} else if (expandedPath.startsWith('./')) {
-						expandedPath = `${packageName}/${expandedPath.slice(2)}`;
-					}
-
-					return {
-						type: entry.type,
-						from: expandedPath,
-					};
-				},
-			});
-		}
-	}
 
-	return mappings;
-};
+import { exportCommandSchema, runExport } from './commands/export.js';
+import { generateCommandSchema, runGenerate } from './commands/generate.js';
+import { pullCommandSchema, runPull } from './commands/pull.js';
 
-const parser = command(
-	'generate',
-	object({
-		type: constant('generate'),
-		config: option('-c', '--config', pathParser({ metavar: 'CONFIG' })),
-	}),
-);
+const parser = or(generateCommandSchema, pullCommandSchema, exportCommandSchema);
 
-const result = run(parser, { programName: 'lex-cli' });
+const result = run(parser, { programName: 'lex-cli', help: 'both' });
 
 if (result.type === 'generate') {
-	const config = await loadConfig(result.config);
-
-	// Resolve imports to mappings
-	const importMappings = config.imports ? await resolveImportsToMappings(config.imports, config.root) : [];
-	const allMappings = [...importMappings, ...(config.mappings ?? [])];
-
-	const documents: LexiconDoc[] = [];
-
-	for await (const filename of fs.glob(config.files, { cwd: config.root })) {
-		let source: string;
-		try {
-			source = await fs.readFile(path.join(config.root, filename), 'utf8');
-		} catch (err) {
-			console.error(pc.bold(pc.red(`file read error with "${filename}"`)));
-			console.error(err);
-
-			process.exit(1);
-		}
-
-		let json: unknown;
-		try {
-			json = JSON.parse(source);
-		} catch (err) {
-			console.error(pc.bold(pc.red(`json parse error in "${filename}"`)));
-			console.error(err);
-
-			process.exit(1);
-		}
-
-		const result = lexiconDoc.try(json, { mode: 'strip' });
-		if (!result.ok) {
-			console.error(pc.bold(pc.red(`schema validation failed for "${filename}"`)));
-			console.error(result.message);
-
-			for (const issue of result.issues) {
-				console.log(`- ${issue.code} at .${issue.path.join('.')}`);
-			}
-
-			process.exit(1);
-		}
-
-		const issues = refineLexiconDoc(result.value, true);
-		if (issues.length > 0) {
-			console.error(pc.bold(pc.red(`lint validation failed for "${filename}"`)));
-
-			for (const issue of issues) {
-				console.log(`- ${issue.message} at .${issue.path.join('.')}`);
-			}
-
-			process.exit(1);
-		}
-
-		documents.push(result.value);
-	}
-
-	const generationResult = await generateLexiconApi({
-		documents: documents,
-		mappings: allMappings,
-		modules: {
-			importSuffix: config.modules?.importSuffix ?? '.js',
-		},
-		prettier: {
-			cwd: process.cwd(),
-		},
-	});
-
-	const outdir = path.join(config.root, config.outdir);
-
-	for (const file of generationResult.files) {
-		const filename = path.join(outdir, file.filename);
-		const dirname = path.dirname(filename);
-
-		await fs.mkdir(dirname, { recursive: true });
-		await fs.writeFile(filename, file.code);
-	}
+	await runGenerate(result);
+} else if (result.type === 'pull') {
+	await runPull(result);
+} else if (result.type === 'export') {
+	await runExport(result);
 }