@atproto/lex-cli 0.5.6 → 0.6.0

package/src/codegen/lex-gen.ts

@@ -1,18 +1,18 @@
+import { relative as getRelativePath } from 'node:path'
 import { SourceFile, VariableDeclarationKind } from 'ts-morph'
-import { relative as getRelativePath } from 'path'
 import {
-  Lexicons,
-  LexUserType,
-  LexObject,
   LexArray,
-  LexPrimitive,
   LexBlob,
-  LexToken,
-  LexCidLink,
   LexBytes,
+  LexCidLink,
   LexIpldType,
+  LexObject,
+  LexPrimitive,
+  LexToken,
+  LexUserType,
+  Lexicons,
 } from '@atproto/lexicon'
-import { toCamelCase, toTitleCase, toScreamingSnakeCase } from './util'
+import { toCamelCase, toScreamingSnakeCase, toTitleCase } from './util'
 
 interface Commentable<T> {
   addJsDoc: ({ description }: { description: string }) => T
@@ -27,6 +27,67 @@ export function genComment<T>(
   return commentable as T
 }
 
+export function genCommonImports(file: SourceFile, baseNsid: string) {
+  //= import {ValidationResult, BlobRef} from '@atproto/lexicon'
+  file
+    .addImportDeclaration({
+      moduleSpecifier: '@atproto/lexicon',
+    })
+    .addNamedImports([{ name: 'ValidationResult' }, { name: 'BlobRef' }])
+
+  //= import {CID} from 'multiformats/cid'
+  file
+    .addImportDeclaration({
+      moduleSpecifier: 'multiformats/cid',
+    })
+    .addNamedImports([{ name: 'CID' }])
+
+  //= import { validate as _validate } from '../../lexicons.ts'
+  file
+    .addImportDeclaration({
+      moduleSpecifier: `${baseNsid
+        .split('.')
+        .map((_str) => '..')
+        .join('/')}/lexicons`,
+    })
+    .addNamedImports([{ name: 'validate', alias: '_validate' }])
+
+  //= import { $Typed, is$typed as _is$typed, OmitKey } from '../[...]/util.ts'
+  file
+    .addImportDeclaration({
+      moduleSpecifier: `${baseNsid
+        .split('.')
+        .map((_str) => '..')
+        .join('/')}/util`,
+    })
+    .addNamedImports([
+      { name: '$Typed' },
+      { name: 'is$typed', alias: '_is$typed' },
+      { name: 'OmitKey' },
+    ])
+
+  // tsc adds protection against circular imports, which hurts bundle size.
+  // Since we know that lexicon.ts and util.ts do not depend on the file being
+  // generated, we can safely bypass this protection.
+  // Note that we are not using `import * as util from '../../util'` because
+  // typescript will emit is own helpers for the import, which we want to avoid.
+  file.addVariableStatement({
+    isExported: false,
+    declarationKind: VariableDeclarationKind.Const,
+    declarations: [
+      { name: 'is$typed', initializer: '_is$typed' },
+      { name: 'validate', initializer: '_validate' },
+    ],
+  })
+
+  //= const id = "{baseNsid}"
+  file.addVariableStatement({
+    isExported: false, // Do not export to allow tree-shaking
+    declarationKind: VariableDeclarationKind.Const,
+    declarations: [{ name: 'id', initializer: JSON.stringify(baseNsid) }],
+  })
+}
+
 export function genImports(
   file: SourceFile,
   imports: Set<string>,
@@ -35,12 +96,13 @@ export function genImports(
   const startPath = '/' + baseNsid.split('.').slice(0, -1).join('/')
 
   for (const nsid of imports) {
-    const targetPath = '/' + nsid.split('.').join('/')
+    const targetPath = '/' + nsid.split('.').join('/') + '.js'
     let resolvedPath = getRelativePath(startPath, targetPath)
     if (!resolvedPath.startsWith('.')) {
       resolvedPath = `./${resolvedPath}`
     }
     file.addImportDeclaration({
+      isTypeOnly: true,
       moduleSpecifier: resolvedPath,
       namespaceImport: toTitleCase(nsid),
     })
@@ -61,10 +123,16 @@ export function genUserType(
     case 'token':
       genToken(file, lexUri, def)
       break
-    case 'object':
-      genObject(file, imports, lexUri, def)
-      genObjHelpers(file, lexUri)
+    case 'object': {
+      const ifaceName: string = toTitleCase(getHash(lexUri))
+      genObject(file, imports, lexUri, def, ifaceName, {
+        typeProperty: true,
+      })
+      genObjHelpers(file, lexUri, ifaceName, {
+        requireTypeProperty: false,
+      })
       break
+    }
 
     case 'blob':
     case 'bytes':
@@ -83,19 +151,44 @@ export function genUserType(
   }
 }
 
-export function genObject(
+function genObject(
   file: SourceFile,
   imports: Set<string>,
   lexUri: string,
   def: LexObject,
-  ifaceName?: string,
-  defaultsArePresent = true,
+  ifaceName: string,
+  {
+    defaultsArePresent = true,
+    allowUnknownProperties = false,
+    typeProperty = false,
+  }: {
+    defaultsArePresent?: boolean
+    allowUnknownProperties?: boolean
+    typeProperty?: boolean | 'required'
+  } = {},
 ) {
   const iface = file.addInterface({
-    name: ifaceName || toTitleCase(getHash(lexUri)),
+    name: ifaceName,
     isExported: true,
   })
   genComment(iface, def)
+
+  if (typeProperty) {
+    const hash = getHash(lexUri)
+    const baseNsid = stripScheme(stripHash(lexUri))
+
+    //= $type?: <uri>
+    iface.addProperty({
+      name: typeProperty === 'required' ? `$type` : `$type?`,
+      type:
+        // Not using $Type here because it is less readable than a plain string
+        // `$Type<${JSON.stringify(baseNsid)}, ${JSON.stringify(hash)}>`
+        hash === 'main'
+          ? JSON.stringify(`${baseNsid}`)
+          : JSON.stringify(`${baseNsid}#${hash}`),
+    })
+  }
+
   const nullableProps = new Set(def.nullable)
   if (def.properties) {
     for (const propKey in def.properties) {
@@ -108,12 +201,12 @@ export function genObject(
           propDef.default !== undefined)
       if (propDef.type === 'ref' || propDef.type === 'union') {
         //= propName: External|External
-        const refs = propDef.type === 'union' ? propDef.refs : [propDef.ref]
-        const types = refs.map((ref) =>
-          refToType(ref, stripScheme(stripHash(lexUri)), imports),
-        )
+        const types =
+          propDef.type === 'union'
+            ? propDef.refs.map((ref) => refToUnionType(ref, lexUri, imports))
+            : [refToType(propDef.ref, stripScheme(stripHash(lexUri)), imports)]
         if (propDef.type === 'union' && !propDef.closed) {
-          types.push('{$type: string; [k: string]: unknown}')
+          types.push('{ $type: string }')
         }
         iface.addProperty({
           name: `${propKey}${req ? '' : '?'}`,
@@ -141,10 +234,10 @@ export function genObject(
             })
           } else if (propDef.items.type === 'union') {
             const types = propDef.items.refs.map((ref) =>
-              refToType(ref, stripScheme(stripHash(lexUri)), imports),
+              refToUnionType(ref, lexUri, imports),
             )
             if (!propDef.items.closed) {
-              types.push('{$type: string; [k: string]: unknown}')
+              types.push('{ $type: string }')
             }
             propAst = iface.addProperty({
               name: `${propKey}${req ? '' : '?'}`,
@@ -177,16 +270,21 @@ export function genObject(
         }
       }
     }
-    //= [k: string]: unknown
-    iface.addIndexSignature({
-      keyName: 'k',
-      keyType: 'string',
-      returnType: 'unknown',
-    })
+
+    if (allowUnknownProperties) {
+      //= [k: string]: unknown
+      iface.addIndexSignature({
+        keyName: 'k',
+        keyType: 'string',
+        returnType: 'unknown',
+      })
+    }
   }
 }
 
 export function genToken(file: SourceFile, lexUri: string, def: LexToken) {
+  //= /** <comment> */
+  //= export const <TOKEN> = `${id}#<token>`
   genComment(
     file.addVariableStatement({
       isExported: true,
@@ -194,7 +292,7 @@ export function genToken(file: SourceFile, lexUri: string, def: LexToken) {
       declarations: [
         {
           name: toScreamingSnakeCase(getHash(lexUri)),
-          initializer: `"${stripScheme(lexUri)}"`,
+          initializer: `\`\${id}#${getHash(lexUri)}\``,
         },
       ],
     }),
@@ -220,10 +318,10 @@ export function genArray(
     })
   } else if (def.items.type === 'union') {
     const types = def.items.refs.map((ref) =>
-      refToType(ref, stripScheme(stripHash(lexUri)), imports),
+      refToUnionType(ref, lexUri, imports),
     )
     if (!def.items.closed) {
-      types.push('{$type: string; [k: string]: unknown}')
+      types.push('{ $type: string }')
     }
     file.addTypeAlias({
       name: toTitleCase(getHash(lexUri)),
@@ -308,15 +406,22 @@ export function genXrpcInput(
   if (def.type === 'procedure' && def.input?.schema) {
     if (def.input.schema.type === 'ref' || def.input.schema.type === 'union') {
       //= export type InputSchema = ...
-      const refs =
+
+      const types =
         def.input.schema.type === 'union'
-          ? def.input.schema.refs
-          : [def.input.schema.ref]
-      const types = refs.map((ref) =>
-        refToType(ref, stripScheme(stripHash(lexUri)), imports),
-      )
+          ? def.input.schema.refs.map((ref) =>
+              refToUnionType(ref, lexUri, imports),
+            )
+          : [
+              refToType(
+                def.input.schema.ref,
+                stripScheme(stripHash(lexUri)),
+                imports,
+              ),
+            ]
+
       if (def.input.schema.type === 'union' && !def.input.schema.closed) {
-        types.push('{$type: string; [k: string]: unknown}')
+        types.push('{ $type: string }')
       }
       file.addTypeAlias({
         name: 'InputSchema',
@@ -325,14 +430,9 @@ export function genXrpcInput(
       })
     } else {
       //= export interface InputSchema {...}
-      genObject(
-        file,
-        imports,
-        lexUri,
-        def.input.schema,
-        `InputSchema`,
+      genObject(file, imports, lexUri, def.input.schema, `InputSchema`, {
         defaultsArePresent,
-      )
+      })
     }
   } else if (def.type === 'procedure' && def.input?.encoding) {
     //= export type InputSchema = string | Uint8Array | Blob
@@ -369,12 +469,12 @@ export function genXrpcOutput(
   if (schema) {
     if (schema.type === 'ref' || schema.type === 'union') {
       //= export type OutputSchema = ...
-      const refs = schema.type === 'union' ? schema.refs : [schema.ref]
-      const types = refs.map((ref) =>
-        refToType(ref, stripScheme(stripHash(lexUri)), imports),
-      )
+      const types =
+        schema.type === 'union'
+          ? schema.refs.map((ref) => refToUnionType(ref, lexUri, imports))
+          : [refToType(schema.ref, stripScheme(stripHash(lexUri)), imports)]
       if (schema.type === 'union' && !schema.closed) {
-        types.push('{$type: string; [k: string]: unknown}')
+        types.push('{ $type: string }')
       }
       file.addTypeAlias({
         name: 'OutputSchema',
@@ -383,50 +483,79 @@ export function genXrpcOutput(
       })
     } else {
       //= export interface OutputSchema {...}
-      genObject(
-        file,
-        imports,
-        lexUri,
-        schema,
-        `OutputSchema`,
+      genObject(file, imports, lexUri, schema, `OutputSchema`, {
         defaultsArePresent,
-      )
+      })
     }
   }
 }
 
-export function genObjHelpers(
+export function genRecord(
+  file: SourceFile,
+  imports: Set<string>,
+  lexicons: Lexicons,
+  lexUri: string,
+) {
+  const def = lexicons.getDefOrThrow(lexUri, ['record'])
+
+  //= export interface Record {...}
+  genObject(file, imports, lexUri, def.record, 'Record', {
+    defaultsArePresent: true,
+    allowUnknownProperties: true,
+    typeProperty: 'required',
+  })
+
+  //= export function isRecord(v: unknown): v is Record {...}
+  genObjHelpers(file, lexUri, 'Record', {
+    requireTypeProperty: true,
+  })
+}
+
+function genObjHelpers(
   file: SourceFile,
   lexUri: string,
-  ifaceName?: string,
+  ifaceName: string,
+  {
+    requireTypeProperty,
+  }: {
+    requireTypeProperty: boolean
+  },
 ) {
   const hash = getHash(lexUri)
 
-  //= export function is{X}(v: unknown): v is X {...}
+  const hashVar = `hash${ifaceName}`
+
+  file.addVariableStatement({
+    isExported: false,
+    declarationKind: VariableDeclarationKind.Const,
+    declarations: [{ name: hashVar, initializer: JSON.stringify(hash) }],
+  })
+
+  const isX = toCamelCase(`is-${ifaceName}`)
+
+  //= export function is{X}<V>(v: V) {...}
   file
     .addFunction({
-      name: toCamelCase(`is-${ifaceName || hash}`),
-      parameters: [{ name: `v`, type: `unknown` }],
-      returnType: `v is ${ifaceName || toTitleCase(hash)}`,
+      name: isX,
+      typeParameters: [{ name: `V` }],
+      parameters: [{ name: `v`, type: `V` }],
       isExported: true,
     })
-    .setBodyText(
-      hash === 'main'
-        ? `return isObj(v) && hasProp(v, '$type') && (v.$type === "${lexUri}" || v.$type === "${stripHash(
-            lexUri,
-          )}")`
-        : `return isObj(v) && hasProp(v, '$type') && v.$type === "${lexUri}"`,
-    )
+    .setBodyText(`return is$typed(v, id, ${hashVar})`)
+
+  const validateX = toCamelCase(`validate-${ifaceName}`)
 
-  //= export function validate{X}(v: unknown): ValidationResult {...}
+  //= export function validate{X}(v: unknown) {...}
   file
     .addFunction({
-      name: toCamelCase(`validate-${ifaceName || hash}`),
-      parameters: [{ name: `v`, type: `unknown` }],
-      returnType: `ValidationResult`,
+      name: validateX,
+      typeParameters: [{ name: `V` }],
+      parameters: [{ name: `v`, type: `V` }],
       isExported: true,
     })
-    .setBodyText(`return lexicons.validate("${lexUri}", v)`)
+    .setBodyText(
+      `return validate<${ifaceName} & V>(v, id, ${hashVar}${requireTypeProperty ? ', true' : ''})`,
+    )
 }
 
 export function stripScheme(uri: string): string {
@@ -449,7 +578,16 @@ export function ipldToType(def: LexCidLink | LexBytes) {
   return 'CID'
 }
 
-export function refToType(
+function refToUnionType(
+  ref: string,
+  lexUri: string,
+  imports: Set<string>,
+): string {
+  const baseNsid = stripScheme(stripHash(lexUri))
+  return `$Typed<${refToType(ref, baseNsid, imports)}>`
+}
+
+function refToType(
   ref: string,
   baseNsid: string,
   imports: Set<string>,
@@ -510,7 +648,10 @@ export function primitiveToType(def: LexPrimitive): string {
       }
       return 'boolean'
     case 'unknown':
-      return '{}'
+      // @TODO Should we use "object" here ?
+      // the "Record" identifier from typescript get overwritten by the Record
+      // interface created by lex-cli.
+      return '{ [_ in string]: unknown }' // Record<string, unknown>
     default:
       throw new Error(`Unexpected primitive type: ${JSON.stringify(def)}`)
   }

package/src/codegen/server.ts

@@ -4,26 +4,26 @@ import {
   SourceFile,
   VariableDeclarationKind,
 } from 'ts-morph'
-import { Lexicons, LexiconDoc } from '@atproto/lexicon'
+import { LexiconDoc, Lexicons } from '@atproto/lexicon'
 import { NSID } from '@atproto/syntax'
-import { gen, lexiconsTs, utilTs } from './common'
 import { GeneratedAPI } from '../types'
+import { gen, lexiconsTs, utilTs } from './common'
 import {
+  genCommonImports,
   genImports,
+  genRecord,
   genUserType,
-  genObject,
-  genXrpcParams,
   genXrpcInput,
   genXrpcOutput,
-  genObjHelpers,
+  genXrpcParams,
 } from './lex-gen'
 import {
-  lexiconsToDefTree,
   DefTreeNode,
+  lexiconsToDefTree,
   schemasToNsidTokens,
   toCamelCase,
-  toTitleCase,
   toScreamingSnakeCase,
+  toTitleCase,
 } from './util'
 
 export async function genServerApi(
@@ -73,10 +73,10 @@ const indexTs = (
         { name: 'StreamAuthVerifier' },
       ],
     })
-    //= import {schemas} from './lexicons'
+    //= import {schemas} from './lexicons.js'
     file
       .addImportDeclaration({
-        moduleSpecifier: './lexicons',
+        moduleSpecifier: './lexicons.js',
       })
       .addNamedImport({
         name: 'schemas',
@@ -93,7 +93,7 @@ const indexTs = (
       }
       file
         .addImportDeclaration({
-          moduleSpecifier: `./types/${lexiconDoc.id.split('.').join('/')}`,
+          moduleSpecifier: `./types/${lexiconDoc.id.split('.').join('/')}.js`,
         })
         .setNamespaceImport(toTitleCase(lexiconDoc.id))
     }
@@ -319,8 +319,6 @@ const lexiconTs = (project, lexicons: Lexicons, lexiconDoc: LexiconDoc) =>
     project,
     `/types/${lexiconDoc.id.split('.').join('/')}.ts`,
     async (file) => {
-      const imports: Set<string> = new Set()
-
       const main = lexiconDoc.defs.main
       if (main?.type === 'query' || main?.type === 'procedure') {
         //= import express from 'express'
@@ -335,44 +333,17 @@ const lexiconTs = (project, lexicons: Lexicons, lexiconDoc: LexiconDoc) =>
           !main.input.schema
         const streamingOutput = main.output?.encoding && !main.output.schema
         if (streamingInput || streamingOutput) {
-          //= import stream from 'stream'
+          //= import stream from 'node:stream'
           file.addImportDeclaration({
-            moduleSpecifier: 'stream',
+            moduleSpecifier: 'node:stream',
             defaultImport: 'stream',
           })
         }
       }
-      //= import {ValidationResult, BlobRef} from '@atproto/lexicon'
-      file
-        .addImportDeclaration({
-          moduleSpecifier: '@atproto/lexicon',
-        })
-        .addNamedImports([{ name: 'ValidationResult' }, { name: 'BlobRef' }])
-      //= import {lexicons} from '../../lexicons.ts'
-      file
-        .addImportDeclaration({
-          moduleSpecifier: `${lexiconDoc.id
-            .split('.')
-            .map((_str) => '..')
-            .join('/')}/lexicons`,
-        })
-        .addNamedImports([{ name: 'lexicons' }])
-      //= import {isObj, hasProp} from '../../util.ts'
-      file
-        .addImportDeclaration({
-          moduleSpecifier: `${lexiconDoc.id
-            .split('.')
-            .map((_str) => '..')
-            .join('/')}/util`,
-        })
-        .addNamedImports([{ name: 'isObj' }, { name: 'hasProp' }])
-      //= import {CID} from 'multiformats/cid'
-      file
-        .addImportDeclaration({
-          moduleSpecifier: 'multiformats/cid',
-        })
-        .addNamedImports([{ name: 'CID' }])
 
+      genCommonImports(file, lexiconDoc.id)
+
+      const imports: Set<string> = new Set()
       for (const defId in lexiconDoc.defs) {
         const def = lexiconDoc.defs[defId]
         const lexUri = `${lexiconDoc.id}#${defId}`
@@ -387,7 +358,7 @@ const lexiconTs = (project, lexicons: Lexicons, lexiconDoc: LexiconDoc) =>
             genXrpcOutput(file, imports, lexicons, lexUri, false)
             genServerXrpcStreaming(file, lexicons, lexUri)
           } else if (def.type === 'record') {
-            genServerRecord(file, imports, lexicons, lexUri)
+            genRecord(file, imports, lexicons, lexUri)
           } else {
             genUserType(file, imports, lexicons, lexUri)
           }
@@ -547,7 +518,7 @@ function genServerXrpcStreaming(
   })
 
   file.addImportDeclaration({
-    moduleSpecifier: 'http',
+    moduleSpecifier: 'node:http',
     namedImports: [{ name: 'IncomingMessage' }],
   })
 
@@ -589,20 +560,6 @@ function genServerXrpcStreaming(
   })
 }
 
-function genServerRecord(
-  file: SourceFile,
-  imports: Set<string>,
-  lexicons: Lexicons,
-  lexUri: string,
-) {
-  const def = lexicons.getDefOrThrow(lexUri, ['record'])
-
-  //= export interface Record {...}
-  genObject(file, imports, lexUri, def.record, 'Record')
-  //= export function isRecord(v: unknown): v is Record {...}
-  genObjHelpers(file, lexUri, 'Record')
-}
-
 function arrayToUnion(arr?: string[]) {
   if (!arr?.length) {
     return 'never'

package/src/codegen/util.ts

@@ -1,4 +1,4 @@
-import { LexiconDoc, LexUserType } from '@atproto/lexicon'
+import { LexUserType, LexiconDoc } from '@atproto/lexicon'
 import { NSID } from '@atproto/syntax'
 
 export interface DefTreeNodeUserType {

package/src/index.ts

@@ -1,18 +1,18 @@
 #!/usr/bin/env node
 
-import path from 'path'
+import path from 'node:path'
 import { Command } from 'commander'
 import yesno from 'yesno'
+import { genClientApi } from './codegen/client'
+import { genServerApi } from './codegen/server'
+import * as mdGen from './mdgen'
 import {
-  readAllLexicons,
-  genTsObj,
+  applyFileDiff,
   genFileDiff,
+  genTsObj,
   printFileDiff,
-  applyFileDiff,
+  readAllLexicons,
 } from './util'
-import * as mdGen from './mdgen'
-import { genClientApi } from './codegen/client'
-import { genServerApi } from './codegen/server'
 
 const program = new Command()
 program.name('lex').description('Lexicon CLI').version('0.0.0')

package/src/mdgen/index.ts

@@ -1,4 +1,4 @@
-import fs from 'fs'
+import fs from 'node:fs'
 import { LexiconDoc } from '@atproto/lexicon'
 
 const INSERT_START = [

package/src/util.ts

@@ -1,9 +1,9 @@
-import fs from 'fs'
-import { join } from 'path'
-import { parseLexiconDoc, LexiconDoc } from '@atproto/lexicon'
-import { ZodError, ZodFormattedError } from 'zod'
+import fs from 'node:fs'
+import { join } from 'node:path'
 import chalk from 'chalk'
-import { GeneratedAPI, FileDiff } from './types'
+import { ZodError, ZodFormattedError } from 'zod'
+import { LexiconDoc, parseLexiconDoc } from '@atproto/lexicon'
+import { FileDiff, GeneratedAPI } from './types'
 
 export function readAllLexicons(paths: string[]): LexiconDoc[] {
   const docs: LexiconDoc[] = []