@atproto/lex-installer 0.0.0

package/package.json

@@ -0,0 +1,51 @@
+{
+  "name": "@atproto/lex-installer",
+  "version": "0.0.0",
+  "license": "MIT",
+  "description": "Lexicon document packet manager for AT Lexicons",
+  "keywords": [
+    "atproto",
+    "lexicon",
+    "install",
+    "lex"
+  ],
+  "homepage": "https://atproto.com",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/bluesky-social/atproto",
+    "directory": "packages/lex/lex-installer"
+  },
+  "files": [
+    "./src",
+    "./dist"
+  ],
+  "sideEffects": false,
+  "type": "commonjs",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "browser": "./dist/index.js",
+      "import": "./dist/index.js",
+      "require": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "dependencies": {
+    "@atproto/lex-builder": "workspace:*",
+    "@atproto/lex-cbor": "workspace:*",
+    "@atproto/lex-data": "workspace:*",
+    "@atproto/lex-document": "workspace:*",
+    "@atproto/lex-resolver": "workspace:*",
+    "@atproto/lex-schema": "workspace:*",
+    "@atproto/syntax": "workspace:*",
+    "tslib": "^2.8.1"
+  },
+  "devDependencies": {
+    "jest": "^28.1.2"
+  },
+  "scripts": {
+    "build": "tsc --build tsconfig.build.json",
+    "test": "jest"
+  }
+}

package/src/fs.ts

@@ -0,0 +1,24 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises'
+import { dirname } from 'node:path'
+
+export async function readJsonFile(path: string): Promise<unknown> {
+  const contents = await readFile(path, 'utf8')
+  return JSON.parse(contents)
+}
+
+export async function writeJsonFile(
+  path: string,
+  data: unknown,
+): Promise<void> {
+  await mkdir(dirname(path), { recursive: true })
+  const contents = JSON.stringify(data, null, 2)
+  await writeFile(path, contents, {
+    encoding: 'utf8',
+    mode: 0o644,
+    flag: 'w', // override
+  })
+}
+
+export function isEnoentError(err: unknown): boolean {
+  return err instanceof Error && 'code' in err && err.code === 'ENOENT'
+}

package/src/index.ts

@@ -0,0 +1,41 @@
+import { isEnoentError, readJsonFile } from './fs.js'
+import { LexInstaller, LexInstallerOptions } from './lex-installer.js'
+import {
+  LexiconsManifest,
+  lexiconsManifestSchema,
+} from './lexicons-manifest.js'
+
+export type LexInstallOptions = LexInstallerOptions & {
+  add?: string[]
+  save?: boolean
+  ci?: boolean
+}
+
+export async function install(options: LexInstallOptions) {
+  const manifest: LexiconsManifest | undefined = await readJsonFile(
+    options.manifest,
+  ).then(
+    (json) => lexiconsManifestSchema.parse(json),
+    (cause: unknown) => {
+      if (isEnoentError(cause)) return undefined
+      throw new Error('Failed to read lexicons manifest', { cause })
+    },
+  )
+
+  const additions = new Set(options.add)
+
+  // Perform the installation using the existing manifest as "hint"
+  await using installer = new LexInstaller(options)
+
+  await installer.install({ additions, manifest })
+
+  // Verify lockfile
+  if (options.ci && (!manifest || !installer.matches(manifest))) {
+    throw new Error('Lexicons manifest is out of date')
+  }
+
+  // Save changes if requested
+  if (options.save) {
+    await installer.save()
+  }
+}

package/src/lex-installer.ts

@@ -0,0 +1,314 @@
+import { join } from 'node:path'
+import { LexiconDirectoryIndexer } from '@atproto/lex-builder'
+import { cidForLex } from '@atproto/lex-cbor'
+import { lexEquals } from '@atproto/lex-data'
+import {
+  LexiconDocument,
+  LexiconParameters,
+  LexiconPermission,
+  LexiconRef,
+  LexiconRefUnion,
+  LexiconUnknown,
+  MainLexiconDefinition,
+  NamedLexiconDefinition,
+} from '@atproto/lex-document'
+import { LexResolver, LexResolverOptions } from '@atproto/lex-resolver'
+import { AtUri as AtUriString, Nsid as NsidString } from '@atproto/lex-schema'
+import { AtUri, NSID } from '@atproto/syntax'
+import { isEnoentError, writeJsonFile } from './fs.js'
+import { LexiconsManifest } from './lexicons-manifest.js'
+import { NsidMap } from './nsid-map.js'
+import { NsidSet } from './nsid-set.js'
+
+export type LexInstallerOptions = LexResolverOptions & {
+  lexicons: string
+  manifest: string
+  update?: boolean
+}
+
+export class LexInstaller implements AsyncDisposable {
+  protected readonly lexiconResolver: LexResolver
+  protected readonly indexer: LexiconDirectoryIndexer
+  protected readonly documents = new NsidMap<LexiconDocument>()
+  protected readonly manifest: LexiconsManifest = {
+    version: 1,
+    lexicons: [],
+    resolutions: {},
+  }
+
+  constructor(protected readonly options: LexInstallerOptions) {
+    this.lexiconResolver = new LexResolver(options)
+    this.indexer = new LexiconDirectoryIndexer({
+      lexicons: options.lexicons,
+    })
+  }
+
+  async [Symbol.asyncDispose](): Promise<void> {
+    await this.indexer[Symbol.asyncDispose]()
+  }
+
+  matches(manifest: LexiconsManifest): boolean {
+    return lexEquals(this.manifest, manifest)
+  }
+
+  async install({
+    additions,
+    manifest,
+  }: {
+    additions?: Iterable<string>
+    manifest?: LexiconsManifest
+  } = {}): Promise<void> {
+    const roots = new NsidMap<AtUri | null>()
+
+    // First, process explicit additions
+    for (const lexicon of new Set(additions)) {
+      const [nsid, uri]: [NSID, AtUri | null] = lexicon.startsWith('at://')
+        ? ((uri) => [NSID.from(uri.rkey), uri])(new AtUri(lexicon))
+        : [NSID.from(lexicon), null]
+
+      if (roots.has(nsid)) {
+        throw new Error(
+          `Duplicate lexicon addition: ${nsid} (${roots.get(nsid) ?? lexicon})`,
+        )
+      }
+
+      roots.set(nsid, uri)
+      console.debug(`Adding new lexicon: ${nsid} (${uri ?? 'from NSID'})`)
+    }
+
+    // Next, restore previously existing manifest entries
+    if (manifest) {
+      for (const lexicon of manifest.lexicons) {
+        const nsid = NSID.from(lexicon)
+
+        // Skip entries already added explicitly
+        if (!roots.has(nsid)) {
+          const uri = manifest.resolutions[lexicon]
+            ? new AtUri(manifest.resolutions[lexicon].uri)
+            : null
+
+          roots.set(nsid, uri)
+
+          console.debug(
+            `Adding lexicon from manifest: ${nsid} (${uri ?? 'from NSID'})`,
+          )
+        }
+      }
+    }
+
+    // Install all root lexicons (and store them in the manifest)
+    await Promise.all(
+      Array.from(roots, async ([nsid, sourceUri]) => {
+        console.debug(`Installing lexicon: ${nsid}`)
+
+        const { document } = sourceUri
+          ? await this.installFromUri(sourceUri)
+          : await this.installFromNsid(nsid)
+
+        // Store the direct reference in the new manifest
+        this.manifest.lexicons.push(document.id)
+        this.manifest.lexicons.sort()
+      }),
+    )
+
+    // Then recursively install all referenced lexicons
+    let results: unknown[]
+    do {
+      results = await Promise.all(
+        Array.from(this.getMissingIds(), async (nsid) => {
+          console.debug(`Resolving dependency lexicon: ${nsid}`)
+
+          const nsidStr = nsid.toString() as NsidString
+          const resolvedUri = manifest?.resolutions[nsidStr]?.uri
+            ? new AtUri(manifest.resolutions[nsidStr].uri)
+            : null
+          if (resolvedUri) {
+            await this.installFromUri(resolvedUri)
+          } else {
+            await this.installFromNsid(nsid)
+          }
+        }),
+      )
+    } while (results.length > 0)
+  }
+
+  protected getMissingIds(): NsidSet {
+    const missing = new NsidSet()
+
+    for (const document of this.documents.values()) {
+      for (const nsid of listDocumentNsidRefs(document)) {
+        if (!this.documents.has(nsid)) {
+          missing.add(nsid)
+        }
+      }
+    }
+
+    return missing
+  }
+
+  protected async installFromNsid(nsid: NSID) {
+    const uri = await this.lexiconResolver.resolve(nsid)
+    return this.installFromUri(uri)
+  }
+
+  protected async installFromUri(uri: AtUri): Promise<{
+    document: LexiconDocument
+    uri: AtUri
+  }> {
+    const document = this.options.update
+      ? await this.fetch(uri)
+      : await this.indexer.get(uri.rkey).then(
+          (document) => {
+            console.debug(`Re-using existing lexicon ${uri.rkey} from indexer`)
+            return document
+          },
+          (err) => {
+            if (isEnoentError(err)) return this.fetch(uri)
+            throw err
+          },
+        )
+
+    this.documents.set(NSID.from(document.id), document)
+    this.manifest.resolutions[document.id] = {
+      cid: (await cidForLex(document)).toString(),
+      uri: uri.toString() as AtUriString,
+    }
+
+    return { document, uri }
+  }
+
+  async fetch(uri: AtUri): Promise<LexiconDocument> {
+    console.debug(`Fetching lexicon from ${uri}...`)
+
+    const document = await this.lexiconResolver.fetch(uri, {
+      noCache: this.options.update,
+    })
+
+    const basePath = join(this.options.lexicons, ...document.id.split('.'))
+    await writeJsonFile(`${basePath}.json`, document)
+
+    return document
+  }
+
+  async save(): Promise<void> {
+    await writeJsonFile(this.options.manifest, this.manifest)
+  }
+}
+
+function* listDocumentNsidRefs(doc: LexiconDocument): Iterable<NSID> {
+  try {
+    for (const def of Object.values(doc.defs)) {
+      if (def) {
+        for (const ref of defRefs(def)) {
+          const [nsid] = ref.split('#', 1)
+          if (nsid) yield NSID.from(nsid)
+        }
+      }
+    }
+  } catch (cause) {
+    throw new Error(`Failed to extract refs from lexicon ${doc.id}`, { cause })
+  }
+}
+
+function* defRefs(
+  def:
+    | MainLexiconDefinition
+    | NamedLexiconDefinition
+    | LexiconPermission
+    | LexiconUnknown
+    | LexiconParameters
+    | LexiconRef
+    | LexiconRefUnion,
+): Iterable<string> {
+  switch (def.type) {
+    case 'string':
+      if (def.knownValues) {
+        for (const val of def.knownValues) {
+          // Tokens ?
+          const { length, 0: nsid, 1: hash } = val.split('#')
+          if (length === 2 && hash) {
+            try {
+              NSID.from(nsid)
+              yield val
+            } catch {
+              // ignore invalid nsid
+            }
+          }
+        }
+      }
+      return
+    case 'array':
+      return yield* defRefs(def.items)
+    case 'params':
+    case 'object':
+      for (const prop of Object.values(def.properties)) {
+        yield* defRefs(prop)
+      }
+      return
+    case 'union':
+      yield* def.refs
+      return
+    case 'ref': {
+      yield def.ref
+      return
+    }
+    case 'record':
+      yield* defRefs(def.record)
+      return
+    case 'procedure':
+      if (def.input?.schema) {
+        yield* defRefs(def.input.schema)
+      }
+    // fallthrough
+    case 'query':
+      if (def.output?.schema) {
+        yield* defRefs(def.output.schema)
+      }
+    // fallthrough
+    case 'subscription':
+      if (def.parameters) {
+        yield* defRefs(def.parameters)
+      }
+      if ('message' in def && def.message?.schema) {
+        yield* defRefs(def.message.schema)
+      }
+      return
+    case 'permission-set':
+      for (const permission of def.permissions) {
+        yield* defRefs(permission)
+      }
+      return
+    case 'permission':
+      if (def.resource === 'rpc') {
+        if (Array.isArray(def.lxm)) {
+          for (const lxm of def.lxm) {
+            if (typeof lxm === 'string') {
+              yield lxm
+            }
+          }
+        }
+      } else if (def.resource === 'repo') {
+        if (Array.isArray(def.collection)) {
+          for (const lxm of def.collection) {
+            if (typeof lxm === 'string') {
+              yield lxm
+            }
+          }
+        }
+      }
+      return
+    case 'boolean':
+    case 'cid-link':
+    case 'token':
+    case 'bytes':
+    case 'blob':
+    case 'integer':
+    case 'unknown':
+      // @NOTE We explicitly list all types here to ensure exhaustiveness
+      // causing TS to error if a new type is added without updating this switch
+      return
+    default:
+      // @ts-expect-error
+      throw new Error(`Unknown lexicon def type: ${def.type}`)
+  }
+}

package/src/lexicons-manifest.test.ts

@@ -0,0 +1,50 @@
+import { lexiconsManifestSchema } from './lexicons-manifest.js'
+
+describe('lexiconsManifestSchema', () => {
+  it('parses a valid manifest', () => {
+    expect(
+      lexiconsManifestSchema.parse({
+        version: 1,
+        lexicons: ['com.example.lexicon'],
+        resolutions: {
+          'com.example.lexicon': {
+            uri: 'at://did:plc:foobar/com.atproto.lexicon.schema/com.example.lexicon',
+            cid: 'bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku',
+          },
+        },
+      }),
+    ).toEqual({
+      version: 1,
+      lexicons: ['com.example.lexicon'],
+      resolutions: {
+        'com.example.lexicon': {
+          uri: 'at://did:plc:foobar/com.atproto.lexicon.schema/com.example.lexicon',
+          cid: 'bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku',
+        },
+      },
+    })
+  })
+
+  it('rejects an invalid manifest', () => {
+    expect(() =>
+      lexiconsManifestSchema.parse({
+        version: 1,
+        lexicons: ['com.example.lexicon'],
+        resolutions: {
+          'com.example.lexicon': {
+            uri: 'invalid-uri',
+            cid: 'not-a-cid',
+          },
+        },
+      }),
+    ).toThrow()
+
+    expect(() =>
+      lexiconsManifestSchema.parse({
+        version: 2,
+        lexicons: ['com.example.lexicon'],
+        resolutions: {},
+      }),
+    ).toThrow()
+  })
+})

package/src/lexicons-manifest.ts

@@ -0,0 +1,23 @@
+import { l } from '@atproto/lex-schema'
+
+export const lexiconsManifestSchema = l.object(
+  {
+    version: l.literal(1),
+    lexicons: l.array(l.string({ format: 'nsid' })),
+    resolutions: l.dict(
+      l.string({ format: 'nsid' }),
+      l.object(
+        {
+          cid: l.string({ format: 'cid' }),
+          uri: l.string({ format: 'at-uri' }),
+        },
+        {
+          required: ['cid', 'uri'],
+        },
+      ),
+    ),
+  },
+  { required: ['version', 'lexicons', 'resolutions'] },
+)
+
+export type LexiconsManifest = l.Infer<typeof lexiconsManifestSchema>

package/src/nsid-map.ts

@@ -0,0 +1,83 @@
+import { NSID } from '@atproto/syntax'
+
+/**
+ * A Map implementation that maps keys of type K to an internal representation
+ * I. Key identity is determined by the {@link Object.is} comparison of the
+ * encoded keys.
+ *
+ * This is typically useful for values that can be serialized to a unique string
+ * representation.
+ */
+class MappedMap<K, V, I = any> implements Map<K, V> {
+  private map = new Map<I, V>()
+
+  constructor(
+    private readonly encodeKey: (key: K) => I,
+    private readonly decodeKey: (enc: I) => K,
+  ) {}
+
+  get size(): number {
+    return this.map.size
+  }
+
+  clear(): void {
+    this.map.clear()
+  }
+
+  set(key: K, value: V): this {
+    this.map.set(this.encodeKey(key), value)
+    return this
+  }
+
+  get(key: K): V | undefined {
+    return this.map.get(this.encodeKey(key))
+  }
+
+  has(key: K): boolean {
+    return this.map.has(this.encodeKey(key))
+  }
+
+  delete(key: K): boolean {
+    return this.map.delete(this.encodeKey(key))
+  }
+
+  values(): IterableIterator<V> {
+    return this.map.values()
+  }
+
+  *keys(): IterableIterator<K> {
+    for (const key of this.map.keys()) {
+      yield this.decodeKey(key)
+    }
+  }
+
+  *entries(): IterableIterator<[K, V]> {
+    for (const [key, value] of this.map.entries()) {
+      yield [this.decodeKey(key), value]
+    }
+  }
+
+  forEach(
+    callbackfn: (value: V, key: K, map: MappedMap<K, V>) => void,
+    thisArg?: any,
+  ): void {
+    for (const [key, value] of this) {
+      callbackfn.call(thisArg, value, key, this)
+    }
+  }
+
+  [Symbol.iterator](): IterableIterator<[K, V]> {
+    return this.entries()
+  }
+
+  [Symbol.toStringTag]: string = 'MappedMap'
+}
+
+export class NsidMap<T> extends MappedMap<NSID, T, string> {
+  constructor() {
+    super(
+      (key) => key.toString(),
+      (enc) => NSID.from(enc),
+    )
+  }
+}

package/src/nsid-set.ts

@@ -0,0 +1,77 @@
+import { NSID } from '@atproto/syntax'
+
+/**
+ * A Set implementation that maps values of type K to an internal representation
+ * I. Value identity is determined by the {@link Object.is} comparison of the
+ * encoded values.
+ *
+ * This is typically useful for values that can be serialized to a unique string
+ * representation.
+ */
+export class MappedSet<K, I = any> implements Set<K> {
+  private set = new Set<I>()
+
+  constructor(
+    private readonly encodeValue: (val: K) => I,
+    private readonly decodeValue: (enc: I) => K,
+  ) {}
+
+  get size(): number {
+    return this.set.size
+  }
+
+  clear(): void {
+    this.set.clear()
+  }
+
+  add(val: K): this {
+    this.set.add(this.encodeValue(val))
+    return this
+  }
+
+  has(val: K): boolean {
+    return this.set.has(this.encodeValue(val))
+  }
+
+  delete(val: K): boolean {
+    return this.set.delete(this.encodeValue(val))
+  }
+
+  *values(): IterableIterator<K> {
+    for (const val of this.set.values()) {
+      yield this.decodeValue(val)
+    }
+  }
+
+  keys(): SetIterator<K> {
+    return this.values()
+  }
+
+  *entries(): IterableIterator<[K, K]> {
+    for (const val of this) yield [val, val]
+  }
+
+  forEach(
+    callbackfn: (value: K, value2: K, set: Set<K>) => void,
+    thisArg?: any,
+  ): void {
+    for (const val of this) {
+      callbackfn.call(thisArg, val, val, this)
+    }
+  }
+
+  [Symbol.iterator](): IterableIterator<K> {
+    return this.values()
+  }
+
+  [Symbol.toStringTag]: string = 'MappedSet'
+}
+
+export class NsidSet extends MappedSet<NSID, string> {
+  constructor() {
+    super(
+      (val) => val.toString(),
+      (enc) => NSID.from(enc),
+    )
+  }
+}