@sanity/cli 3.86.1 → 3.86.2-experimental.0

package/src/util/workerChannel.ts

@@ -0,0 +1,312 @@
+// NOTE: this file was originally copied from
+// https://github.com/sanity-io/sanity/blob/4c4e03d407106dbda12f52cfd9511fbfe75a9696/packages/sanity/src/_internal/cli/util/workerChannels.ts
+import {type MessagePort, type Worker} from 'node:worker_threads'
+
+type StreamReporter<TPayload = unknown> = {emit: (payload: TPayload) => void; end: () => void}
+type EventReporter<TPayload = unknown> = (payload: TPayload) => void
+type EventReceiver<TPayload = unknown> = () => Promise<TPayload>
+type StreamReceiver<TPayload = unknown> = () => AsyncIterable<TPayload>
+
+type EventKeys<TWorkerChannel extends WorkerChannel> = {
+  [K in keyof TWorkerChannel]: TWorkerChannel[K] extends WorkerChannelEvent<any> ? K : never
+}[keyof TWorkerChannel]
+type StreamKeys<TWorkerChannel extends WorkerChannel> = {
+  [K in keyof TWorkerChannel]: TWorkerChannel[K] extends WorkerChannelStream<any> ? K : never
+}[keyof TWorkerChannel]
+
+type EventMessage<TPayload = unknown> = {type: 'event'; name: string; payload: TPayload}
+type StreamEmissionMessage<TPayload = unknown> = {type: 'emission'; name: string; payload: TPayload}
+type StreamEndMessage = {type: 'end'; name: string}
+type WorkerChannelMessage = EventMessage | StreamEmissionMessage | StreamEndMessage
+
+/**
+ * Represents the definition of a "worker channel" to report progress from the
+ * worker to the parent. Worker channels can define named events or streams and
+ * the worker will report events and streams while the parent will await them.
+ * This allows the control flow of the parent to follow the control flow of the
+ * worker 1-to-1.
+ *
+ * @example
+ *
+ * ```ts
+ * // Define the channel interface (shared between parent and worker)
+ * type MyWorkerChannel = WorkerChannel<{
+ *   compileStart: WorkerChannelEvent<void>
+ *   compileProgress: WorkerChannelStream<{ file: string; progress: number }>
+ *   compileEnd: WorkerChannelEvent<{ duration: number }>
+ * }>;
+ *
+ * // --- In the worker file (e.g., worker.ts) ---
+ * import { parentPort } from 'node:worker_threads';
+ * import { createReporter } from './workerChannels';
+ *
+ * const report = createReporter<MyWorkerChannel>(parentPort);
+ *
+ * async function runCompilation() {
+ *   report.event.compileStart(); // Signal start
+ *
+ *   const files = ['a.js', 'b.js', 'c.js'];
+ *   for (const file of files) {
+ *     // Simulate work and report progress
+ *     await new Promise(resolve => setTimeout(resolve, 100));
+ *     report.stream.compileProgress.emit({ file, progress: 100 });
+ *   }
+ *   report.stream.compileProgress.end(); // Signal end of progress stream
+ *
+ *   report.event.compileEnd({ duration: 300 }); // Signal end with result
+ * }
+ *
+ * runCompilation();
+ *
+ * // --- In the parent file (e.g., main.ts) ---
+ * import { Worker } from 'node:worker_threads';
+ * import { createReceiver } from './workerChannels';
+ *
+ * const worker = new Worker('./worker.js');
+ * const receiver = createReceiver<MyWorkerChannel>(worker);
+ *
+ * async function monitorCompilation() {
+ *   console.log('Waiting for compilation to start...');
+ *   await receiver.event.compileStart();
+ *   console.log('Compilation started.');
+ *
+ *   console.log('Receiving progress:');
+ *   for await (const progress of receiver.stream.compileProgress()) {
+ *     console.log(`  - ${progress.file}: ${progress.progress}%`);
+ *   }
+ *
+ *   console.log('Waiting for compilation to end...');
+ *   const { duration } = await receiver.event.compileEnd();
+ *   console.log(`Compilation finished in ${duration}ms.`);
+ *
+ *   await receiver.dispose(); // Clean up listeners and terminate worker
+ * }
+ *
+ * monitorCompilation();
+ * ```
+ *
+ * @internal
+ */
+export type WorkerChannel<
+  TWorkerChannel extends Record<
+    string,
+    WorkerChannelEvent<unknown> | WorkerChannelStream<unknown>
+  > = Record<string, WorkerChannelEvent<unknown> | WorkerChannelStream<unknown>>,
+> = TWorkerChannel
+
+/** @internal */
+export type WorkerChannelEvent<TPayload = void> = {type: 'event'; payload: TPayload}
+/** @internal */
+export type WorkerChannelStream<TPayload = void> = {type: 'stream'; payload: TPayload}
+
+export interface WorkerChannelReporter<TWorkerChannel extends WorkerChannel> {
+  event: {
+    [K in EventKeys<TWorkerChannel>]: TWorkerChannel[K] extends WorkerChannelEvent<infer TPayload>
+      ? EventReporter<TPayload>
+      : void
+  }
+  stream: {
+    [K in StreamKeys<TWorkerChannel>]: TWorkerChannel[K] extends WorkerChannelStream<infer TPayload>
+      ? StreamReporter<TPayload>
+      : void
+  }
+}
+
+export interface WorkerChannelReceiver<TWorkerChannel extends WorkerChannel> {
+  event: {
+    [K in EventKeys<TWorkerChannel>]: TWorkerChannel[K] extends WorkerChannelEvent<infer TPayload>
+      ? EventReceiver<TPayload>
+      : void
+  }
+  stream: {
+    [K in StreamKeys<TWorkerChannel>]: TWorkerChannel[K] extends WorkerChannelStream<infer TPayload>
+      ? StreamReceiver<TPayload>
+      : void
+  }
+  // TODO: good candidate for [Symbol.asyncDispose] when our tooling better supports it
+  dispose: () => Promise<number>
+}
+
+/**
+ * A simple queue that has two primary methods: `push(message)` and
+ * `await next()`. This message queue is used by the "receiver" of the worker
+ * channel and this class handles buffering incoming messages if the worker is
+ * producing faster than the parent as well as returning a promise if there is
+ * no message yet in the queue when the parent awaits `next()`.
+ */
+class MessageQueue<T> {
+  resolver: ((result: IteratorResult<T>) => void) | null = null
+  queue: T[] = []
+  private ended = false // Flag to indicate if end() was called
+
+  push(message: T) {
+    if (this.ended) {
+      // Don't push messages after the queue has ended
+      return
+    }
+    if (this.resolver) {
+      this.resolver({value: message, done: false})
+      this.resolver = null
+    } else {
+      this.queue.push(message)
+    }
+  }
+
+  next(): Promise<IteratorResult<T>> {
+    if (this.queue.length) {
+      return Promise.resolve({value: this.queue.shift()!, done: false})
+    }
+
+    if (this.ended) {
+      // If end() was called before and queue is empty, resolve immediately as done
+      return Promise.resolve({value: undefined, done: true})
+    }
+
+    return new Promise((resolve) => (this.resolver = resolve))
+  }
+
+  end() {
+    if (this.resolver) {
+      this.resolver({value: undefined, done: true})
+      this.resolver = null // Clear resolver after ending
+    } else {
+      // If resolver is null, it means next() hasn't been called yet or
+      // previous next() was resolved by a push(). Mark as ended so the
+      // *next* call to next() resolves immediately as done.
+      this.ended = true
+    }
+  }
+}
+
+function isWorkerChannelMessage(message: unknown): message is WorkerChannelMessage {
+  if (typeof message !== 'object') return false
+  if (!message) return false
+  if (!('type' in message)) return false
+  if (typeof message.type !== 'string') return false
+  const types: string[] = ['event', 'emission', 'end'] satisfies WorkerChannelMessage['type'][]
+  return types.includes(message.type)
+}
+
+/**
+ * Creates a "worker channel receiver" that subscribes to incoming messages
+ * from the given worker and returns promises for worker channel events and
+ * async iterators for worker channel streams.
+ */
+export function createReceiver<TWorkerChannel extends WorkerChannel>(
+  worker: Worker,
+): WorkerChannelReceiver<TWorkerChannel> {
+  const _events = new Map<string, MessageQueue<EventMessage>>()
+  const _streams = new Map<string, MessageQueue<StreamEmissionMessage>>()
+  const errors = new MessageQueue<{type: 'error'; error: unknown}>()
+
+  const eventQueue = (name: string) => {
+    const queue = _events.get(name) ?? new MessageQueue()
+    if (!_events.has(name)) _events.set(name, queue)
+    return queue
+  }
+
+  const streamQueue = (name: string) => {
+    const queue = _streams.get(name) ?? new MessageQueue()
+    if (!_streams.has(name)) _streams.set(name, queue)
+    return queue
+  }
+
+  const handleMessage = (message: unknown) => {
+    if (!isWorkerChannelMessage(message)) return
+    if (message.type === 'event') eventQueue(message.name).push(message)
+    if (message.type === 'emission') streamQueue(message.name).push(message)
+    if (message.type === 'end') streamQueue(message.name).end()
+  }
+
+  const handleError = (error: unknown) => {
+    errors.push({type: 'error', error})
+  }
+
+  worker.addListener('message', handleMessage)
+  worker.addListener('error', handleError)
+
+  return {
+    event: new Proxy({} as WorkerChannelReceiver<TWorkerChannel>['event'], {
+      get: (target, name) => {
+        if (typeof name !== 'string') return target[name as keyof typeof target]
+
+        const eventReceiver: EventReceiver = async () => {
+          const {value} = await Promise.race([eventQueue(name).next(), errors.next()])
+          if (value.type === 'error') throw value.error
+          return value.payload
+        }
+
+        return eventReceiver
+      },
+    }),
+    stream: new Proxy({} as WorkerChannelReceiver<TWorkerChannel>['stream'], {
+      get: (target, prop) => {
+        if (typeof prop !== 'string') return target[prop as keyof typeof target]
+        const name = prop // alias for better typescript narrowing
+
+        async function* streamReceiver() {
+          while (true) {
+            const {value, done} = await Promise.race([streamQueue(name).next(), errors.next()])
+            if (done) return
+            if (value.type === 'error') throw value.error
+            yield value.payload
+          }
+        }
+
+        return streamReceiver satisfies StreamReceiver
+      },
+    }),
+    dispose: () => {
+      worker.removeListener('message', handleMessage)
+      worker.removeListener('error', handleError)
+      return worker.terminate()
+    },
+  }
+}
+
+/**
+ * Creates a "worker channel reporter" that sends messages to the given
+ * `parentPort` to be received by a worker channel receiver.
+ *
+ * @internal
+ */
+export function createReporter<TWorkerChannel extends WorkerChannel>(
+  parentPort: MessagePort | null,
+): WorkerChannelReporter<TWorkerChannel> {
+  if (!parentPort) {
+    throw new Error('parentPart was falsy')
+  }
+
+  return {
+    event: new Proxy({} as WorkerChannelReporter<TWorkerChannel>['event'], {
+      get: (target, name) => {
+        if (typeof name !== 'string') return target[name as keyof typeof target]
+
+        const eventReporter: EventReporter = (payload) => {
+          const message: EventMessage = {type: 'event', name, payload}
+          parentPort.postMessage(message)
+        }
+
+        return eventReporter
+      },
+    }),
+    stream: new Proxy({} as WorkerChannelReporter<TWorkerChannel>['stream'], {
+      get: (target, name) => {
+        if (typeof name !== 'string') return target[name as keyof typeof target]
+
+        const streamReporter: StreamReporter = {
+          emit: (payload) => {
+            const message: StreamEmissionMessage = {type: 'emission', name, payload}
+            parentPort.postMessage(message)
+          },
+          end: () => {
+            const message: StreamEndMessage = {type: 'end', name}
+            parentPort.postMessage(message)
+          },
+        }
+
+        return streamReporter
+      },
+    }),
+  }
+}

package/src/workers/typegenGenerate.ts

@@ -2,228 +2,90 @@ import {isMainThread, parentPort, workerData as _workerData} from 'node:worker_t
 
 import {
   findQueriesInPath,
+  type GeneratedQueries,
+  type GeneratedSchema,
+  type GeneratedTypemap,
+  generateTypes,
   getResolver,
   readSchema,
   registerBabel,
-  safeParseQuery,
-  TypeGenerator,
 } from '@sanity/codegen'
 import createDebug from 'debug'
-import {typeEvaluate, type TypeNode} from 'groq-js'
+import {type SchemaType} from 'groq-js'
+
+import {
+  createReporter,
+  type WorkerChannel,
+  type WorkerChannelEvent,
+  type WorkerChannelStream,
+} from '../util/workerChannel'
 
 const $info = createDebug('sanity:codegen:generate:info')
-const $warn = createDebug('sanity:codegen:generate:warn')
 
 export interface TypegenGenerateTypesWorkerData {
   workDir: string
-  workspaceName?: string
-  schemaPath: string
+  schemas: {
+    projectId: string | 'default'
+    dataset: string | 'default'
+    schemaPath: string
+  }[]
   searchPath: string | string[]
-  overloadClientMethods?: boolean
+  overloadClientMethods: boolean
+  augmentGroqModule: boolean
 }
 
-export type TypegenGenerateTypesWorkerMessage =
-  | {
-      type: 'error'
-      error: Error
-      fatal: boolean
-      query?: string
-      filename?: string
-    }
-  | {
-      type: 'types'
-      filename: string
-      types: {
-        queryName: string
-        query: string
-        type: string
-        unknownTypeNodesGenerated: number
-        typeNodesGenerated: number
-        emptyUnionTypeNodesGenerated: number
-      }[]
-    }
-  | {
-      type: 'schema'
-      filename: string
-      schema: string
-      length: number
-    }
-  | {
-      type: 'typemap'
-      typeMap: string
-    }
-  | {
-      type: 'complete'
-    }
+/** @internal */
+export type TypegenWorkerChannel = WorkerChannel<{
+  schema: WorkerChannelEvent<GeneratedSchema>
+  queries: WorkerChannelStream<GeneratedQueries>
+  typemap: WorkerChannelEvent<GeneratedTypemap>
+}>
 
 if (isMainThread || !parentPort) {
   throw new Error('This module must be run as a worker thread')
 }
 
+const report = createReporter<TypegenWorkerChannel>(parentPort)
 const opts = _workerData as TypegenGenerateTypesWorkerData
 
-registerBabel()
-
 async function main() {
-  const schema = await readSchema(opts.schemaPath)
+  const schemas: {
+    schema: SchemaType
+    projectId: string | 'default'
+    dataset: string | 'default'
+    filename: string
+  }[] = []
+
+  for (const schemaConfig of opts.schemas) {
+    $info(`Reading schema from ${schemaConfig.schemaPath}...`)
+    const schema = await readSchema(schemaConfig.schemaPath)
+    schemas.push({
+      schema,
+      projectId: schemaConfig.projectId,
+      dataset: schemaConfig.dataset,
+      filename: schemaConfig.schemaPath,
+    })
+  }
+  $info(`Read ${schemas.length} schema definition${schemas.length === 1 ? '' : 's'} successfully.`)
 
-  const typeGenerator = new TypeGenerator(schema)
-  const schemaTypes = [typeGenerator.generateSchemaTypes(), TypeGenerator.generateKnownTypes()]
-    .join('\n')
-    .trim()
   const resolver = getResolver()
 
-  parentPort?.postMessage({
-    type: 'schema',
-    schema: `${schemaTypes.trim()}\n`,
-    filename: 'schema.json',
-    length: schema.length,
-  } satisfies TypegenGenerateTypesWorkerMessage)
-
-  const queries = findQueriesInPath({
-    path: opts.searchPath,
-    resolver,
+  const result = generateTypes({
+    schemas,
+    queriesByFile: findQueriesInPath({path: opts.searchPath, resolver}),
+    augmentGroqModule: opts.augmentGroqModule,
+    overloadClientMethods: opts.overloadClientMethods,
   })
 
-  const allQueries = []
-
-  for await (const result of queries) {
-    if (result.type === 'error') {
-      parentPort?.postMessage({
-        type: 'error',
-        error: result.error,
-        fatal: false,
-        filename: result.filename,
-      } satisfies TypegenGenerateTypesWorkerMessage)
-      continue
-    }
-    $info(`Processing ${result.queries.length} queries in "${result.filename}"...`)
-
-    const fileQueryTypes: {
-      queryName: string
-      query: string
-      type: string
-      typeName: string
-      typeNode: TypeNode
-      unknownTypeNodesGenerated: number
-      typeNodesGenerated: number
-      emptyUnionTypeNodesGenerated: number
-    }[] = []
-    for (const {name: queryName, result: query} of result.queries) {
-      try {
-        const ast = safeParseQuery(query)
-        const queryTypes = typeEvaluate(ast, schema)
-
-        const typeName = `${queryName}Result`
-        const type = typeGenerator.generateTypeNodeTypes(typeName, queryTypes)
-
-        const queryTypeStats = walkAndCountQueryTypeNodeStats(queryTypes)
-        fileQueryTypes.push({
-          queryName,
-          query,
-          typeName,
-          typeNode: queryTypes,
-          type: `${type.trim()}\n`,
-          unknownTypeNodesGenerated: queryTypeStats.unknownTypes,
-          typeNodesGenerated: queryTypeStats.allTypes,
-          emptyUnionTypeNodesGenerated: queryTypeStats.emptyUnions,
-        })
-      } catch (err) {
-        parentPort?.postMessage({
-          type: 'error',
-          error: new Error(
-            `Error generating types for query "${queryName}" in "${result.filename}": ${err.message}`,
-            {cause: err},
-          ),
-          fatal: false,
-          query,
-        } satisfies TypegenGenerateTypesWorkerMessage)
-      }
-    }
+  report.event.schema(await result.generatedSchema())
 
-    if (fileQueryTypes.length > 0) {
-      $info(`Generated types for ${fileQueryTypes.length} queries in "${result.filename}"\n`)
-      parentPort?.postMessage({
-        type: 'types',
-        types: fileQueryTypes,
-        filename: result.filename,
-      } satisfies TypegenGenerateTypesWorkerMessage)
-    }
-
-    if (fileQueryTypes.length > 0) {
-      allQueries.push(...fileQueryTypes)
-    }
-  }
-
-  if (opts.overloadClientMethods && allQueries.length > 0) {
-    const typeMap = `${typeGenerator.generateQueryMap(allQueries).trim()}\n`
-    parentPort?.postMessage({
-      type: 'typemap',
-      typeMap,
-    } satisfies TypegenGenerateTypesWorkerMessage)
+  for await (const {filename, results} of result.generatedQueries()) {
+    report.stream.queries.emit({filename, results})
   }
+  report.stream.queries.end()
 
-  parentPort?.postMessage({
-    type: 'complete',
-  } satisfies TypegenGenerateTypesWorkerMessage)
-}
-
-function walkAndCountQueryTypeNodeStats(typeNode: TypeNode): {
-  allTypes: number
-  unknownTypes: number
-  emptyUnions: number
-} {
-  switch (typeNode.type) {
-    case 'unknown': {
-      return {allTypes: 1, unknownTypes: 1, emptyUnions: 0}
-    }
-    case 'array': {
-      const acc = walkAndCountQueryTypeNodeStats(typeNode.of)
-      acc.allTypes += 1 // count the array type itself
-      return acc
-    }
-    case 'object': {
-      // if the rest is unknown, we count it as one unknown type
-      if (typeNode.rest && typeNode.rest.type === 'unknown') {
-        return {allTypes: 2, unknownTypes: 1, emptyUnions: 0} // count the object type itself as well
-      }
-
-      const restStats = typeNode.rest
-        ? walkAndCountQueryTypeNodeStats(typeNode.rest)
-        : {allTypes: 1, unknownTypes: 0, emptyUnions: 0} // count the object type itself
-
-      return Object.values(typeNode.attributes).reduce((acc, attribute) => {
-        const {allTypes, unknownTypes, emptyUnions} = walkAndCountQueryTypeNodeStats(
-          attribute.value,
-        )
-        return {
-          allTypes: acc.allTypes + allTypes,
-          unknownTypes: acc.unknownTypes + unknownTypes,
-          emptyUnions: acc.emptyUnions + emptyUnions,
-        }
-      }, restStats)
-    }
-    case 'union': {
-      if (typeNode.of.length === 0) {
-        return {allTypes: 1, unknownTypes: 0, emptyUnions: 1}
-      }
-
-      return typeNode.of.reduce(
-        (acc, type) => {
-          const {allTypes, unknownTypes, emptyUnions} = walkAndCountQueryTypeNodeStats(type)
-          return {
-            allTypes: acc.allTypes + allTypes,
-            unknownTypes: acc.unknownTypes + unknownTypes,
-            emptyUnions: acc.emptyUnions + emptyUnions,
-          }
-        },
-        {allTypes: 1, unknownTypes: 0, emptyUnions: 0}, // count the union type itself
-      )
-    }
-    default: {
-      return {allTypes: 1, unknownTypes: 0, emptyUnions: 0}
-    }
-  }
+  report.event.typemap(await result.generatedTypemap())
 }
 
+registerBabel()
 main()