@tamagui/compiler-core 0.0.0-bootstrap.0 → 3.0.0-beta.637.1

package/src/output.ts

@@ -0,0 +1,175 @@
+import {
+  GenMapping,
+  addSegment,
+  setSourceContent,
+  toEncodedMap,
+} from '@jridgewell/gen-mapping'
+import MagicString from 'magic-string'
+
+import type { ResolvedModuleId, SourceSpan } from './contracts'
+import { contentHash } from './hash'
+
+export interface SourceEdit {
+  /** UTF-16 source-string index, inclusive. */
+  start: number
+  /** UTF-16 source-string index, exclusive. Equal to start for an insertion. */
+  end: number
+  content: string
+  origin: SourceSpan
+}
+
+export interface CompilerSourceMap {
+  version: 3
+  file?: string
+  names: readonly string[]
+  sources: readonly (string | null)[]
+  sourcesContent: readonly (string | null)[]
+  mappings: string
+}
+
+export interface AppliedLoweredModule {
+  changed: boolean
+  code: string
+  map: CompilerSourceMap | null
+}
+
+export interface ApplicableLoweredModulePlan {
+  id: ResolvedModuleId
+  sourceHash: string
+  edits: readonly SourceEdit[]
+}
+
+function compareEdits(
+  left: SourceEdit & { index: number },
+  right: SourceEdit & { index: number }
+): number {
+  return left.start - right.start || left.end - right.end || left.index - right.index
+}
+
+export function sourceContentHash(source: string): string {
+  return contentHash(source)
+}
+
+export function validateSourceEdits(source: string, edits: readonly SourceEdit[]): void {
+  const sorted = edits.map((edit, index) => ({ ...edit, index })).sort(compareEdits)
+  let previousEnd = 0
+  for (const edit of sorted) {
+    if (
+      !Number.isInteger(edit.start) ||
+      !Number.isInteger(edit.end) ||
+      edit.start < 0 ||
+      edit.end < edit.start ||
+      edit.end > source.length
+    ) {
+      throw new Error(`Invalid UTF-16 source edit [${edit.start}, ${edit.end})`)
+    }
+    if (edit.start < previousEnd) {
+      throw new Error(`Overlapping UTF-16 source edit at ${edit.start}`)
+    }
+    previousEnd = Math.max(previousEnd, edit.end)
+  }
+}
+
+interface Position {
+  line: number
+  column: number
+}
+
+function sourcePositions(source: string): Position[] {
+  const positions: Position[] = new Array(source.length + 1)
+  let line = 0
+  let column = 0
+  for (let index = 0; index <= source.length; index++) {
+    positions[index] = { line, column }
+    if (index === source.length) break
+    if (source.charCodeAt(index) === 10) {
+      line++
+      column = 0
+    } else {
+      column++
+    }
+  }
+  return positions
+}
+
+function mappedSourceMap(
+  source: string,
+  id: ResolvedModuleId,
+  edits: readonly (SourceEdit & { index: number })[]
+): CompilerSourceMap {
+  const map = new GenMapping()
+  setSourceContent(map, id, source)
+  const positions = sourcePositions(source)
+  let generatedLine = 0
+  let generatedColumn = 0
+  let sourceCursor = 0
+
+  const emit = (content: string, sourceStart: number, sourceEnd: number) => {
+    const available = Math.max(1, sourceEnd - sourceStart)
+    for (let offset = 0; offset < content.length; offset++) {
+      const originalIndex = Math.min(sourceStart + offset, sourceStart + available - 1)
+      const original = positions[originalIndex]!
+      addSegment(map, generatedLine, generatedColumn, id, original.line, original.column)
+      if (content.charCodeAt(offset) === 10) {
+        generatedLine++
+        generatedColumn = 0
+      } else {
+        generatedColumn++
+      }
+    }
+  }
+
+  for (const edit of edits) {
+    emit(source.slice(sourceCursor, edit.start), sourceCursor, edit.start)
+    if (edit.origin.id !== id) {
+      throw new Error(`Source edit for ${id} must use a local mapping origin`)
+    }
+    emit(
+      edit.content,
+      edit.origin.start,
+      edit.start === edit.end ? edit.origin.start + 1 : edit.origin.end
+    )
+    sourceCursor = edit.end
+  }
+  emit(source.slice(sourceCursor), sourceCursor, source.length)
+  const encoded = toEncodedMap(map)
+  return {
+    version: 3,
+    file: encoded.file ?? undefined,
+    names: encoded.names,
+    sources: encoded.sources,
+    sourcesContent: encoded.sourcesContent ?? [source],
+    mappings: encoded.mappings,
+  }
+}
+
+/** The only compiler-core path that applies source edits and owns their source map. */
+export function applyLoweredModule(
+  source: string,
+  id: ResolvedModuleId,
+  plan: ApplicableLoweredModulePlan
+): AppliedLoweredModule {
+  if (plan.id !== id) {
+    throw new Error(`Lowered module plan ${plan.id} cannot be applied to ${id}`)
+  }
+  if (plan.sourceHash !== sourceContentHash(source)) {
+    throw new Error(`Lowered module plan for ${id} does not match the supplied source`)
+  }
+  if (plan.edits.length === 0) return { changed: false, code: source, map: null }
+  validateSourceEdits(source, plan.edits)
+
+  const output = new MagicString(source)
+  const sorted = plan.edits.map((edit, index) => ({ ...edit, index })).sort(compareEdits)
+  for (const edit of sorted) {
+    if (edit.start === edit.end) {
+      output.appendLeft(edit.start, edit.content)
+    } else {
+      output.overwrite(edit.start, edit.end, edit.content)
+    }
+  }
+  return {
+    changed: true,
+    code: output.toString(),
+    map: mappedSourceMap(source, id, sorted),
+  }
+}

package/src/planCache.ts

@@ -0,0 +1,327 @@
+import { randomBytes } from 'node:crypto'
+import {
+  mkdir,
+  readdir,
+  readFile,
+  rename,
+  stat,
+  unlink,
+  writeFile,
+} from 'node:fs/promises'
+import { dirname, join } from 'node:path'
+
+import type { HostModuleInput, ResolvedModuleId } from './contracts'
+import { contentHash, stableStringify } from './hash'
+import { LOWERED_MODULE_PLAN_VERSION, type LoweredModulePlan } from './lower'
+import { moduleContentHash } from './graph'
+
+export const PLAN_CACHE_SCHEMA_VERSION = 1
+
+/**
+ * Entries are content addressed, so every edit of a shared module writes new
+ * files and nothing ever replaces the old ones. Twenty edits of one module a
+ * hundred consumers import leaves two thousand dead entries that would live
+ * forever. A pruned entry is just a miss, and a miss recompiles, so dropping
+ * entries can never be wrong - only slower.
+ *
+ * The cap has to sit well above the module count of a real project, or an
+ * unchanged rebuild of a project larger than the cap would miss on every module
+ * the last build evicted. Twenty thousand entries is a few hundred MB at the
+ * high end and comfortably holds a large app plus several generations of edits.
+ *
+ * It is a soft cap: pruning runs between batches of writes, so a store can sit
+ * up to one batch above it. Bounding growth is the point, not an exact size.
+ */
+export const PLAN_CACHE_MAX_ENTRIES = 20_000
+
+/**
+ * The one on-disk cache location for a project. Metro's plan manifest already
+ * lives under it, and the build-time benchmark's "cold" state is defined as
+ * deleting it, so everything the compiler persists belongs here and nowhere
+ * else.
+ */
+export function tamaguiCacheRoot(projectRoot: string): string {
+  return join(projectRoot, 'node_modules', '.cache', 'tamagui')
+}
+
+export function defaultPlanCacheRoot(projectRoot: string, target: string): string {
+  return join(tamaguiCacheRoot(projectRoot), 'plans', target)
+}
+
+/**
+ * Everything a plan depends on that is not a module: the compiler build, the
+ * evaluated Tamagui config and component registry, the platform, and the
+ * structural pass. `stamp` comes from `loadCompilerProject`'s `cacheStamp`.
+ */
+export interface PlanCacheIdentity {
+  stamp: string
+  target: string
+  structuralPassHash: string
+}
+
+export interface PlanCacheEntry {
+  schemaVersion: typeof PLAN_CACHE_SCHEMA_VERSION
+  moduleId: ResolvedModuleId
+  /** Digest of this module plus its whole non-external import closure. */
+  closureDigest: string
+  plan: LoweredModulePlan
+}
+
+export interface ModuleClosureNode {
+  contentHash: string
+  /** Non-external dependencies only: the edges symbol resolution can cross. */
+  dependencies: readonly ResolvedModuleId[]
+}
+
+export type ModuleClosureLookup = (id: ResolvedModuleId) => ModuleClosureNode | null
+
+/**
+ * Reads a closure node straight off host module records, for callers that have
+ * not built a ProjectGraph (Metro's prepass skips it entirely on a full hit).
+ */
+export function moduleClosureNode(input: HostModuleInput): ModuleClosureNode {
+  return {
+    contentHash: moduleContentHash(input),
+    dependencies: input.imports
+      .filter(({ external }) => !external)
+      .map(({ resolvedId }) => resolvedId),
+  }
+}
+
+/**
+ * Identity of a module's whole compile input: itself plus every module reachable
+ * from it over non-external imports, each by content hash.
+ *
+ * This is deliberately the import closure rather than the dependency set a
+ * successful compile recorded. The recorded set is only known after compiling,
+ * and it omits the module that made a value bail, so a dependency edit that
+ * turns a bailout into a static value would not invalidate its consumer, which
+ * is exactly how a stale style ships. The import closure is the same edge set
+ * `ProjectGraph.affectedBy` propagates over in memory, so it can never
+ * invalidate less than a live session would.
+ *
+ * Null when any module in the closure is unknown: an unknown input is a miss,
+ * never a guess.
+ */
+export function moduleClosureDigest(
+  id: ResolvedModuleId,
+  lookup: ModuleClosureLookup,
+  memo?: Map<ResolvedModuleId, string | null>
+): string | null {
+  const cached = memo?.get(id)
+  if (cached !== undefined) return cached
+  const reached = new Map<ResolvedModuleId, string>()
+  const queue: ResolvedModuleId[] = [id]
+  let complete = true
+  while (queue.length) {
+    const current = queue.pop()!
+    if (reached.has(current)) continue
+    const node = lookup(current)
+    if (!node) {
+      complete = false
+      break
+    }
+    reached.set(current, node.contentHash)
+    for (const dependency of node.dependencies) queue.push(dependency)
+  }
+  const digest = complete
+    ? contentHash(
+        [...reached]
+          .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
+          .map(([current, hash]) => `${current}\0${hash}`)
+          .join('\0')
+      )
+    : null
+  memo?.set(id, digest)
+  return digest
+}
+
+export function planCacheKey(
+  identity: PlanCacheIdentity,
+  id: ResolvedModuleId,
+  closureDigest: string
+): string {
+  return contentHash(
+    stableStringify({
+      schema: PLAN_CACHE_SCHEMA_VERSION,
+      plan: LOWERED_MODULE_PLAN_VERSION,
+      stamp: identity.stamp,
+      target: identity.target,
+      structuralPassHash: identity.structuralPassHash,
+      id,
+      closureDigest,
+    })
+  )
+}
+
+/**
+ * Content-addressed JSON entries on disk, one file per key. There is no
+ * manifest on purpose: a manifest is what makes a cache all-or-nothing, and the
+ * point of these caches is that editing one module leaves every other module's
+ * entry valid.
+ */
+export class JsonFileCache {
+  #hits = 0
+  #misses = 0
+  #writes = 0
+  #writesSincePrune = 0
+  #prunedThisProcess = false
+  readonly #created = new Set<string>()
+
+  constructor(
+    readonly root: string,
+    readonly schemaVersion: number,
+    readonly maxEntries: number = PLAN_CACHE_MAX_ENTRIES
+  ) {}
+
+  get stats(): { hits: number; misses: number; writes: number } {
+    return { hits: this.#hits, misses: this.#misses, writes: this.#writes }
+  }
+
+  resetStats(): void {
+    this.#hits = 0
+    this.#misses = 0
+    this.#writes = 0
+  }
+
+  #path(key: string): string {
+    return join(this.root, `v${this.schemaVersion}`, key.slice(0, 2), `${key}.json`)
+  }
+
+  /**
+   * Null on anything short of an entry `validate` fully accepts. A miss
+   * recompiles; nothing here repairs or partially trusts an entry, and a later
+   * successful compile overwrites the bad file.
+   */
+  async read<T>(key: string, validate: (value: unknown) => T | null): Promise<T | null> {
+    let parsed: unknown
+    try {
+      parsed = JSON.parse(await readFile(this.#path(key), 'utf8'))
+    } catch {
+      this.#misses++
+      return null
+    }
+    const entry = validate(parsed)
+    if (entry === null) {
+      this.#misses++
+      return null
+    }
+    this.#hits++
+    return entry
+  }
+
+  async write(key: string, value: unknown): Promise<void> {
+    const path = this.#path(key)
+    const directory = dirname(path)
+    // a full first build writes one entry per module, so the directory syscall
+    // is worth remembering
+    if (!this.#created.has(directory)) {
+      await mkdir(directory, { recursive: true })
+      this.#created.add(directory)
+    }
+    const temporaryPath = `${path}.${process.pid}-${randomBytes(6).toString('hex')}.tmp`
+    await writeFile(temporaryPath, `${stableStringify(value)}\n`, 'utf8')
+    await rename(temporaryPath, path)
+    this.#writes++
+    // Once per process, plus once per cap-sized batch inside a long-lived one.
+    // Checking only the batch counter would never prune the common case, where
+    // every build is a fresh process writing far fewer entries than the cap.
+    this.#writesSincePrune++
+    const batch = Math.max(1, Math.floor(this.maxEntries / 4))
+    if (!this.#prunedThisProcess || this.#writesSincePrune >= batch) {
+      this.#prunedThisProcess = true
+      this.#writesSincePrune = 0
+      await this.#prune()
+    }
+  }
+
+  /**
+   * Drops the oldest entries once the store exceeds its cap, oldest by the later
+   * of read and write time so an entry a build keeps hitting is not treated as
+   * dead on filesystems that maintain access times.
+   */
+  async #prune(): Promise<void> {
+    const versionRoot = join(this.root, `v${this.schemaVersion}`)
+    let shards: string[]
+    try {
+      shards = await readdir(versionRoot)
+    } catch {
+      return
+    }
+    const paths: string[] = []
+    for (const shard of shards) {
+      const directory = join(versionRoot, shard)
+      let files: string[]
+      try {
+        files = await readdir(directory)
+      } catch {
+        continue
+      }
+      for (const file of files) {
+        if (file.endsWith('.json')) paths.push(join(directory, file))
+      }
+    }
+    // names are enough to know whether pruning is needed, so a store under its
+    // cap costs one readdir per shard and no stat at all
+    if (paths.length <= this.maxEntries) return
+    const entries: { path: string; usedAt: number }[] = []
+    for (const path of paths) {
+      try {
+        const stats = await stat(path)
+        entries.push({ path, usedAt: Math.max(stats.atimeMs, stats.mtimeMs) })
+      } catch {
+        // a concurrent build may have pruned it already
+      }
+    }
+    entries.sort((left, right) => left.usedAt - right.usedAt)
+    for (const entry of entries.slice(0, Math.max(0, entries.length - this.maxEntries))) {
+      await unlink(entry.path).catch(() => {})
+    }
+  }
+}
+
+/** Per-module lowering plans, keyed by `planCacheKey`. */
+export class ModulePlanCache {
+  readonly #files: JsonFileCache
+
+  constructor(readonly root: string) {
+    this.#files = new JsonFileCache(root, PLAN_CACHE_SCHEMA_VERSION)
+  }
+
+  get stats(): { hits: number; misses: number; writes: number } {
+    return this.#files.stats
+  }
+
+  resetStats(): void {
+    this.#files.resetStats()
+  }
+
+  read(
+    key: string,
+    id: ResolvedModuleId,
+    closureDigest: string
+  ): Promise<PlanCacheEntry | null> {
+    return this.#files.read(key, (value) => {
+      const entry = value as PlanCacheEntry | null
+      const plan = entry?.plan
+      return entry?.schemaVersion === PLAN_CACHE_SCHEMA_VERSION &&
+        entry.moduleId === id &&
+        entry.closureDigest === closureDigest &&
+        plan &&
+        plan.version === LOWERED_MODULE_PLAN_VERSION &&
+        plan.id === id &&
+        typeof plan.sourceHash === 'string' &&
+        typeof plan.css === 'string' &&
+        Array.isArray(plan.edits) &&
+        Array.isArray(plan.diagnostics) &&
+        Array.isArray(plan.dependencies) &&
+        !!plan.stats
+        ? entry
+        : null
+    })
+  }
+
+  write(key: string, entry: PlanCacheEntry): Promise<void> {
+    return this.#files.write(key, entry)
+  }
+}

package/src/session.ts

@@ -0,0 +1,172 @@
+import type { HostModuleInput, ResolvedModuleId } from './contracts'
+import type { GraphInvalidation } from './graph'
+import { ProjectGraph } from './graph'
+import type {
+  CompilerLoweringHost,
+  CompilerTarget,
+  LoweredModulePlan,
+  StructuralModulePass,
+} from './lower'
+import { lowerModule } from './lower'
+import { materializeModule } from './materialize'
+import type { AppliedLoweredModule } from './output'
+import { applyLoweredModule } from './output'
+import type { ModulePlanCache } from './planCache'
+import { PLAN_CACHE_SCHEMA_VERSION, moduleClosureDigest, planCacheKey } from './planCache'
+import { yukuFactory } from './yuku'
+
+export interface CompilerAdapter {
+  target: CompilerTarget
+  projectGeneration: string
+  host: CompilerLoweringHost
+  load(id: ResolvedModuleId): Promise<HostModuleInput | null>
+  /**
+   * Persistent per-module plan reuse across processes. Absent means the host
+   * could not produce a content stamp for this project, so nothing is cached
+   * rather than cached under a stamp that does not describe the config.
+   */
+  planCache?: { store: ModulePlanCache; stamp: string }
+}
+
+export interface CompileModuleInput {
+  module: HostModuleInput
+  adapter: CompilerAdapter
+  structuralPass?: StructuralModulePass
+}
+
+export interface CompilerSessionResult {
+  plan: LoweredModulePlan
+  output: AppliedLoweredModule
+  invalidatedIds: ResolvedModuleId[]
+}
+
+function compareIds(left: ResolvedModuleId, right: ResolvedModuleId): number {
+  return left < right ? -1 : left > right ? 1 : 0
+}
+
+/**
+ * Bundler-neutral compiler state. The adapter owns module resolution and loading;
+ * the session only accepts canonical host-resolved module records.
+ */
+export class CompilerSession {
+  readonly #graph = new ProjectGraph(yukuFactory, { modules: [] })
+  #queue: Promise<unknown> = Promise.resolve()
+
+  compile(input: CompileModuleInput): Promise<CompilerSessionResult> {
+    const operation = this.#queue.then(() => this.#compile(input))
+    this.#queue = operation.catch(() => undefined)
+    return operation
+  }
+
+  update(module: HostModuleInput): Promise<ResolvedModuleId[]> {
+    return this.#enqueue(() => this.#graph.updateModule(module).invalidatedIds)
+  }
+
+  has(id: ResolvedModuleId): boolean {
+    return this.#graph.contentHash(id) !== null
+  }
+
+  dependentsOf(id: ResolvedModuleId): ResolvedModuleId[] {
+    return this.#graph.dependentsOf(id)
+  }
+
+  remove(id: ResolvedModuleId): Promise<GraphInvalidation> {
+    return this.#enqueue(() => this.#graph.removeModule(id))
+  }
+
+  parseCount(id: ResolvedModuleId): number {
+    return this.#graph.parseCount(id)
+  }
+
+  #enqueue<T>(operation: () => T | Promise<T>): Promise<T> {
+    const queued = this.#queue.then(operation)
+    this.#queue = queued.catch(() => undefined)
+    return queued
+  }
+
+  async #compile({
+    module,
+    adapter,
+    structuralPass,
+  }: CompileModuleInput): Promise<CompilerSessionResult> {
+    const invalidated = new Set<ResolvedModuleId>()
+    await this.#install(module, adapter, new Set(), invalidated)
+    const graph = this.#graph
+    // an empty stamp is not an identity: a key without one would be shared by
+    // every project and every config, so such a project simply does not cache
+    const cache = adapter.planCache?.stamp ? adapter.planCache : null
+    const closureDigest = cache
+      ? moduleClosureDigest(module.id, (id) => {
+          const hash = graph.contentHash(id)
+          return hash
+            ? { contentHash: hash, dependencies: graph.dependenciesOf(id) }
+            : null
+        })
+      : null
+    const entry =
+      cache && closureDigest
+        ? {
+            store: cache.store,
+            digest: closureDigest,
+            key: planCacheKey(
+              {
+                stamp: cache.stamp,
+                target: adapter.target,
+                structuralPassHash:
+                  structuralPass?.versionHash ?? `${adapter.target}-noop-v1`,
+              },
+              module.id,
+              closureDigest
+            ),
+          }
+        : null
+
+    const cached = entry
+      ? await entry.store.read(entry.key, module.id, entry.digest)
+      : null
+    const plan =
+      cached?.plan ??
+      lowerModule({
+        module: materializeModule(graph, module.id),
+        source: module.source,
+        target: adapter.target,
+        host: adapter.host,
+        options: { projectGeneration: adapter.projectGeneration },
+        structuralPass,
+      })
+    if (entry && !cached) {
+      await entry.store.write(entry.key, {
+        schemaVersion: PLAN_CACHE_SCHEMA_VERSION,
+        moduleId: module.id,
+        closureDigest: entry.digest,
+        plan,
+      })
+    }
+    return {
+      plan,
+      output: applyLoweredModule(module.source, module.id, plan),
+      invalidatedIds: [...invalidated].sort(compareIds),
+    }
+  }
+
+  async #install(
+    module: HostModuleInput,
+    adapter: CompilerAdapter,
+    visited: Set<ResolvedModuleId>,
+    invalidated: Set<ResolvedModuleId>
+  ): Promise<void> {
+    if (visited.has(module.id)) return
+    visited.add(module.id)
+
+    for (const dependency of module.imports) {
+      if (dependency.external || this.has(dependency.resolvedId)) continue
+      const loaded = await adapter.load(dependency.resolvedId)
+      if (loaded) {
+        await this.#install(loaded, adapter, visited, invalidated)
+      }
+    }
+
+    const update = this.#graph.updateModule(module)
+    for (const id of update.invalidatedIds) invalidated.add(id)
+  }
+}