cross-packer 0.1.0

package/src/asar/fileset/walker.ts

@@ -0,0 +1,115 @@
+import { type Dirent, existsSync, lstatSync, readdirSync, readlinkSync, statSync } from 'node:fs'
+import path from 'node:path'
+import type { FileStat } from '../../shared/types.ts'
+import { EntryKind, entryKind } from '../../shared/types.ts'
+
+export function walkDirectory(
+  dirPath: string,
+  files: string[],
+  metadata: Map<string, FileStat>,
+  projectDir: string,
+): void {
+  if (!existsSync(dirPath)) return
+
+  let entries: Dirent[]
+  try {
+    entries = readdirSync(dirPath, { withFileTypes: true })
+  } catch {
+    return
+  }
+
+  for (const entry of entries) {
+    const fullPath = path.join(dirPath, entry.name)
+
+    let stat: ReturnType<typeof lstatSync>
+    try {
+      stat = lstatSync(fullPath)
+    } catch {
+      continue
+    }
+
+    const type = entryKind(stat)
+    if (type === null) continue
+
+    const meta: FileStat = { type, size: stat.size, mode: stat.mode }
+
+    const nodeModulesIdx = fullPath.lastIndexOf(`${path.sep}node_modules${path.sep}`)
+    if (nodeModulesIdx !== -1) {
+      const afterNodeModules = fullPath.slice(nodeModulesIdx + '/node_modules/'.length)
+      const moduleName = afterNodeModules.split(path.sep)[0]
+      if (moduleName.startsWith('@')) {
+        const scopedName = afterNodeModules.split(path.sep).slice(0, 2).join(path.sep)
+        meta.moduleRootPath = fullPath.slice(
+          0,
+          nodeModulesIdx + '/node_modules/'.length + scopedName.length,
+        )
+        meta.moduleName = scopedName
+      } else {
+        meta.moduleRootPath = fullPath.slice(
+          0,
+          nodeModulesIdx + '/node_modules/'.length + moduleName.length,
+        )
+        meta.moduleName = moduleName
+      }
+      meta.moduleFullFilePath = fullPath
+    }
+
+    if (meta.type === EntryKind.LINK) {
+      let linkTarget: string
+      try {
+        linkTarget = readlinkSync(fullPath)
+      } catch {
+        continue
+      }
+      const resolvedLinkTarget = path.resolve(dirPath, linkTarget)
+      const link = path.relative(projectDir, resolvedLinkTarget)
+      if (link.startsWith('..')) {
+        try {
+          const targetStat = statSync(resolvedLinkTarget)
+          const targetType = entryKind(targetStat)
+          if (targetType === null) continue
+          metadata.set(fullPath, {
+            type: targetType,
+            size: targetStat.size,
+            mode: targetStat.mode,
+          })
+          files.push(fullPath)
+          continue
+        } catch {
+          continue
+        }
+      }
+      meta.relativeLink = link
+      meta.linkRelativeToFile = path.relative(dirPath, resolvedLinkTarget)
+    }
+
+    files.push(fullPath)
+    metadata.set(fullPath, meta)
+
+    if (meta.type === EntryKind.DIRECTORY) {
+      walkDirectory(fullPath, files, metadata, projectDir)
+    }
+  }
+}
+
+export function mergeFileSets(
+  set1: { files: string[]; metadata: Map<string, FileStat> },
+  set2: { files: string[]; metadata: Map<string, FileStat> },
+): { files: string[]; metadata: Map<string, FileStat> } {
+  const metadata = new Map([...set1.metadata, ...set2.metadata])
+  const fileSet = new Set([...set1.files, ...set2.files])
+  const files = Array.from(fileSet).sort()
+  return { files, metadata }
+}
+
+/** Derive the deepest common ancestor directory of a file list. */
+export function deriveProjectDir(files: string[]): string {
+  if (files.length === 0) return process.cwd()
+  let prefix = files[0]
+  for (const f of files) {
+    while (!f.startsWith(prefix + path.sep) && f !== prefix) {
+      prefix = path.dirname(prefix)
+    }
+  }
+  return prefix
+}

package/src/asar/pack.ts

@@ -0,0 +1,186 @@
+import os from 'node:os'
+import path from 'node:path'
+import { Minimatch } from 'minimatch'
+import type { NormalizedConfig } from '../config/index.ts'
+import { logger } from '../shared/logger.ts'
+import { type Arch, NodePlatform, nodePlatformOf, Platform } from '../shared/platform.ts'
+import type { FileFilter, FileSet, FileStat, JsonObject } from '../shared/types.ts'
+import { createFilter, hasMagic } from './fileset/matcher.ts'
+import { derivePlatformFileSet } from './fileset/platform-fileset.ts'
+import { collectAllFiles } from './packer/collector.ts'
+import { mapWithConcurrency } from './packer/concurrency.ts'
+import { packAsarFromFileSets } from './packer/stream-builder.ts'
+
+const TRANSFORM_CONCURRENCY = Number(
+  process.env.CROSS_PACKER_ASAR_TRANSFORM_CONCURRENCY ?? Math.max(4, os.cpus().length),
+)
+
+const PLATFORM_NAMES: Record<Platform, NodePlatform> = {
+  [Platform.WIN]: NodePlatform.WIN32,
+  [Platform.MAC]: NodePlatform.DARWIN,
+  [Platform.LINUX]: NodePlatform.LINUX,
+}
+
+/** A FileSet plus the transformed contents collected during packing. */
+interface PackedFileSet extends FileSet {
+  transformedFiles?: Map<number, string>
+}
+
+type Transformer = (file: string) => Promise<string | null> | string | null
+
+/** A files-config entry: either a glob string or a mapping object with a filter. */
+interface FilesConfigEntry {
+  from?: string
+  filter?: string | string[]
+}
+
+export async function collectProjectFiles(projectDir: string, config: NormalizedConfig) {
+  const nodeModuleFileFilter = createNodeModuleFileFilter(config.files, projectDir)
+
+  logger.info('\n  Collecting all files (single traversal) ...')
+  const hostPlatform = nodePlatformOf(process.platform)
+  const { appFileSet, nmFileSets, transformer } = await collectAllFiles(projectDir, {
+    platform: hostPlatform,
+    appDistDir: config.appDistDir,
+    includePdb: false,
+    disableDefaultIgnoredFiles: config.disableDefaultIgnoredFiles ?? false,
+    workspaceRoot: projectDir,
+    extraMetadata: config.extraMetadata,
+    nodeModuleFileFilter: nodeModuleFileFilter,
+    onNodeModuleFile: config.onNodeModuleFile,
+  })
+  logger.info(`  App files: ${appFileSet.files.length} entries`)
+  logger.info(`  node_modules fileSets: ${nmFileSets.length}`)
+
+  return { appFileSet, nmFileSets, transformer }
+}
+
+export async function packAsarForTarget({
+  platform,
+  arch,
+  appFileSet,
+  nmFileSets,
+  transformer,
+  config,
+  projectDir,
+  stagingDir,
+}: {
+  platform: Platform
+  arch: Arch
+  appFileSet: FileSet
+  nmFileSets: FileSet[]
+  transformer: Transformer
+  config: NormalizedConfig
+  projectDir: string
+  stagingDir: string
+}) {
+  const platformName = PLATFORM_NAMES[platform]
+  const asarOutputPath = path.join(stagingDir, `app-${platform}-${arch}.asar`)
+
+  const label = `${platform}-${arch}`
+  logger.info(`  [${label}] Deriving platform file set ...`)
+  const platformAppFileSet = derivePlatformFileSet(appFileSet, platformName, arch, projectDir)
+  const platformNmFileSets = nmFileSets.map((s) =>
+    derivePlatformFileSet(s, platformName, arch, projectDir),
+  )
+  const allFileSets: PackedFileSet[] = [platformAppFileSet, ...platformNmFileSets]
+  logger.info(`  [${label}] Total fileSets: ${allFileSets.length}`)
+
+  interface TransformTask {
+    fileSet: PackedFileSet
+    index: number
+    file: string
+    fileStat: FileStat
+  }
+  const transformTasks: TransformTask[] = []
+  for (const fileSet of allFileSets) {
+    fileSet.transformedFiles = new Map()
+    for (let i = 0; i < fileSet.files.length; i++) {
+      const file = fileSet.files[i]
+      const fileStat = fileSet.metadata.get(file)
+      if (fileStat?.type !== 'file') continue
+      transformTasks.push({ fileSet, index: i, file, fileStat })
+    }
+  }
+
+  await mapWithConcurrency(
+    transformTasks,
+    TRANSFORM_CONCURRENCY,
+    async ({ fileSet, index, file, fileStat }: TransformTask) => {
+      const fileToRead = fileStat.realFilePath || file
+      const transformedValue = await transformer(fileToRead)
+      if (transformedValue != null) {
+        fileSet.transformedFiles?.set(index, transformedValue)
+      }
+    },
+  )
+
+  const packStartedAt = performance.now()
+  logger.info(`  [${label}] Packing asar ...`)
+  const { asarPath, unpackedDir } = await packAsarFromFileSets(allFileSets, {
+    outputPath: asarOutputPath,
+    unpackPattern: createAsarUnpackFilter(config.asarUnpack, projectDir),
+    smartUnpack: config.smartUnpack,
+    defaultDestination: projectDir,
+    workspaceRoot: projectDir,
+  })
+  const packElapsedSec = ((performance.now() - packStartedAt) / 1000).toFixed(1)
+  logger.info(`  [${label}] app.asar: ${asarPath} (${packElapsedSec}s)`)
+
+  return { asarPath, unpackedDir }
+}
+
+function isFilesConfigEntry(value: unknown): value is FilesConfigEntry {
+  return typeof value === 'object' && value !== null && !Array.isArray(value)
+}
+
+/** Collect exclusion patterns from a files config entry. */
+function addPatterns(patterns: unknown, excludePatterns: string[]): void {
+  if (patterns == null) return
+  if (!Array.isArray(patterns)) {
+    if (typeof patterns === 'string' && patterns.startsWith('!')) {
+      excludePatterns.push(patterns)
+    }
+    return
+  }
+  for (const pattern of patterns) {
+    if (typeof pattern === 'string') {
+      if (pattern.startsWith('!')) {
+        excludePatterns.push(pattern)
+      }
+    } else if (isFilesConfigEntry(pattern) && (pattern.from == null || pattern.from === '.')) {
+      const filter = pattern.filter
+      const filterList = Array.isArray(filter) ? filter : typeof filter === 'string' ? [filter] : []
+      for (const p of filterList) {
+        excludePatterns.push(p)
+      }
+    }
+  }
+}
+
+function buildMinimatchPatterns(patterns: string[]): Minimatch[] {
+  const parsedPatterns: Minimatch[] = []
+  for (const pattern of patterns) {
+    const parsedPattern = new Minimatch(pattern, { dot: true })
+    parsedPatterns.push(parsedPattern)
+    if (!pattern.includes('.') && !pattern.startsWith('!') && !hasMagic(parsedPattern)) {
+      parsedPatterns.push(new Minimatch(`${pattern}/**/*`, { dot: true }))
+    }
+  }
+  return parsedPatterns
+}
+
+function createAsarUnpackFilter(patterns: string[], baseDir: string): FileFilter {
+  return createFilter(baseDir, buildMinimatchPatterns(patterns))
+}
+
+function createNodeModuleFileFilter(
+  filesConfig: JsonObject | undefined,
+  baseDir: string,
+): FileFilter | undefined {
+  const excludePatterns: string[] = []
+  addPatterns(filesConfig, excludePatterns)
+  if (excludePatterns.length === 0) return undefined
+  const allPatterns = ['**/*', ...excludePatterns.map((p) => (p.startsWith('!') ? p : `!${p}`))]
+  return createFilter(baseDir, buildMinimatchPatterns(allPatterns))
+}

package/src/asar/packer/app-collector.ts

@@ -0,0 +1,74 @@
+import { existsSync } from 'node:fs'
+import { lstat } from 'node:fs/promises'
+import path from 'node:path'
+import { Minimatch } from 'minimatch'
+import { entryKind, type FileSet, type FileStat } from '../../shared/types.ts'
+import { createFilter, excludedExts, excludedNames } from '../fileset/matcher.ts'
+import { walkDirectory } from '../fileset/walker.ts'
+
+export async function collectAppFileSet(
+  projectDir: string,
+  distDir: string,
+  includePdb: boolean,
+): Promise<FileSet> {
+  const metadata = new Map<string, FileStat>()
+  const files: string[] = []
+
+  walkDirectory(distDir, files, metadata, projectDir)
+
+  const pkgJsonPath = path.join(projectDir, 'package.json')
+  if (existsSync(pkgJsonPath)) {
+    const stat = await lstat(pkgJsonPath)
+    const type = entryKind(stat)
+    if (type === null) {
+      throw new Error(`package.json is not a regular file: ${pkgJsonPath}`)
+    }
+    files.push(pkgJsonPath)
+    metadata.set(pkgJsonPath, { type, size: stat.size, mode: stat.mode })
+  }
+
+  const patterns = computeAppPatterns(projectDir, includePdb)
+  const appFilter = createFilter(projectDir, patterns)
+
+  const filtered = files.filter((f) => {
+    if (f === projectDir) return true
+    const stat = metadata.get(f)
+    if (!stat) return false
+    return appFilter(f, stat)
+  })
+  const filteredMetadata = new Map<string, FileStat>()
+  for (const f of filtered) {
+    const m = metadata.get(f)
+    if (m) filteredMetadata.set(f, m)
+  }
+
+  filtered.sort()
+  return {
+    src: projectDir,
+    destination: projectDir,
+    files: filtered,
+    metadata: filteredMetadata,
+  }
+}
+
+function computeAppPatterns(src: string, includePdb: boolean): Minimatch[] {
+  const patterns: Minimatch[] = []
+  patterns.push(new Minimatch('**/*', { dot: true }))
+  patterns.push(new Minimatch('!**/node_modules/**', { dot: true }))
+  const relativeOutDir = path.relative(src, path.join(src, '..', 'dist'))
+  if (relativeOutDir.length !== 0 && !relativeOutDir.startsWith('.')) {
+    patterns.push(new Minimatch(`!${relativeOutDir}{,/**/*}`, { dot: true }))
+  }
+  const effectiveExts = includePdb === true ? excludedExts : `${excludedExts},pdb`
+  patterns.push(new Minimatch(`!**/*.{${effectiveExts}}`, { dot: true }))
+  patterns.push(new Minimatch('!**/._*', { dot: true }))
+  patterns.push(new Minimatch('!**/electron-builder.{yaml,yml,json,json5,toml,ts}', { dot: true }))
+  const excludedNamesList = excludedNames.split(',')
+  for (const name of excludedNamesList) {
+    patterns.push(new Minimatch(`!**/${name}`, { dot: true }))
+  }
+  patterns.push(new Minimatch('!.yarn{,/**/*}', { dot: true }))
+  patterns.push(new Minimatch('!.editorconfig', { dot: true }))
+  patterns.push(new Minimatch('!.yarnrc.yml', { dot: true }))
+  return patterns
+}

package/src/asar/packer/collector.ts

@@ -0,0 +1,134 @@
+import { existsSync } from 'node:fs'
+import path from 'node:path'
+import { logger } from '../../shared/logger.ts'
+import type { NodePlatform } from '../../shared/platform.ts'
+import type { DependencyEntry, FileFilter, FileSet, FileVisitor } from '../../shared/types.ts'
+import { collectProductionDeps } from '../dependency/collect.ts'
+import { createTransformer } from '../fileset/transformer.ts'
+import { collectAppFileSet } from './app-collector.ts'
+import { collectNodeModuleFiles } from './nm-collector.ts'
+
+export interface CollectAllFilesOptions {
+  platform: NodePlatform
+  /** Directory under projectDir holding the built app (e.g. "out" for electron-vite). */
+  appDistDir: string
+  includePdb: boolean
+  disableDefaultIgnoredFiles: boolean
+  workspaceRoot?: string
+  extraMetadata?: Record<string, unknown>
+  nodeModuleFileFilter?: FileFilter
+  onNodeModuleFile?: FileVisitor
+}
+
+export async function collectAllFiles(
+  projectDir: string,
+  {
+    platform,
+    appDistDir,
+    includePdb,
+    disableDefaultIgnoredFiles,
+    workspaceRoot,
+    extraMetadata,
+    nodeModuleFileFilter,
+    onNodeModuleFile,
+  }: CollectAllFilesOptions,
+): Promise<{
+  appFileSet: FileSet
+  nmFileSets: FileSet[]
+  transformer: (file: string) => Promise<string | null> | string | null
+}> {
+  const distDir = path.join(projectDir, appDistDir)
+  if (!existsSync(distDir)) {
+    throw new Error(
+      `${appDistDir}/ directory not found. Build the app first, or set files.appDistDir in the config.`,
+    )
+  }
+
+  logger.info(`    Collecting app files (${appDistDir}/ + package.json) ...`)
+  const appFileSet = await collectAppFileSet(projectDir, distDir, includePdb)
+  logger.info(`    App files: ${appFileSet.files.length} entries`)
+
+  logger.info('    Collecting production node_modules ...')
+  const nmFileSets = await collectNodeModuleFileSets(projectDir, {
+    platform,
+    includePdb,
+    disableDefaultIgnoredFiles,
+    workspaceRoot,
+    filter: nodeModuleFileFilter,
+    onNodeModuleFile,
+  })
+  logger.info(`    node_modules fileSets: ${nmFileSets.length}`)
+
+  const transformer = createTransformer(projectDir, extraMetadata)
+
+  return { appFileSet, nmFileSets, transformer }
+}
+
+interface CollectNodeModuleFileSetsOptions {
+  platform: NodePlatform
+  includePdb: boolean
+  disableDefaultIgnoredFiles: boolean
+  workspaceRoot?: string
+  filter?: FileFilter
+  onNodeModuleFile?: FileVisitor
+}
+
+async function collectNodeModuleFileSets(
+  projectDir: string,
+  {
+    platform,
+    includePdb,
+    disableDefaultIgnoredFiles,
+    workspaceRoot,
+    filter,
+    onNodeModuleFile,
+  }: CollectNodeModuleFileSetsOptions,
+): Promise<FileSet[]> {
+  const nmDir = path.join(projectDir, 'node_modules')
+  if (!existsSync(nmDir)) {
+    return []
+  }
+
+  const productionPackages = await collectProductionDeps(projectDir, workspaceRoot)
+  logger.info(`    Production packages: ${productionPackages.length}`)
+
+  const result: FileSet[] = []
+
+  const collectModule = async (dep: DependencyEntry, parentDestination: string): Promise<void> => {
+    const pkgDir = dep.dir
+    if (!pkgDir || !existsSync(pkgDir)) return
+
+    const NODE_MODULES = 'node_modules'
+    const destination = path.join(parentDestination, NODE_MODULES, dep.name)
+
+    const { files: pkgFiles, metadata: pkgMeta } = await collectNodeModuleFiles(
+      pkgDir,
+      dep.name,
+      destination,
+      platform,
+      includePdb,
+      disableDefaultIgnoredFiles,
+      filter,
+      onNodeModuleFile,
+    )
+
+    result.push({
+      src: pkgDir,
+      destination,
+      files: pkgFiles,
+      metadata: pkgMeta,
+    })
+
+    if (dep.dependencies && dep.dependencies.length > 0) {
+      for (const child of dep.dependencies) {
+        await collectModule(child, destination)
+      }
+    }
+  }
+
+  for (const dep of productionPackages) {
+    await collectModule(dep, projectDir)
+  }
+
+  return result
+}

package/src/asar/packer/concurrency.ts

@@ -0,0 +1,24 @@
+export async function mapWithConcurrency<T, R>(
+  items: T[],
+  concurrency: number,
+  mapper: (item: T, index: number) => Promise<R>,
+): Promise<R[]> {
+  if (items.length === 0) {
+    return []
+  }
+
+  const limit = Math.max(1, Math.min(concurrency, items.length))
+  const results = new Array<R>(items.length)
+  let nextIndex = 0
+
+  async function worker(): Promise<void> {
+    while (nextIndex < items.length) {
+      const index = nextIndex
+      nextIndex += 1
+      results[index] = await mapper(items[index], index)
+    }
+  }
+
+  await Promise.all(Array.from({ length: limit }, () => worker()))
+  return results
+}

package/src/asar/packer/integrity.ts

@@ -0,0 +1,53 @@
+import crypto from 'node:crypto'
+import { createRequire } from 'node:module'
+import fsExtra from 'fs-extra'
+
+const require = createRequire(import.meta.url)
+const chromiumPickle = require('chromium-pickle-js')
+
+interface PickleIterator {
+  readUInt32(): number
+  readString(): string
+}
+
+interface Pickle {
+  createIterator(): PickleIterator
+}
+
+export async function readAsarHeader(archive: string): Promise<{ header: string; size: number }> {
+  const fd = await fsExtra.open(archive, 'r')
+  let size: number
+  let headerBuf: Buffer
+
+  try {
+    const sizeBuf = Buffer.allocUnsafe(8)
+    const { bytesRead: sizeBytesRead } = await fsExtra.read(fd, sizeBuf, 0, 8, null)
+    if (sizeBytesRead !== 8) {
+      throw new Error('Unable to read header size')
+    }
+
+    const sizePickle: Pickle = chromiumPickle.createFromBuffer(sizeBuf)
+    size = sizePickle.createIterator().readUInt32()
+
+    headerBuf = Buffer.allocUnsafe(size)
+    const { bytesRead: headerBytesRead } = await fsExtra.read(fd, headerBuf, 0, size, null)
+    if (headerBytesRead !== size) {
+      throw new Error('Unable to read header')
+    }
+  } finally {
+    await fsExtra.close(fd)
+  }
+
+  const headerPickle: Pickle = chromiumPickle.createFromBuffer(headerBuf)
+  return { header: headerPickle.createIterator().readString(), size }
+}
+
+export async function hashHeader(file: string): Promise<{ algorithm: string; hash: string }> {
+  const hash = crypto.createHash('sha256')
+  const { header } = await readAsarHeader(file)
+  hash.update(header)
+  return {
+    algorithm: 'SHA256',
+    hash: hash.digest('hex'),
+  }
+}