@drawcall/market 0.3.0 → 0.5.0

package/src/install.ts

@@ -11,16 +11,23 @@
  */
 
 import { execFile } from 'child_process'
-import { createHash } from 'crypto'
 import * as fs from 'fs/promises'
 import * as path from 'path'
 import { detectPackageManager, installDependencies } from 'nypm'
 import type { MarketClient } from './client.js'
 import type { AssetFilesResult, AssetInstallMetadata } from './contract.js'
 import { withProjectInstallLock } from './install-lock.js'
+import { md5, resolveDependencyLayout } from './pack.js'
+import { findInstallRoot, walkWithGitignore } from './project-walk.js'
 import { packageJsonAssetDependencies, parsePackageJson, type PackageJson } from './package-json.js'
 import type { ResolveResult } from './resolve.js'
-import { assetDependencyAlias, assetDependencyRange, assetDependencyValue } from './schemas.js'
+export { findInstallRoot }
+import {
+  assetDependencyAlias,
+  assetDependencyRange,
+  assetDependencyValue,
+  type AssetDependencyAlias,
+} from './schemas.js'
 
 export interface InstallOptions {
   /** Directory to start project root discovery from (default: cwd) */
@@ -72,14 +79,25 @@ interface DownloadResult {
   assets: InstalledAsset[]
   wrotePackageJson: boolean
   warnings: string[]
+  /** Renames adopted during this install (per asset: canonical path → adopted project path). */
+  adoptedAliases: Record<string, Record<string, string>>
 }
 
 interface DownloadOptions {
   force: boolean
   log: (message: string) => void
   key?: string
-  /** File aliases per asset name: canonical install path → the project's current path for it. */
-  aliases: Record<string, Record<string, string>>
+  /** File aliases per asset name: canonical install path → the project's current path for it,
+   *  or `false` for a tombstone (the file is intentionally not installed). */
+  aliases: Record<string, AssetDependencyAlias>
+  /** md5 of every project file, computed at most once per install (rename adoption). */
+  projectHashes: () => Promise<Map<string, string>>
+  /**
+   * Etags across an asset's published versions — the "recognized bytes" set for content-aware
+   * updates. `null` when the server cannot answer: fail-open in the conservative direction
+   * (never overwrite on uncertainty).
+   */
+  knownHashes: (name: string) => Promise<Set<string> | null>
 }
 
 export async function install(
@@ -97,6 +115,8 @@ export async function install(
     log,
     key: opts.key,
     aliases,
+    projectHashes: memoizedProjectHashes(installRoot),
+    knownHashes: memoizedKnownHashes(client),
   })
   const integration = await withProjectInstallLock(
     installRoot,
@@ -105,6 +125,7 @@ export async function install(
         rootRequests: opts.rootRequests ?? [],
         installMetadata: metadata,
         packageManagerNeeded: download.wrotePackageJson,
+        adoptedAliases: download.adoptedAliases,
       })
       if (packageJsonUpdate.packageManagerNeeded) {
         await (opts.runPackageManagerInstall ?? runPackageManagerInstall)(installRoot, log)
@@ -137,19 +158,6 @@ export async function install(
   }
 }
 
-export async function findInstallRoot(cwd: string = process.cwd()): Promise<string> {
-  const start = path.resolve(cwd)
-
-  for (let dir = start; ; dir = path.dirname(dir)) {
-    if (await isFile(path.join(dir, 'package.json'))) {
-      return dir
-    }
-    if (path.dirname(dir) === dir) {
-      return start
-    }
-  }
-}
-
 /** Max asset downloads in flight at once. Assets write disjoint file trees, so downloading them
  *  concurrently is safe and turns a ~40-asset install from serial round-trips into a few batched
  *  waves. Kept modest so one install does not itself overwhelm the worker's per-request memory. */
@@ -175,6 +183,11 @@ async function downloadAssets(
     // the package manager to bring its npm deps.
     wrotePackageJson: perAsset.some((r) => r.wrotePackageJson),
     warnings: perAsset.flatMap((r) => r.warnings),
+    adoptedAliases: Object.fromEntries(
+      perAsset
+        .filter((r) => Object.keys(r.adopted).length > 0)
+        .map((r) => [r.asset.name, r.adopted]),
+    ),
   }
 }
 
@@ -183,53 +196,81 @@ async function downloadOneAsset(
   asset: ResolveResult['assets'][number],
   projectRoot: string,
   opts: DownloadOptions,
-): Promise<{ asset: InstalledAsset; wrotePackageJson: boolean; warnings: string[] }> {
+): Promise<{
+  asset: InstalledAsset
+  wrotePackageJson: boolean
+  warnings: string[]
+  adopted: Record<string, string>
+}> {
   opts.log(`Downloading ${asset.name}@${asset.version}...`)
 
   // Only the network fetches are retried — downloads are the sole transient step; file writes are
   // deterministic local work and a failure there is a real error, not worth re-attempting.
-  const files = await fetchAssetEntries(client, asset, opts)
-  const alias = opts.aliases[asset.name] ?? {}
-  const installedFiles: string[] = []
-  const warnings: string[] = []
-  let wrotePackageJson = false
-
-  for (const [relativePath, content] of files) {
+  const entries = await fetchAssetEntries(client, asset, opts)
+  const planned = entries.flatMap((entry) => {
     const normalizedPath = safeRelativePath(
-      relativePath,
-      () => `Zip contains an unsafe path: ${relativePath}`,
+      entry.path,
+      () => `Zip contains an unsafe path: ${entry.path}`,
     )
     if (
       normalizedPath === '.' ||
       (normalizedPath === 'README.md' && asset.type !== 'template') ||
       normalizedPath.endsWith('/')
     ) {
-      continue
+      return []
     }
+    return [{ ...entry, normalizedPath }]
+  })
+
+  // Alias targets come from metadata; reject traversal before any path is stat'd or resolved —
+  // adoption must never quietly route around unsafe metadata.
+  const declaredAlias = opts.aliases[asset.name] ?? {}
+  for (const [canonical, target] of Object.entries(declaredAlias)) {
+    if (target === false) continue
+    safeRelativePath(
+      target,
+      () => `Alias for ${canonical} in ${asset.name} is an unsafe path: ${target}`,
+    )
+  }
+  const adopted = await adoptRenamedFiles(planned, asset.name, declaredAlias, projectRoot, opts)
+  const alias = { ...declaredAlias, ...adopted }
+
+  const installedFiles: string[] = []
+  const warnings: string[] = []
+  let wrotePackageJson = false
 
-    // An alias redirects the file to where this project keeps it (a rename recorded by pack or a
-    // dependent asset). Alias targets come from metadata, so they pass the same traversal guard.
-    const aliased = alias[normalizedPath]
+  for (const entry of planned) {
+    // An alias redirects the file to where this project keeps it (a rename recorded by pack, a
+    // dependent asset, or adoption above); `false` is a tombstone — a deliberate deletion that
+    // must not resurrect on reinstall. Alias targets pass the same traversal guard.
+    const aliased = alias[entry.normalizedPath]
+    if (aliased === false) continue
     const targetPath =
       aliased === undefined
-        ? normalizedPath
+        ? entry.normalizedPath
         : safeRelativePath(
             aliased,
-            () => `Alias for ${normalizedPath} in ${asset.name} is an unsafe path: ${aliased}`,
+            () =>
+              `Alias for ${entry.normalizedPath} in ${asset.name} is an unsafe path: ${aliased}`,
           )
 
     const filePath = path.join(projectRoot, targetPath)
     const existing = await maybeReadFile(filePath)
-    if (existing && !bytesEqual(existing, content) && !opts.force) {
-      warnings.push(
-        `Skipped ${targetPath} from ${asset.name}@${asset.version}; file already exists. Re-run with --force to overwrite.`,
-      )
-      continue
+    if (existing && !bytesEqual(existing, entry.bytes)) {
+      // Recognized bytes — a published version of this same dependency file — are replaceable
+      // without --force; unrecognized bytes are the user's and keep the skip-and-warn behavior.
+      const known = opts.force ? null : await opts.knownHashes(asset.name)
+      if (!opts.force && !known?.has(md5(existing))) {
+        warnings.push(
+          `Skipped ${targetPath} from ${asset.name}@${asset.version}; file already exists. Re-run with --force to overwrite.`,
+        )
+        continue
+      }
     }
 
-    if (!existing || !bytesEqual(existing, content)) {
+    if (!existing || !bytesEqual(existing, entry.bytes)) {
       await fs.mkdir(path.dirname(filePath), { recursive: true })
-      await fs.writeFile(filePath, content)
+      await fs.writeFile(filePath, entry.bytes)
       if (targetPath === 'package.json') {
         wrotePackageJson = true
       }
@@ -249,6 +290,108 @@ async function downloadOneAsset(
     },
     wrotePackageJson,
     warnings,
+    adopted,
+  }
+}
+
+interface PlannedEntry {
+  normalizedPath: string
+  etag: string
+  bytes: Uint8Array
+}
+
+/**
+ * Install-time adoption of unsynced renames: when a file's write target is absent but identical
+ * bytes already live elsewhere in the project, that location is used instead of creating a
+ * duplicate — an implicit `sync` before writing. Uses the same layout resolution as pack/sync
+ * (recorded alias → canonical → first match), so the selection rule stays identical everywhere.
+ * The project is only hashed when some target is actually absent; a reinstall with every file in
+ * place never scans.
+ */
+async function adoptRenamedFiles(
+  planned: PlannedEntry[],
+  name: string,
+  declaredAlias: AssetDependencyAlias,
+  projectRoot: string,
+  opts: DownloadOptions,
+): Promise<Record<string, string>> {
+  const installable = planned.filter((entry) => declaredAlias[entry.normalizedPath] !== false)
+  const provisionalTargets = installable.map((entry) => {
+    const target = declaredAlias[entry.normalizedPath]
+    return typeof target === 'string' ? target : entry.normalizedPath
+  })
+  if (!(await anyPathMissing(projectRoot, provisionalTargets))) return {}
+
+  const layout = resolveDependencyLayout(
+    { [name]: assetDependencyValue('*', declaredAlias) },
+    {
+      files: installable.map((entry) => ({
+        name,
+        canonicalPath: entry.normalizedPath,
+        hash: entry.etag,
+      })),
+      fetched: new Set([name]),
+    },
+    await opts.projectHashes(),
+  )
+  const resolved = assetDependencyAlias(layout.assetDependencies[name])
+  return Object.fromEntries(
+    Object.entries(resolved).filter(
+      (pair): pair is [string, string] =>
+        typeof pair[1] === 'string' && declaredAlias[pair[0]] !== pair[1],
+    ),
+  )
+}
+
+async function anyPathMissing(projectRoot: string, targets: string[]): Promise<boolean> {
+  for (const target of targets) {
+    if (!(await isFile(path.join(projectRoot, target)))) return true
+  }
+  return false
+}
+
+/** md5 of every project file (gitignored trees excluded), computed at most once per install. */
+function memoizedProjectHashes(projectRoot: string): () => Promise<Map<string, string>> {
+  let cached: Promise<Map<string, string>> | undefined
+  return () =>
+    (cached ??= (async () => {
+      const walk = await walkWithGitignore(projectRoot)
+      const hashes = new Map<string, string>()
+      for (const relative of walk.kept) {
+        hashes.set(
+          relative,
+          md5(new Uint8Array(await fs.readFile(path.join(projectRoot, relative)))),
+        )
+      }
+      return hashes
+    })())
+}
+
+/** Newest published versions contributing to the recognized-bytes set (same bound as pack). */
+const KNOWN_HASH_VERSION_CAP = 20
+
+function memoizedKnownHashes(client: MarketClient): (name: string) => Promise<Set<string> | null> {
+  const cache = new Map<string, Promise<Set<string> | null>>()
+  return (name) => {
+    const hit = cache.get(name)
+    if (hit) return hit
+    const result = (async () => {
+      try {
+        const { versions } = await client.asset.versions({ name })
+        const recent = versions.map((entry) => entry.version).slice(-KNOWN_HASH_VERSION_CAP)
+        const etags = await Promise.all(
+          recent.map(async (version) =>
+            (await client.asset.files({ name, version })).files.map((file) => file.etag),
+          ),
+        )
+        return new Set(etags.flat())
+      } catch {
+        // Offline or not visible — the conservative fail-open: the caller keeps skip-and-warn.
+        return null
+      }
+    })()
+    cache.set(name, result)
+    return result
   }
 }
 
@@ -273,8 +416,8 @@ function safeRelativePath(candidate: string, message: () => string): string {
 async function projectAssetAliases(
   projectRoot: string,
   resolution: ResolveResult,
-): Promise<Record<string, Record<string, string>>> {
-  const merged: Record<string, Record<string, string>> = {}
+): Promise<Record<string, AssetDependencyAlias>> {
+  const merged: Record<string, AssetDependencyAlias> = {}
   for (const [name, alias] of Object.entries(resolution.assetAliases ?? {})) {
     merged[name] = { ...alias }
   }
@@ -304,7 +447,7 @@ async function fetchAssetEntries(
   client: MarketClient,
   asset: ResolveResult['assets'][number],
   opts: DownloadOptions,
-): Promise<Array<[string, Uint8Array]>> {
+): Promise<Array<{ path: string; etag: string; bytes: Uint8Array }>> {
   const index = await client.asset.files({
     name: asset.name,
     version: asset.version,
@@ -317,7 +460,7 @@ async function fetchAssetEntries(
       attempts: DOWNLOAD_ATTEMPTS,
       onRetry: (msg) => opts.log(`Downloading ${label} ${file.path}: ${msg}`),
     })
-    return [file.path, bytes] as [string, Uint8Array]
+    return { path: file.path, etag: file.etag, bytes }
   })
 }
 
@@ -329,9 +472,9 @@ async function fetchVerifiedFile(file: AssetFilesResult['files'][number]): Promi
   const bytes = new Uint8Array(await res.arrayBuffer())
   // A digest mismatch is a corrupt transfer; thrown without a status so withRetry treats it as
   // transient and re-fetches.
-  const md5 = createHash('md5').update(bytes).digest('hex')
-  if (md5 !== file.etag) {
-    throw new Error(`md5 mismatch for ${file.path}: expected ${file.etag}, got ${md5}`)
+  const digest = md5(bytes)
+  if (digest !== file.etag) {
+    throw new Error(`md5 mismatch for ${file.path}: expected ${file.etag}, got ${digest}`)
   }
   return bytes
 }
@@ -424,6 +567,7 @@ async function updatePackageJson(
     rootRequests: InstallRootRequest[]
     installMetadata: Record<string, AssetInstallMetadata>
     packageManagerNeeded: boolean
+    adoptedAliases: Record<string, Record<string, string>>
   },
 ): Promise<{ packageManagerNeeded: boolean }> {
   const pkgPath = path.join(projectRoot, 'package.json')
@@ -448,6 +592,9 @@ async function updatePackageJson(
   if (mergeAssetDependencyRanges(pkg, assetDependencies)) {
     changed = true
   }
+  if (mergeAdoptedAliases(pkg, opts.adoptedAliases)) {
+    changed = true
+  }
 
   if (changed) {
     await fs.writeFile(pkgPath, JSON.stringify(pkg, null, 2) + '\n')
@@ -456,6 +603,30 @@ async function updatePackageJson(
   return { packageManagerNeeded }
 }
 
+/** Record renames adopted during this install on the entries package.json already declares —
+ *  adoption is thereby persisted, not just applied. Undeclared (transitive) deps stay out. */
+function mergeAdoptedAliases(
+  pkg: PackageJson,
+  adopted: Record<string, Record<string, string>>,
+): boolean {
+  const current = pkg.assetDependencies
+  if (!current) return false
+
+  let changed = false
+  for (const [name, alias] of Object.entries(adopted)) {
+    const existing = current[name]
+    if (existing === undefined) continue
+    const next = assetDependencyValue(assetDependencyRange(existing), {
+      ...assetDependencyAlias(existing),
+      ...alias,
+    })
+    if (JSON.stringify(existing) === JSON.stringify(next)) continue
+    current[name] = next
+    changed = true
+  }
+  return changed
+}
+
 async function runPackageManagerInstall(projectRoot: string, log: (msg: string) => void) {
   log('Installing npm dependencies...')
   const pm = await detectPackageManager(projectRoot)

package/src/output.ts

@@ -2,7 +2,7 @@ import type { AssetInstallMetadata, AssetSearchResult } from './contract.js'
 import type { ListInstalledAssetsResult } from './commands/list.js'
 import type { InstallResult } from './install.js'
 import type { AssetDependencySync, PackedAsset } from './pack.js'
-import { assetDependencyAlias, assetDependencyRange } from './schemas.js'
+import { assetDependencyAlias, assetDependencyRange, type AssetDependencyValue } from './schemas.js'
 
 export function assetVersionRef(name: string, version?: string): string {
   return version ? `${name}@${version}` : name
@@ -90,7 +90,7 @@ export function listResult(result: ListInstalledAssetsResult): string {
   for (const asset of result.assets) {
     lines.push(`- ${assetVersionRef(asset.name, asset.range)}`)
     const aliases = Object.entries(asset.alias).sort(([a], [b]) => a.localeCompare(b))
-    lines.push(...aliases.map(([canonical, current]) => `    ${canonical} -> ${current}`))
+    lines.push(...aliases.map(([canonical, current]) => `    ${aliasLine(canonical, current)}`))
   }
 
   return lines.join('\n')
@@ -198,7 +198,7 @@ export function syncResult(result: AssetDependencySync): string {
     const aliases = Object.entries(assetDependencyAlias(value)).sort(([a], [b]) =>
       a.localeCompare(b),
     )
-    lines.push(...aliases.map(([canonical, current]) => `    ${canonical} -> ${current}`))
+    lines.push(...aliases.map(([canonical, current]) => `    ${aliasLine(canonical, current)}`))
   }
 
   if (result.missing.length > 0) {
@@ -219,12 +219,40 @@ export function packResult(out: string, packed: PackedAsset): string {
   if (packed.omittedUnchangedInstalledFiles) {
     lines.push('Omitted unchanged installed dependency files.')
   }
+  lines.push(...packWarnings(packed))
   lines.push(...dependencyLines('npm dependencies', packed.npmDependencies))
   lines.push(...dependencyLines('asset dependencies', packed.assetDependencies))
   lines.push(...dependencyLines('skill dependencies', packed.skillDependencies))
   return lines.join('\n')
 }
 
+/**
+ * What pack could not check — the silent-fail-open cases with real consequences: an unfetchable
+ * dependency ships its files inside the zip, and a missing dependency file means a local edit or
+ * deletion the packer should know about.
+ */
+export function packWarnings(packed: PackedAsset): string[] {
+  const lines: string[] = []
+  if (packed.skippedDependencies.length > 0) {
+    lines.push(
+      `Dependency file indexes unavailable (their files ship in the zip): ${[...packed.skippedDependencies].sort().join(', ')}`,
+    )
+  }
+  if (packed.missingDependencyFiles.length > 0) {
+    lines.push('Dependency files not found (locally modified or deleted):')
+    lines.push(
+      ...packed.missingDependencyFiles.map((file) => `- ${file.name}: ${file.path}`).sort(),
+    )
+  }
+  return lines
+}
+
+function aliasLine(canonical: string, current: string | false): string {
+  return current === false
+    ? `${canonical} -> (tombstoned: not installed)`
+    : `${canonical} -> ${current}`
+}
+
 export function previewResult(name: string, version: string, out: string): string {
   return `Saved preview for ${assetVersionRef(name, version)}: ${out}`
 }
@@ -254,10 +282,7 @@ function unique<T>(values: T[]): T[] {
   return [...new Set(values)]
 }
 
-function dependencyLines(
-  label: string,
-  deps: Record<string, string | { version: string; alias?: Record<string, string> }>,
-): string[] {
+function dependencyLines(label: string, deps: Record<string, AssetDependencyValue>): string[] {
   const entries = Object.entries(deps)
   if (entries.length === 0) return []
   return [
@@ -268,9 +293,7 @@ function dependencyLines(
   ]
 }
 
-function dependencyValueLabel(
-  value: string | { version: string; alias?: Record<string, string> },
-): string {
+function dependencyValueLabel(value: AssetDependencyValue): string {
   if (typeof value === 'string') return value
   const aliased = Object.keys(value.alias ?? {}).length
   return aliased > 0 ? `${value.version} (${aliased} aliased file(s))` : value.version