@drawcall/market 0.3.0 → 0.4.0

package/src/pack.ts

@@ -3,9 +3,17 @@ import * as fs from 'fs/promises'
 import * as path from 'path'
 import { unzipSync, zipSync } from 'fflate'
 import ignore from 'ignore'
+import semver from 'semver'
+import {
+  ALWAYS_IGNORED_DIRS,
+  GITIGNORE_FILENAME,
+  isGitignored,
+  findInstallRoot,
+  walkWithGitignore,
+  type GitignoreMatcher,
+} from './project-walk.js'
 import type { MarketClient } from './client.js'
 import type { AssetInstallMetadata } from './contract.js'
-import { findInstallRoot } from './install.js'
 import {
   packageJsonAssetDependencies,
   packageJsonAssetDependenciesFromFiles,
@@ -46,10 +54,10 @@ export interface PackAssetOptions {
   dependencies: ParsedPackDependencies
   policy?: PackPolicy
   /**
-   * Fetch the server's content hashes for a dependency's files (see `asset.fileManifest`), given
-   * the declared range. Pack matches local files against these hashes by content — independent of
-   * path, so renamed installed files are omitted too and recorded as aliases. Omit this (or a
-   * failing fetch) and the dependency's files are kept.
+   * Fetch `path → etag` (md5) for a dependency's files given the declared range (see
+   * `fileManifestForRange`). Pack matches local files against these hashes by content —
+   * independent of path, so renamed installed files are omitted too and recorded as aliases.
+   * Omit this (or a failing fetch) and the dependency's files are kept.
    */
   fetchFileManifest?: FetchFileManifest
 }
@@ -63,36 +71,85 @@ export interface PackedAsset {
   omittedUnchangedInstalledFiles: boolean
   /** Number of zip entries dropped because a `.gitignore` inside the zip excluded them. */
   gitignoredFiles: number
+  /** Dependencies whose file index could not be fetched: files kept, aliases untouched. */
+  skippedDependencies: string[]
+  /** Dependency files whose canonical bytes were found nowhere (locally modified or deleted). */
+  missingDependencyFiles: Array<{ name: string; path: string }>
 }
 
-export function sha256(bytes: Uint8Array): string {
-  return createHash('sha256').update(bytes).digest('hex')
+/** md5 hex — the same digest R2 serves as every file's etag, so local bytes match server hashes. */
+export function md5(bytes: Uint8Array): string {
+  return createHash('md5').update(bytes).digest('hex')
 }
 
 /**
- * A `FetchFileManifest` backed by the Market API. The manifest endpoint needs an exact version, so
- * an exact declared range is used directly and any other range resolves to the latest version —
- * without a local lockfile the installed version is not recorded, and content addressing keeps a
- * mismatch safe: a stale hash simply fails to match and the file is kept.
+ * Cap on how many in-range versions contribute hashes to one dependency's manifest. High enough
+ * that any real project's installed version is covered; bounds the API calls for assets with long
+ * histories (an out-of-cap version's files simply stay unmatched and ship — the fail-open
+ * direction).
+ */
+const MANIFEST_VERSION_CAP = 20
+
+/**
+ * A `FetchFileManifest` backed by the Market API: `path → etag` (md5) per dependency file. Without
+ * a local lockfile the installed version is unknown, so a non-exact range merges the file indexes
+ * of every published version satisfying it (newest wins per path) — a hash match identifies the
+ * file regardless of which version it came from. Content addressing keeps mistakes safe: a stale
+ * hash simply fails to match and the file is kept.
  */
 export function fileManifestForRange(asset: MarketClient['asset']): FetchFileManifest {
   return async (name, range) => {
     const exact = exactRangeVersion(range)
-    const meta = await asset.exact({
-      name,
-      ...(exact ? { version: exact } : {}),
-      includeUnapproved: true,
-    })
-    if (!meta) return {}
-    return asset.fileManifest({ name, version: meta.version ?? meta.latestVersion })
+    if (exact) return fileIndexHashes(asset, name, exact)
+
+    const { versions } = await asset.versions({ name })
+    const inRange = versions
+      .map((entry) => entry.version)
+      .filter((version) => semver.satisfies(version, range))
+      .sort(semver.compare)
+      .slice(-MANIFEST_VERSION_CAP)
+
+    // Oldest first, so a newer version's hash wins when a path recurs across versions.
+    const merged: Record<string, string> = {}
+    for (const hashes of await mapWithConcurrency(inRange, 4, (version) =>
+      fileIndexHashes(asset, name, version),
+    )) {
+      Object.assign(merged, hashes)
+    }
+    return merged
   }
 }
 
+async function fileIndexHashes(
+  asset: MarketClient['asset'],
+  name: string,
+  version: string,
+): Promise<Record<string, string>> {
+  const index = await asset.files({ name, version })
+  return Object.fromEntries(index.files.map((file) => [file.path, file.etag]))
+}
+
 function exactRangeVersion(range: string): string | null {
   const candidate = range.trim().replace(/^=/u, '')
   return semverSchema.safeParse(candidate).success ? candidate : null
 }
 
+async function mapWithConcurrency<T, R>(
+  items: readonly T[],
+  concurrency: number,
+  fn: (item: T) => Promise<R>,
+): Promise<R[]> {
+  const results = new Array<R>(items.length)
+  let next = 0
+  async function worker(): Promise<void> {
+    for (let index = next++; index < items.length; index = next++) {
+      results[index] = await fn(items[index])
+    }
+  }
+  await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker()))
+  return results
+}
+
 export async function packAsset(zipFilter: string, opts: PackAssetOptions): Promise<PackedAsset> {
   const cwd = opts.cwd ?? process.cwd()
 
@@ -131,6 +188,8 @@ export async function packAsset(zipFilter: string, opts: PackAssetOptions): Prom
   const { kept: withoutIgnored, gitignored } = applyGitignore(sourceFiles)
   let files = withoutIgnored
   let finalAssetDependencies = assetDependencies
+  let skippedDependencies: string[] = []
+  let missingDependencyFiles: Array<{ name: string; path: string }> = []
   if (policy.omitUnchangedInstalledFiles) {
     const omitted = await withoutInstalledDependencyFiles(
       withoutIgnored,
@@ -139,6 +198,8 @@ export async function packAsset(zipFilter: string, opts: PackAssetOptions): Prom
     )
     files = omitted.kept
     finalAssetDependencies = omitted.assetDependencies
+    skippedDependencies = omitted.skippedDependencies
+    missingDependencyFiles = omitted.missingDependencyFiles
   }
 
   const originalCount = Object.keys(sourceFiles).length
@@ -153,6 +214,8 @@ export async function packAsset(zipFilter: string, opts: PackAssetOptions): Prom
     skillDependencies: opts.dependencies.skillDependencies,
     omittedUnchangedInstalledFiles: finalCount < afterIgnoreCount,
     gitignoredFiles: gitignored,
+    skippedDependencies,
+    missingDependencyFiles,
   }
 }
 
@@ -207,7 +270,7 @@ async function packDirectory(dir: string, opts: PackAssetOptions): Promise<Packe
       files[relative] = content
       continue
     }
-    const hash = sha256(content)
+    const hash = md5(content)
     hashesByPath.set(relative, hash)
     if (dependencyHashes.has(hash)) {
       deferred.push(relative)
@@ -240,57 +303,16 @@ async function packDirectory(dir: string, opts: PackAssetOptions): Promise<Packe
     skillDependencies: opts.dependencies.skillDependencies,
     omittedUnchangedInstalledFiles: layout.omit.size > 0,
     gitignoredFiles: walk.ignored,
+    skippedDependencies: policy.omitUnchangedInstalledFiles
+      ? Object.keys(assetDependencies).filter((name) => !index.fetched.has(name))
+      : [],
+    missingDependencyFiles: layout.missing.map((file) => ({
+      name: file.name,
+      path: file.canonicalPath,
+    })),
   }
 }
 
-// Walk a directory applying git semantics as we descend: each directory's `.gitignore` governs its
-// own subtree, an ignored directory is pruned without entering it, and `.git`/`node_modules` never
-// ship. Returns kept files as posix paths relative to the root, plus the count of ignored entries.
-async function walkWithGitignore(root: string): Promise<{ kept: string[]; ignored: number }> {
-  const kept: string[] = []
-  let ignored = 0
-
-  async function descend(dir: string, matchers: GitignoreMatcher[]): Promise<void> {
-    const scoped = [...matchers]
-    const gitignore = await maybeStat(path.join(dir, GITIGNORE_FILENAME))
-    if (gitignore?.isFile()) {
-      const prefix = toPosix(path.relative(root, dir))
-      scoped.push({
-        dir: prefix ? `${prefix}/` : '',
-        filter: ignore().add(await fs.readFile(path.join(dir, GITIGNORE_FILENAME), 'utf8')),
-      })
-    }
-
-    const entries = await fs.readdir(dir, { withFileTypes: true })
-    for (const entry of entries) {
-      if (ALWAYS_IGNORED_DIRS.has(entry.name)) continue
-      const relative = toPosix(path.relative(root, path.join(dir, entry.name)))
-      const candidate = entry.isDirectory() ? `${relative}/` : relative
-      if (isGitignored(candidate, scoped)) {
-        ignored += 1
-        continue
-      }
-      if (entry.isDirectory()) await descend(path.join(dir, entry.name), scoped)
-      else if (entry.isFile()) kept.push(relative)
-    }
-  }
-
-  await descend(root, [])
-  return { kept, ignored }
-}
-
-function toPosix(p: string): string {
-  return p.split(path.sep).join('/')
-}
-
-const GITIGNORE_FILENAME = '.gitignore'
-
-// Directories that never belong in a published asset — VCS/tooling metadata, reinstallable
-// dependencies, and the market's own state dir (`.drawcall`, holding the install mutex). Excluded
-// structurally in every pack path, regardless of any `.gitignore`, so a re-uploaded asset never
-// ships them.
-const ALWAYS_IGNORED_DIRS = new Set(['.git', 'node_modules', '.drawcall'])
-
 function hasAlwaysIgnoredSegment(posixPath: string): boolean {
   return posixPath.split('/').some((segment) => ALWAYS_IGNORED_DIRS.has(segment))
 }
@@ -318,12 +340,6 @@ function applyGitignore(files: Record<string, Uint8Array>): {
   return { kept, gitignored }
 }
 
-interface GitignoreMatcher {
-  /** Directory the `.gitignore` governs: '' for the zip root, or e.g. 'sub/'. */
-  dir: string
-  filter: ReturnType<typeof ignore>
-}
-
 function gitignoreMatchers(files: Record<string, Uint8Array>): GitignoreMatcher[] {
   const matchers: GitignoreMatcher[] = []
   for (const [name, content] of Object.entries(files)) {
@@ -334,16 +350,6 @@ function gitignoreMatchers(files: Record<string, Uint8Array>): GitignoreMatcher[
   return matchers
 }
 
-function isGitignored(name: string, matchers: GitignoreMatcher[]): boolean {
-  // fflate keeps directory entries with a trailing slash; the `ignore` lib matches directory patterns
-  // (`dist/`) against exactly that form, so the path is passed through unchanged.
-  return matchers.some((matcher) => {
-    if (!name.startsWith(matcher.dir)) return false
-    const relative = name.slice(matcher.dir.length)
-    return relative.length > 0 && matcher.filter.ignores(relative)
-  })
-}
-
 export function parsePackDependencies(specs: PackDependencySpecs): ParsedPackDependencies {
   return {
     npmDependencies: parseVersionedDeps(specs.npm ?? [], 'npm'),
@@ -451,16 +457,26 @@ async function withoutInstalledDependencyFiles(
   files: Record<string, Uint8Array>,
   assetDependencies: AssetDependencies,
   fetchFileManifest: FetchFileManifest | undefined,
-): Promise<{ kept: Record<string, Uint8Array>; assetDependencies: AssetDependencies }> {
+): Promise<{
+  kept: Record<string, Uint8Array>
+  assetDependencies: AssetDependencies
+  skippedDependencies: string[]
+  missingDependencyFiles: Array<{ name: string; path: string }>
+}> {
   const index = await dependencyFileIndex(assetDependencies, fetchFileManifest)
-  if (index.files.length === 0) return { kept: files, assetDependencies }
+  const skippedDependencies = Object.keys(assetDependencies).filter(
+    (name) => !index.fetched.has(name),
+  )
+  if (index.files.length === 0) {
+    return { kept: files, assetDependencies, skippedDependencies, missingDependencyFiles: [] }
+  }
 
   const hashesByPath = new Map<string, string>()
   const entryByNormalized = new Map<string, string>()
   for (const [file, content] of Object.entries(files)) {
     const normalizedPath = normalizedZipPath(file)
     if (!normalizedPath || normalizedPath.endsWith('/')) continue
-    hashesByPath.set(normalizedPath, sha256(content))
+    hashesByPath.set(normalizedPath, md5(content))
     entryByNormalized.set(normalizedPath, file)
   }
 
@@ -471,20 +487,28 @@ async function withoutInstalledDependencyFiles(
   const kept = Object.fromEntries(
     Object.entries(files).filter(([file]) => !omittedEntries.has(file)),
   )
-  return { kept, assetDependencies: layout.assetDependencies }
+  return {
+    kept,
+    assetDependencies: layout.assetDependencies,
+    skippedDependencies,
+    missingDependencyFiles: layout.missing.map((file) => ({
+      name: file.name,
+      path: file.canonicalPath,
+    })),
+  }
 }
 
-interface DependencyFile {
+export interface DependencyFile {
   name: string
   canonicalPath: string
 }
 
-interface DependencyFileEntry extends DependencyFile {
-  /** sha256 hex of the file's canonical content, from the server manifest. */
+export interface DependencyFileEntry extends DependencyFile {
+  /** md5 hex of the file's canonical content — its R2 etag, from the file index. */
   hash: string
 }
 
-interface DependencyFileIndex {
+export interface DependencyFileIndex {
   files: DependencyFileEntry[]
   /** Dependencies whose manifest was fetched — only their aliases are re-derived. */
   fetched: Set<string>
@@ -496,7 +520,7 @@ function emptyDependencyFileIndex(): DependencyFileIndex {
 
 /**
  * The canonical content hash of every file belonging to a declared asset dependency, fetched from
- * the server (`asset.fileManifest`). Returns an empty index when there's nothing to omit or no way
+ * the server (the version file indexes' etags). Returns an empty index when there's nothing to omit or no way
  * to fetch (offline / no client) — pack then keeps all files rather than guessing.
  */
 async function dependencyFileIndex(
@@ -547,7 +571,7 @@ interface DependencyLayout {
  * user's own content. Stale aliases drop: a renamed file that was since modified ships as user
  * content and reinstall restores the canonical path.
  */
-function resolveDependencyLayout(
+export function resolveDependencyLayout(
   declared: AssetDependencies,
   index: DependencyFileIndex,
   hashesByPath: Map<string, string>,
@@ -632,7 +656,7 @@ export async function syncAssetDependencies(
   for (const relative of walk.kept) {
     hashesByPath.set(
       relative,
-      sha256(new Uint8Array(await fs.readFile(path.join(projectRoot, relative)))),
+      md5(new Uint8Array(await fs.readFile(path.join(projectRoot, relative)))),
     )
   }
 

package/src/project-walk.ts

@@ -0,0 +1,95 @@
+/**
+ * Directory walking with git semantics, shared by pack/sync (dependency-layout resolution) and
+ * install (rename adoption): each directory's `.gitignore` governs its own subtree, an ignored
+ * directory is pruned without entering it, and `.git`/`node_modules`/`.drawcall` never count.
+ */
+import * as fs from 'fs/promises'
+import * as path from 'path'
+import ignore from 'ignore'
+
+export const GITIGNORE_FILENAME = '.gitignore'
+
+// Directories that never belong in a published asset — VCS/tooling metadata, reinstallable
+// dependencies, and the market's own state dir (`.drawcall`, holding the install mutex). Excluded
+// structurally in every walk, regardless of any `.gitignore`.
+export const ALWAYS_IGNORED_DIRS = new Set(['.git', 'node_modules', '.drawcall'])
+
+export interface GitignoreMatcher {
+  /** Directory the `.gitignore` governs: '' for the root, or e.g. 'sub/'. */
+  dir: string
+  filter: ReturnType<typeof ignore>
+}
+
+export function isGitignored(name: string, matchers: GitignoreMatcher[]): boolean {
+  // Directory candidates carry a trailing slash; the `ignore` lib matches directory patterns
+  // (`dist/`) against exactly that form, so the path is passed through unchanged.
+  return matchers.some((matcher) => {
+    if (!name.startsWith(matcher.dir)) return false
+    const relative = name.slice(matcher.dir.length)
+    return relative.length > 0 && matcher.filter.ignores(relative)
+  })
+}
+
+/** Walk a directory tree; returns kept files as posix paths relative to the root, plus the count
+ *  of ignored entries. */
+export async function walkWithGitignore(
+  root: string,
+): Promise<{ kept: string[]; ignored: number }> {
+  const kept: string[] = []
+  let ignored = 0
+
+  async function descend(dir: string, matchers: GitignoreMatcher[]): Promise<void> {
+    const scoped = [...matchers]
+    const gitignore = await maybeStat(path.join(dir, GITIGNORE_FILENAME))
+    if (gitignore?.isFile()) {
+      const prefix = toPosix(path.relative(root, dir))
+      scoped.push({
+        dir: prefix ? `${prefix}/` : '',
+        filter: ignore().add(await fs.readFile(path.join(dir, GITIGNORE_FILENAME), 'utf8')),
+      })
+    }
+
+    const entries = await fs.readdir(dir, { withFileTypes: true })
+    for (const entry of entries) {
+      if (ALWAYS_IGNORED_DIRS.has(entry.name)) continue
+      const relative = toPosix(path.relative(root, path.join(dir, entry.name)))
+      const candidate = entry.isDirectory() ? `${relative}/` : relative
+      if (isGitignored(candidate, scoped)) {
+        ignored += 1
+        continue
+      }
+      if (entry.isDirectory()) await descend(path.join(dir, entry.name), scoped)
+      else if (entry.isFile()) kept.push(relative)
+    }
+  }
+
+  await descend(root, [])
+  return { kept, ignored }
+}
+
+/** The nearest ancestor (including start) holding a package.json — the project root. */
+export async function findInstallRoot(cwd: string = process.cwd()): Promise<string> {
+  const start = path.resolve(cwd)
+
+  for (let dir = start; ; dir = path.dirname(dir)) {
+    if ((await maybeStat(path.join(dir, 'package.json')))?.isFile()) {
+      return dir
+    }
+    if (path.dirname(dir) === dir) {
+      return start
+    }
+  }
+}
+
+function toPosix(p: string): string {
+  return p.split(path.sep).join('/')
+}
+
+async function maybeStat(file: string) {
+  try {
+    return await fs.stat(file)
+  } catch (error) {
+    if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return null
+    throw error
+  }
+}

package/src/schemas.ts

@@ -133,17 +133,20 @@ export const assetFilesSchema = z.object({
   key: accessKeySchema.optional(),
 })
 
-export const fileManifestSchema = z.object({
+// Input of `asset.versions`: every published version of an asset (same credential rules as reads).
+export const assetVersionsSchema = z.object({
   name: assetNameSchema,
-  version: semverSchema,
+  key: accessKeySchema.optional(),
 })
 
-/**
- * The content hash of every file in an asset version, keyed by its install path — sha256 hex, the
- * same digest the CLI computes locally. Lets a caller check whether a local file is identical to the
- * canonical asset without keeping hashes itself (see `pack` omit-unchanged).
- */
-export type AssetFileManifest = Record<string, string>
+export interface AssetVersionListing {
+  version: string
+  approved: boolean
+}
+
+export interface AssetVersionsResult {
+  versions: AssetVersionListing[]
+}
 
 // A phone-photo data URI is a few MB; this bounds worker and model payloads while staying far
 // above what providers keep (images are downscaled upstream).

package/src/v1/contract.ts

@@ -1,6 +1,6 @@
 import { oc } from '@orpc/contract'
 import { z } from 'zod'
-import type { AssetFileManifest } from '../schemas.js'
+import type { AssetVersionsResult } from '../schemas.js'
 import type {
   AssetExactResult,
   AssetFileEntry,
@@ -16,8 +16,8 @@ import {
   agentStartSchema,
   agentStatusSchema,
   assetFilesSchema,
+  assetVersionsSchema,
   exactAssetSchema,
-  fileManifestSchema,
   generateAssetSchema,
   generateJobStatusSchema,
   generateResponseSchema,
@@ -76,10 +76,12 @@ export const contract = {
       .input(assetFilesSchema)
       .output(z.custom<{ files: AssetFileEntry[] }>()),
 
-    fileManifest: oc
-      .route({ method: 'GET', path: '/assets/{name}/{version}/file-manifest' })
-      .input(fileManifestSchema)
-      .output(z.custom<AssetFileManifest>()),
+    // Every published version of an asset. Feeds multi-version content matching: without a local
+    // lockfile the installed version is unknown, so clients merge hashes across the range.
+    versions: oc
+      .route({ method: 'GET', path: '/assets/{name}/versions' })
+      .input(assetVersionsSchema)
+      .output(z.custom<AssetVersionsResult>()),
 
     uploadZip: oc
       .route({ method: 'PUT', path: '/assets/{name}/{version}' })