@drawcall/market 0.1.52 → 0.1.54

package/src/pack.ts

@@ -28,10 +28,18 @@ export interface PackPolicy {
   omitUnchangedInstalledFiles: boolean
 }
 
+export type FetchFileManifest = (name: string, version: string) => Promise<Record<string, string>>
+
 export interface PackAssetOptions {
   cwd?: string
   dependencies: ParsedPackDependencies
   policy?: PackPolicy
+  /**
+   * Fetch the server's content hashes for an installed dependency version (see `asset.fileManifest`),
+   * so a template omits its unchanged installed dependency files. Content hashes are no longer stored
+   * locally, so omitting needs the API — omit this (or a failing fetch) and those files are kept.
+   */
+  fetchFileManifest?: FetchFileManifest
 }
 
 export interface PackedAsset {
@@ -79,23 +87,28 @@ export async function packAsset(zipFilter: string, opts: PackAssetOptions): Prom
   // Drop anything the zip's own .gitignore excludes (node_modules, build output, secrets…), then —
   // for templates — omit unchanged installed dependency files. Only removals happen, so a smaller
   // count means something was dropped; re-zip only then.
-  const withoutIgnored = applyGitignore(sourceFiles)
+  const { kept: withoutIgnored, gitignored } = applyGitignore(sourceFiles)
   const files = policy.omitUnchangedInstalledFiles
-    ? await withoutUnchangedInstalledFiles(withoutIgnored, assetDependencies, cwd)
+    ? await withoutUnchangedInstalledFiles(
+        withoutIgnored,
+        assetDependencies,
+        cwd,
+        opts.fetchFileManifest,
+      )
     : withoutIgnored
 
-  const sourceCount = Object.keys(sourceFiles).length
+  const originalCount = Object.keys(sourceFiles).length
   const afterIgnoreCount = Object.keys(withoutIgnored).length
   const finalCount = Object.keys(files).length
 
   return {
     sourcePath: zipFile,
-    zip: finalCount < sourceCount ? zipSync(files) : sourceZip,
+    zip: finalCount < originalCount ? zipSync(files) : sourceZip,
     npmDependencies,
     assetDependencies,
     skillDependencies: opts.dependencies.skillDependencies,
     omittedUnchangedInstalledFiles: finalCount < afterIgnoreCount,
-    gitignoredFiles: sourceCount - afterIgnoreCount,
+    gitignoredFiles: gitignored,
   }
 }
 
@@ -113,27 +126,28 @@ async function packDirectory(dir: string, opts: PackAssetOptions): Promise<Packe
 
   const packageJsonFiles: Record<string, Uint8Array> = {}
   if (policy.readPackageJsonDependencies && hasRootPackageJson) {
-    packageJsonFiles['package.json'] = new Uint8Array(await fs.readFile(path.join(dir, 'package.json')))
+    packageJsonFiles['package.json'] = new Uint8Array(
+      await fs.readFile(path.join(dir, 'package.json')),
+    )
   }
   const assetDependencies = mergeAssetDependencies(
-    policy.readPackageJsonDependencies ? packageJsonAssetDependenciesFromFiles(packageJsonFiles) : {},
+    policy.readPackageJsonDependencies
+      ? packageJsonAssetDependenciesFromFiles(packageJsonFiles)
+      : {},
     opts.dependencies.assetDependencies,
   )
   const npmDependencies = {
-    ...(policy.readPackageJsonDependencies ? packageJsonNpmDependenciesFromFiles(packageJsonFiles) : {}),
+    ...(policy.readPackageJsonDependencies
+      ? packageJsonNpmDependenciesFromFiles(packageJsonFiles)
+      : {}),
     ...opts.dependencies.npmDependencies,
   }
 
-  // Hashes of installed dependency files, so an unchanged one is dropped after a single read.
-  const hashesByPath = new Map<string, string>()
-  if (policy.omitUnchangedInstalledFiles && Object.keys(assetDependencies).length > 0) {
-    const installRoot = await findInstallRoot(dir)
-    const lock = await readMarketLock(installRoot)
-    for (const [name, asset] of Object.entries(lock.assets)) {
-      if (!(name in assetDependencies)) continue
-      for (const [file, metadata] of Object.entries(asset.files)) hashesByPath.set(file, metadata.sha256)
-    }
-  }
+  // Content hashes of installed dependency files (from the server), so an unchanged one is dropped
+  // after a single read.
+  const hashesByPath = policy.omitUnchangedInstalledFiles
+    ? await installedDependencyHashes(dir, assetDependencies, opts.fetchFileManifest)
+    : new Map<string, string>()
 
   const files: Record<string, Uint8Array> = {}
   let omittedInstalled = 0
@@ -183,7 +197,7 @@ async function walkWithGitignore(root: string): Promise<{ kept: string[]; ignore
 
     const entries = await fs.readdir(dir, { withFileTypes: true })
     for (const entry of entries) {
-      if (entry.name === '.git' || entry.name === 'node_modules') continue
+      if (ALWAYS_IGNORED_DIRS.has(entry.name)) continue
       const relative = toPosix(path.relative(root, path.join(dir, entry.name)))
       const candidate = entry.isDirectory() ? `${relative}/` : relative
       if (isGitignored(candidate, scoped)) {
@@ -205,20 +219,36 @@ function toPosix(p: string): string {
 
 const GITIGNORE_FILENAME = '.gitignore'
 
+// Directories that never belong in a published asset — VCS/tooling metadata, reinstallable
+// dependencies, and the market's own lock dir (`.drawcall/market-lock.json`). Excluded structurally
+// in every pack path, regardless of any `.gitignore`, so a re-uploaded asset never ships them.
+const ALWAYS_IGNORED_DIRS = new Set(['.git', 'node_modules', '.drawcall'])
+
+function hasAlwaysIgnoredSegment(posixPath: string): boolean {
+  return posixPath.split('/').some((segment) => ALWAYS_IGNORED_DIRS.has(segment))
+}
+
 /**
- * Drop every zip entry excluded by a `.gitignore` inside the zip. `.gitignore` files are applied
- * per-directory (git semantics: a nested `.gitignore` only affects its own subtree), so a naively
- * built asset zip never ships `node_modules`, build output, or secrets.
+ * Filter zip entries: always drop the structural dirs above, then drop whatever a `.gitignore`
+ * inside the zip excludes. `.gitignore` files apply per-directory (git semantics: a nested one only
+ * affects its own subtree). Returns the survivors plus the count of `.gitignore`-matched drops.
  */
-function applyGitignore(files: Record<string, Uint8Array>): Record<string, Uint8Array> {
+function applyGitignore(files: Record<string, Uint8Array>): {
+  kept: Record<string, Uint8Array>
+  gitignored: number
+} {
   const matchers = gitignoreMatchers(files)
-  if (matchers.length === 0) return files
-
   const kept: Record<string, Uint8Array> = {}
+  let gitignored = 0
   for (const [name, content] of Object.entries(files)) {
-    if (!isGitignored(name, matchers)) kept[name] = content
+    if (hasAlwaysIgnoredSegment(name)) continue
+    if (matchers.length > 0 && isGitignored(name, matchers)) {
+      gitignored += 1
+      continue
+    }
+    kept[name] = content
   }
-  return kept
+  return { kept, gitignored }
 }
 
 interface GitignoreMatcher {
@@ -343,40 +373,59 @@ function inferPackPolicy(files: Record<string, Uint8Array>): PackPolicy {
   }
 }
 
-// Drop dependency files whose bytes still match what `.drawcall/market-lock.json` recorded as
-// installed, so a template ships only its own (edited or new) files. Returns the surviving entries.
+// Drop dependency files whose bytes still match the installed asset's canonical content, so a
+// template ships only its own (edited or new) files. Returns the surviving entries.
 async function withoutUnchangedInstalledFiles(
   files: Record<string, Uint8Array>,
   assetDependencies: Record<string, string>,
   cwd: string,
+  fetchFileManifest: FetchFileManifest | undefined,
 ): Promise<Record<string, Uint8Array>> {
-  const dependencyNames = new Set(Object.keys(assetDependencies))
-  if (dependencyNames.size === 0) return files
-
-  const installRoot = await findInstallRoot(cwd)
-  const lock = await readMarketLock(installRoot)
-  const hashesByPath = new Map<string, string>()
-
-  for (const [name, asset] of Object.entries(lock.assets)) {
-    if (!dependencyNames.has(name)) continue
-    for (const [file, metadata] of Object.entries(asset.files)) {
-      hashesByPath.set(file, metadata.sha256)
-    }
-  }
-
+  const hashesByPath = await installedDependencyHashes(cwd, assetDependencies, fetchFileManifest)
   if (hashesByPath.size === 0) return files
 
   const kept: Record<string, Uint8Array> = {}
   for (const [file, content] of Object.entries(files)) {
     const normalizedPath = normalizedZipPath(file)
-    const lockedHash = normalizedPath ? hashesByPath.get(normalizedPath) : undefined
-    if (lockedHash && lockedHash === sha256(content)) continue
+    const canonicalHash = normalizedPath ? hashesByPath.get(normalizedPath) : undefined
+    if (canonicalHash && canonicalHash === sha256(content)) continue
     kept[file] = content
   }
 
   return kept
 }
 
+/**
+ * The canonical content hash of every file belonging to a declared asset dependency, keyed by install
+ * path. Read the installed version of each dependency from the lock, then fetch that version's hashes
+ * from the server (`asset.fileManifest`). Returns an empty map when there's nothing to omit or no way
+ * to fetch (offline / no client) — pack then keeps all files rather than guessing.
+ */
+async function installedDependencyHashes(
+  root: string,
+  assetDependencies: Record<string, string>,
+  fetchFileManifest: FetchFileManifest | undefined,
+): Promise<Map<string, string>> {
+  const hashesByPath = new Map<string, string>()
+  const dependencyNames = Object.keys(assetDependencies)
+  if (!fetchFileManifest || dependencyNames.length === 0) return hashesByPath
+
+  const lock = await readMarketLock(await findInstallRoot(root))
+  for (const name of dependencyNames) {
+    const installed = lock.assets[name]
+    if (!installed) continue
+    let manifest: Record<string, string>
+    try {
+      manifest = await fetchFileManifest(name, installed.version)
+    } catch {
+      // Can't reach the server for this dependency — keep its files rather than wrongly omitting them.
+      continue
+    }
+    for (const [file, hash] of Object.entries(manifest)) hashesByPath.set(file, hash)
+  }
+  return hashesByPath
+}
+
 function normalizedZipPath(file: string): string | null {
   const zipPath = file.replace(/\\/g, '/')
   if (

package/src/schemas.ts

@@ -93,6 +93,18 @@ export const downloadZipSchema = z.object({
   version: semverSchema,
 })
 
+export const fileManifestSchema = z.object({
+  name: assetNameSchema,
+  version: semverSchema,
+})
+
+/**
+ * The content hash of every file in an asset version, keyed by its install path — sha256 hex, the
+ * same digest the CLI computes locally. Lets a caller check whether a local file is identical to the
+ * canonical asset without keeping hashes itself (see `pack` omit-unchanged).
+ */
+export type AssetFileManifest = Record<string, string>
+
 export const generateAssetSchema = z.object({
   description: assetDescriptionSchema.min(3),
   type: assetTypeSchema.optional(),

package/src/skill.ts

@@ -26,7 +26,7 @@ market pack scene.zip --out scene.packed.zip
 5. \`preview <name>\` saves the preview image; no \`--type\` is needed. Not every type has previews (e.g. \`humanoid-animation\`, \`template\`, \`sound-effect\`, \`background-music\`); the CLI reports when one is unavailable.
 6. Use \`--unapproved\` only when the user asks for unapproved/private/admin assets. Do not install unapproved assets without explicit acceptance.
 7. \`generate --type <type> "<prompt>"\` creates and installs a generated asset when that asset type has a generator; it requires login. Currently supported generated types are \`sound-effect\`, \`background-music\`, \`flipbook\`, \`humanoid-model\`, and \`environment\` (a fitting HDRI sky + equirectangular background, generated in ~1-2 min). Generation is provider-specific: prompt style, generated files, indexing fields, and install layout are owned by the asset type. If a type does not support generation yet, the CLI reports unsupported generation. Add \`--access public\` to publish the generated asset publicly, or \`--access private\` to keep it owner-only; when omitted the server defaults to private if you hold the \`market:private\` entitlement, else public (\`--access private\` requires that entitlement). \`generate\` waits for the asset and installs it — one command for quick types. For a long one (e.g. \`humanoid-model\`, >2 min) the call returns after ~2 min with a job id instead of hanging your shell; run \`market generate install <jobId>\` to continue — it resumes the SAME job where the last call left off and installs when ready. Just re-run \`generate install <jobId>\` until it prints "Generated and installed" (it exits 0 while still generating, 1 on failure). No type is flagged "slow" — anything that outlasts one wait just continues on the next call.
-8. Use \`pack <zip>\` to create the same Market asset zip that \`upload\` sends. \`pack\` runs offline, infers template packing from a root \`package.json\`, and accepts \`--type\` only when you need to override that inference. \`upload\` runs the shared pack step internally, then publishes: \`market upload <name> <zip> "<description>" --type <type>\`. Declare dependencies with repeatable flags on either command: \`--npm name@range\`, \`--asset name@range\`, \`--skill label=source\`. Template pack/upload also reads root \`package.json.assetDependencies\`; \`--asset\` flags are additive and must not conflict. Template pack/upload omits installed dependency files whose hashes still match \`.drawcall/market-lock.json\`, so edited local files stay in the template. A skill source is a \`skills add\` argument: a whole repo (\`owner/repo\` or a git URL), a single skill via the full URL form \`https://github.com/owner/repo/tree/<branch>/<subpath>\` (the \`tree/<branch>/<subpath>\` shorthand needs the full URL, not \`owner/repo\`), or a local path to a skill directory inside the zip. Example: \`market upload my-scene scene.zip "A scene" --type model --npm three@^0.178.0 --skill web-design=https://github.com/vercel-labs/agent-skills/tree/main/skills/web-design-guidelines\`.
+8. Use \`pack <zip>\` to create the same Market asset zip that \`upload\` sends. \`pack\` runs offline, infers template packing from a root \`package.json\`, and accepts \`--type\` only when you need to override that inference. \`upload\` runs the shared pack step internally, then publishes: \`market upload <name> <zip> "<description>" --type <type>\`. Declare dependencies with repeatable flags on either command: \`--npm name@range\`, \`--asset name@range\`, \`--skill label=source\`. Template pack/upload also reads root \`package.json.assetDependencies\`; \`--asset\` flags are additive and must not conflict. Template pack/upload omits installed dependency files that still match the installed dependency's canonical content (its file hashes are fetched from the server, not stored locally), so edited local files stay in the template; this omit step needs the API, but simple non-template packs stay offline. A skill source is a \`skills add\` argument: a whole repo (\`owner/repo\` or a git URL), a single skill via the full URL form \`https://github.com/owner/repo/tree/<branch>/<subpath>\` (the \`tree/<branch>/<subpath>\` shorthand needs the full URL, not \`owner/repo\`), or a local path to a skill directory inside the zip. Example: \`market upload my-scene scene.zip "A scene" --type model --npm three@^0.178.0 --skill web-design=https://github.com/vercel-labs/agent-skills/tree/main/skills/web-design-guidelines\`.
 9. Installed \`environment\` assets contain \`public/environment/<name>.hdr\` for Three.js IBL lighting and \`public/environment/<name>-background.webp\` for the visible equirectangular background. Use \`market preview\` to fetch the preview image separately.
 10. Installed \`flipbook\` assets contain \`public/flipbook/<name>.ktx2\`. Render them with \`@drawcall/flipbook\`'s \`Flipbook\` class and Three.js \`KTX2Loader\` for Basis-compressed files; \`market preview\` fetches the middle frame from the flipbook.
 
@@ -63,7 +63,7 @@ Saved preview for wooden-chair@1.0.0: /tmp/wooden-chair.png
 
 Assets may also declare \`skill\` dependencies, installed for you via the \`skills\` CLI (\`skills add\`) during \`install\`. Sources are either a GitHub/git ref or a local path to a skill directory shipped inside the asset. This requires \`npx\` to be available.
 
-Installed non-template assets are saved to \`package.json.assetDependencies\`; templates are scaffolds and are not saved as project asset dependencies. Exact installed versions and file hashes are recorded in \`.drawcall/market-lock.json\`.
+Installed non-template assets are saved to \`package.json.assetDependencies\`; templates are scaffolds and are not saved as project asset dependencies. Exact installed versions and file paths are recorded in \`.drawcall/market-lock.json\`; file content hashes now come from the server (\`asset.fileManifest\`), not the lock.
 
 \`list\` is offline: it reads \`.drawcall/market-lock.json\` from the nearest package root and prints exact installed names, versions, types, and installed file paths.