@drawcall/market 0.1.49 → 0.1.51

package/src/pack.ts

@@ -0,0 +1,348 @@
+import * as fs from 'fs/promises'
+import * as path from 'path'
+import { unzipSync, zipSync } from 'fflate'
+import ignore from 'ignore'
+import type { AssetInstallMetadata } from './contract.js'
+import { findInstallRoot } from './install.js'
+import { readMarketLock, sha256 } from './market-lock.js'
+import {
+  packageJsonAssetDependenciesFromFiles,
+  packageJsonNpmDependenciesFromFiles,
+} from './package-json.js'
+import { MAX_UPLOAD_ZIP_SIZE_BYTES, type AssetType } from './schemas.js'
+
+export interface PackDependencySpecs {
+  npm?: string[]
+  asset?: string[]
+  skill?: string[]
+}
+
+export interface ParsedPackDependencies {
+  npmDependencies: Record<string, string>
+  assetDependencies: Record<string, string>
+  skillDependencies: Record<string, string>
+}
+
+export interface PackPolicy {
+  readPackageJsonDependencies: boolean
+  omitUnchangedInstalledFiles: boolean
+}
+
+export interface PackAssetOptions {
+  cwd?: string
+  dependencies: ParsedPackDependencies
+  policy?: PackPolicy
+}
+
+export interface PackedAsset {
+  sourcePath: string
+  zip: Uint8Array
+  npmDependencies: Record<string, string>
+  assetDependencies: Record<string, string>
+  skillDependencies: Record<string, string>
+  omittedUnchangedInstalledFiles: boolean
+  /** Number of zip entries dropped because a `.gitignore` inside the zip excluded them. */
+  gitignoredFiles: number
+}
+
+export async function packAsset(zipFilter: string, opts: PackAssetOptions): Promise<PackedAsset> {
+  const cwd = opts.cwd ?? process.cwd()
+  const zipFile = await resolveOneZipFile(cwd, zipFilter)
+  const zipStat = await fs.stat(zipFile)
+  if (zipStat.size >= MAX_UPLOAD_ZIP_SIZE_BYTES) {
+    throw new Error('Packed zip must be smaller than 1 GB')
+  }
+
+  const sourceZip = new Uint8Array(await fs.readFile(zipFile))
+  const sourceFiles = unzipSync(sourceZip)
+  const policy = opts.policy ?? inferPackPolicy(sourceFiles)
+  const packageJsonAssetDependencies = policy.readPackageJsonDependencies
+    ? packageJsonAssetDependenciesFromFiles(sourceFiles)
+    : {}
+  const assetDependencies = mergeAssetDependencies(
+    packageJsonAssetDependencies,
+    opts.dependencies.assetDependencies,
+  )
+
+  const packageJsonNpmDependencies = policy.readPackageJsonDependencies
+    ? packageJsonNpmDependenciesFromFiles(sourceFiles)
+    : {}
+  const npmDependencies = { ...packageJsonNpmDependencies, ...opts.dependencies.npmDependencies }
+
+  // Drop anything the zip's own .gitignore excludes (node_modules, build output, secrets…), then —
+  // for templates — omit unchanged installed dependency files. Only removals happen, so a smaller
+  // count means something was dropped; re-zip only then.
+  const withoutIgnored = applyGitignore(sourceFiles)
+  const files = policy.omitUnchangedInstalledFiles
+    ? await withoutUnchangedInstalledFiles(withoutIgnored, assetDependencies, cwd)
+    : withoutIgnored
+
+  const sourceCount = Object.keys(sourceFiles).length
+  const afterIgnoreCount = Object.keys(withoutIgnored).length
+  const finalCount = Object.keys(files).length
+
+  return {
+    sourcePath: zipFile,
+    zip: finalCount < sourceCount ? zipSync(files) : sourceZip,
+    npmDependencies,
+    assetDependencies,
+    skillDependencies: opts.dependencies.skillDependencies,
+    omittedUnchangedInstalledFiles: finalCount < afterIgnoreCount,
+    gitignoredFiles: sourceCount - afterIgnoreCount,
+  }
+}
+
+const GITIGNORE_FILENAME = '.gitignore'
+
+/**
+ * Drop every zip entry excluded by a `.gitignore` inside the zip. `.gitignore` files are applied
+ * per-directory (git semantics: a nested `.gitignore` only affects its own subtree), so a naively
+ * built asset zip never ships `node_modules`, build output, or secrets.
+ */
+function applyGitignore(files: Record<string, Uint8Array>): Record<string, Uint8Array> {
+  const matchers = gitignoreMatchers(files)
+  if (matchers.length === 0) return files
+
+  const kept: Record<string, Uint8Array> = {}
+  for (const [name, content] of Object.entries(files)) {
+    if (!isGitignored(name, matchers)) kept[name] = content
+  }
+  return kept
+}
+
+interface GitignoreMatcher {
+  /** Directory the `.gitignore` governs: '' for the zip root, or e.g. 'sub/'. */
+  dir: string
+  filter: ReturnType<typeof ignore>
+}
+
+function gitignoreMatchers(files: Record<string, Uint8Array>): GitignoreMatcher[] {
+  const matchers: GitignoreMatcher[] = []
+  for (const [name, content] of Object.entries(files)) {
+    if (path.posix.basename(name) !== GITIGNORE_FILENAME) continue
+    const dir = name.slice(0, name.length - GITIGNORE_FILENAME.length)
+    matchers.push({ dir, filter: ignore().add(new TextDecoder().decode(content)) })
+  }
+  return matchers
+}
+
+function isGitignored(name: string, matchers: GitignoreMatcher[]): boolean {
+  // fflate keeps directory entries with a trailing slash; the `ignore` lib matches directory patterns
+  // (`dist/`) against exactly that form, so the path is passed through unchanged.
+  return matchers.some((matcher) => {
+    if (!name.startsWith(matcher.dir)) return false
+    const relative = name.slice(matcher.dir.length)
+    return relative.length > 0 && matcher.filter.ignores(relative)
+  })
+}
+
+export function parsePackDependencies(specs: PackDependencySpecs): ParsedPackDependencies {
+  return {
+    npmDependencies: parseVersionedDeps(specs.npm ?? [], 'npm'),
+    assetDependencies: parseVersionedDeps(specs.asset ?? [], 'asset'),
+    skillDependencies: parseSkillDeps(specs.skill ?? []),
+  }
+}
+
+export function packPolicyForType(type: AssetType | undefined): PackPolicy | undefined {
+  if (!type) return undefined
+  return {
+    readPackageJsonDependencies: type === 'template',
+    omitUnchangedInstalledFiles: type === 'template',
+  }
+}
+
+export function packPolicyFromInstallMetadata(
+  metadata: AssetInstallMetadata | undefined,
+): PackPolicy {
+  return {
+    readPackageJsonDependencies: metadata?.readAssetDependenciesFromPackageJson ?? false,
+    omitUnchangedInstalledFiles: metadata?.omitUnchangedInstalledFilesOnUpload ?? false,
+  }
+}
+
+/**
+ * Parse `name@range` specs (npm or asset deps) into a name→range record. The
+ * range is optional and defaults to `*`. A leading `@` is treated as a scope
+ * marker, so `@scope/pkg@^1.0.0` splits into `@scope/pkg` and `^1.0.0`.
+ */
+export function parseVersionedDeps(specs: string[], kind: 'npm' | 'asset'): Record<string, string> {
+  const out: Record<string, string> = {}
+  for (const spec of specs) {
+    const at = spec.lastIndexOf('@')
+    const hasRange = at > 0
+    const name = hasRange ? spec.slice(0, at) : spec
+    const range = hasRange ? spec.slice(at + 1) : '*'
+    if (!name || !range) {
+      throw new Error(`Invalid ${kind} dependency "${spec}". Use name@range (e.g. three@^0.178.0).`)
+    }
+    if (name in out) {
+      throw new Error(`Duplicate ${kind} dependency "${name}".`)
+    }
+    out[name] = range
+  }
+  return out
+}
+
+/**
+ * Parse `label=source` specs into a label→source record. The source is passed
+ * verbatim to `skills add` (a GitHub/git ref or a local path), so only the
+ * first `=` is treated as the separator.
+ */
+export function parseSkillDeps(specs: string[]): Record<string, string> {
+  const out: Record<string, string> = {}
+  for (const spec of specs) {
+    const eq = spec.indexOf('=')
+    if (eq <= 0 || eq === spec.length - 1) {
+      throw new Error(
+        `Invalid skill dependency "${spec}". Use label=source ` +
+          `(e.g. web-design=vercel-labs/agent-skills).`,
+      )
+    }
+    const label = spec.slice(0, eq)
+    if (label in out) {
+      throw new Error(`Duplicate skill dependency "${label}".`)
+    }
+    out[label] = spec.slice(eq + 1)
+  }
+  return out
+}
+
+function mergeAssetDependencies(
+  fromPackageJson: Record<string, string>,
+  explicit: Record<string, string>,
+): Record<string, string> {
+  const merged = { ...fromPackageJson }
+  for (const [name, range] of Object.entries(explicit)) {
+    if (name in merged && merged[name] !== range) {
+      throw new Error(
+        `Conflicting asset dependency "${name}": package.json has ${merged[name]}, --asset has ${range}`,
+      )
+    }
+    merged[name] = range
+  }
+  return merged
+}
+
+function inferPackPolicy(files: Record<string, Uint8Array>): PackPolicy {
+  const hasRootPackageJson = Boolean(files['package.json'])
+  return {
+    readPackageJsonDependencies: hasRootPackageJson,
+    omitUnchangedInstalledFiles: hasRootPackageJson,
+  }
+}
+
+// Drop dependency files whose bytes still match what `.drawcall/market-lock.json` recorded as
+// installed, so a template ships only its own (edited or new) files. Returns the surviving entries.
+async function withoutUnchangedInstalledFiles(
+  files: Record<string, Uint8Array>,
+  assetDependencies: Record<string, string>,
+  cwd: string,
+): Promise<Record<string, Uint8Array>> {
+  const dependencyNames = new Set(Object.keys(assetDependencies))
+  if (dependencyNames.size === 0) return files
+
+  const installRoot = await findInstallRoot(cwd)
+  const lock = await readMarketLock(installRoot)
+  const hashesByPath = new Map<string, string>()
+
+  for (const [name, asset] of Object.entries(lock.assets)) {
+    if (!dependencyNames.has(name)) continue
+    for (const [file, metadata] of Object.entries(asset.files)) {
+      hashesByPath.set(file, metadata.sha256)
+    }
+  }
+
+  if (hashesByPath.size === 0) return files
+
+  const kept: Record<string, Uint8Array> = {}
+  for (const [file, content] of Object.entries(files)) {
+    const normalizedPath = normalizedZipPath(file)
+    const lockedHash = normalizedPath ? hashesByPath.get(normalizedPath) : undefined
+    if (lockedHash && lockedHash === sha256(content)) continue
+    kept[file] = content
+  }
+
+  return kept
+}
+
+function normalizedZipPath(file: string): string | null {
+  const zipPath = file.replace(/\\/g, '/')
+  if (
+    zipPath.split('/').includes('..') ||
+    path.posix.isAbsolute(zipPath) ||
+    path.win32.isAbsolute(zipPath)
+  ) {
+    return null
+  }
+  return path.posix.normalize(zipPath)
+}
+
+async function resolveOneZipFile(cwd: string, zipFilter: string): Promise<string> {
+  const absolute = path.resolve(cwd, zipFilter)
+  const stat = await maybeStat(absolute)
+  if (stat?.isFile()) return assertZipFile(absolute)
+
+  const files = await listFiles(cwd)
+  const matches = files
+    .filter((file) => matchesFilter(path.relative(cwd, file), zipFilter))
+    .filter(isZipFile)
+    .sort()
+
+  if (matches.length === 0) {
+    throw new Error(`No .zip files matched "${zipFilter}"`)
+  }
+  if (matches.length > 1) {
+    throw new Error(`File filter matched ${matches.length} zips; pack one asset at a time`)
+  }
+
+  return matches[0]
+}
+
+function assertZipFile(file: string): string {
+  if (!isZipFile(file)) throw new Error(`Pack source must be a .zip: ${file}`)
+  return file
+}
+
+function isZipFile(file: string): boolean {
+  return /\.zip$/i.test(file)
+}
+
+async function maybeStat(file: string) {
+  try {
+    return await fs.stat(file)
+  } catch {
+    return null
+  }
+}
+
+async function listFiles(dir: string): Promise<string[]> {
+  const entries = await fs.readdir(dir, { withFileTypes: true })
+  const files: string[] = []
+  for (const entry of entries) {
+    if (entry.name === 'node_modules' || entry.name === '.git') continue
+    const fullPath = path.join(dir, entry.name)
+    if (entry.isDirectory()) {
+      files.push(...(await listFiles(fullPath)))
+    } else if (entry.isFile()) {
+      files.push(fullPath)
+    }
+  }
+  return files
+}
+
+function matchesFilter(file: string, filter: string): boolean {
+  const normalizedFile = file.split(path.sep).join('/')
+  const normalizedFilter = filter.split(path.sep).join('/')
+  const pattern =
+    '^' +
+    escapeRegExp(normalizedFilter)
+      .replace(/\\\*\\\*/g, '.*')
+      .replace(/\\\*/g, '[^/]*') +
+    '$'
+  return new RegExp(pattern).test(normalizedFile)
+}
+
+function escapeRegExp(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+}

package/src/skill.ts

@@ -14,6 +14,7 @@ market search "wooden chair" --type model --limit 3
 market install wooden-chair --cwd "$PWD"
 market list --cwd "$PWD"
 market preview wooden-chair --out /tmp/wooden-chair.png
+market pack scene.zip --out scene.packed.zip
 \`\`\`
 
 ## Workflow
@@ -24,8 +25,8 @@ market preview wooden-chair --out /tmp/wooden-chair.png
 4. \`install\` takes zero or more exact asset names (optionally \`name@range\`). With names, it installs those assets; with no names, it installs \`assetDependencies\` from the nearest \`package.json\`. It does not search or generate. Find names with \`search\` first. No \`--type\` is needed — asset names are unique. Use \`--force\` only when the user agrees to overwrite changed local files.
 5. \`preview <name>\` saves the preview image; no \`--type\` is needed. Not every type has previews (e.g. \`humanoid-animation\`, \`template\`, \`sound-effect\`, \`background-music\`); the CLI reports when one is unavailable.
 6. Use \`--unapproved\` only when the user asks for unapproved/private/admin assets. Do not install unapproved assets without explicit acceptance.
-7. \`generate --type <type> "<prompt>"\` creates and installs a generated asset when that asset type has a generator; it requires login. Currently supported generated types are \`sound-effect\`, \`background-music\`, \`flipbook\`, \`humanoid-model\`, and \`environment\` (a fitting HDRI sky + equirectangular background, generated in ~1-2 min). Generation is provider-specific: prompt style, generated files, indexing fields, and install layout are owned by the asset type. If a type does not support generation yet, the CLI reports unsupported generation. Add \`--access public\` to publish the generated asset publicly, or \`--access private\` to keep it owner-only; when omitted the server defaults to private if you hold the \`market:private\` entitlement, else public (\`--access private\` requires that entitlement).
-8. Upload only when publishing is requested: \`market upload <name> <zip> "<description>" --type <type>\`. Declare dependencies with repeatable flags: \`--npm name@range\`, \`--asset name@range\`, \`--skill label=source\`. Template uploads also read root \`package.json.assetDependencies\`; \`--asset\` flags are additive and must not conflict. Template upload omits installed dependency files whose hashes still match \`.drawcall/market-lock.json\`, so edited local files stay in the template. A skill source is a \`skills add\` argument: a whole repo (\`owner/repo\` or a git URL), a single skill via the full URL form \`https://github.com/owner/repo/tree/<branch>/<subpath>\` (the \`tree/<branch>/<subpath>\` shorthand needs the full URL, not \`owner/repo\`), or a local path to a skill directory inside the zip. Example: \`market upload my-scene scene.zip "A scene" --type model --npm three@^0.178.0 --skill web-design=https://github.com/vercel-labs/agent-skills/tree/main/skills/web-design-guidelines\`.
+7. \`generate --type <type> "<prompt>"\` creates and installs a generated asset when that asset type has a generator; it requires login. Currently supported generated types are \`sound-effect\`, \`background-music\`, \`flipbook\`, \`humanoid-model\`, and \`environment\` (a fitting HDRI sky + equirectangular background, generated in ~1-2 min). Generation is provider-specific: prompt style, generated files, indexing fields, and install layout are owned by the asset type. If a type does not support generation yet, the CLI reports unsupported generation. Add \`--access public\` to publish the generated asset publicly, or \`--access private\` to keep it owner-only; when omitted the server defaults to private if you hold the \`market:private\` entitlement, else public (\`--access private\` requires that entitlement). \`generate\` waits for the asset and installs it — one command for quick types. For a long one (e.g. \`humanoid-model\`, >2 min) the call returns after ~2 min with a job id instead of hanging your shell; run \`market generate install <jobId>\` to continue — it resumes the SAME job where the last call left off and installs when ready. Just re-run \`generate install <jobId>\` until it prints "Generated and installed" (it exits 0 while still generating, 1 on failure). No type is flagged "slow" — anything that outlasts one wait just continues on the next call.
+8. Use \`pack <zip>\` to create the same Market asset zip that \`upload\` sends. \`pack\` runs offline, infers template packing from a root \`package.json\`, and accepts \`--type\` only when you need to override that inference. \`upload\` runs the shared pack step internally, then publishes: \`market upload <name> <zip> "<description>" --type <type>\`. Declare dependencies with repeatable flags on either command: \`--npm name@range\`, \`--asset name@range\`, \`--skill label=source\`. Template pack/upload also reads root \`package.json.assetDependencies\`; \`--asset\` flags are additive and must not conflict. Template pack/upload omits installed dependency files whose hashes still match \`.drawcall/market-lock.json\`, so edited local files stay in the template. A skill source is a \`skills add\` argument: a whole repo (\`owner/repo\` or a git URL), a single skill via the full URL form \`https://github.com/owner/repo/tree/<branch>/<subpath>\` (the \`tree/<branch>/<subpath>\` shorthand needs the full URL, not \`owner/repo\`), or a local path to a skill directory inside the zip. Example: \`market upload my-scene scene.zip "A scene" --type model --npm three@^0.178.0 --skill web-design=https://github.com/vercel-labs/agent-skills/tree/main/skills/web-design-guidelines\`.
 9. Installed \`environment\` assets contain \`public/environment/<name>.hdr\` for Three.js IBL lighting and \`public/environment/<name>-background.webp\` for the visible equirectangular background. Use \`market preview\` to fetch the preview image separately.
 10. Installed \`flipbook\` assets contain \`public/flipbook/<name>.ktx2\`. Render them with \`@drawcall/flipbook\`'s \`Flipbook\` class and Three.js \`KTX2Loader\` for Basis-compressed files; \`market preview\` fetches the middle frame from the flipbook.