@drawcall/market 0.1.49 → 0.1.51

package/src/commands/generate.ts

@@ -1,9 +1,10 @@
-import ora from 'ora'
+import ora, { type Ora } from 'ora'
 import { getCliClient } from '../cli-client.js'
-import { generateAndWait } from '../generate.js'
+import { GenerateError, startGeneration, waitForGeneration, type PollOutcome } from '../generate.js'
+import type { MarketClient } from '../client.js'
 import { resolve } from '../resolve.js'
 import { install as runInstall } from '../install.js'
-import { generatedInstallResult } from '../output.js'
+import { generatedInstallResult, generationStartedResult } from '../output.js'
 import type { AssetAccess, AssetType } from '../schemas.js'
 
 export interface GenerateCommandOptions {
@@ -18,40 +19,86 @@ export async function generateCommand(
   opts: GenerateCommandOptions,
 ): Promise<void> {
   const { client, baseUrl } = await getCliClient({ baseUrl: opts.baseUrl, requireAuth: true })
-
-  const spinner = ora({
-    text: `Generating "${description}"`,
-    isEnabled: Boolean(process.stderr.isTTY),
-    isSilent: !process.stderr.isTTY,
-  }).start()
+  const spinner = startSpinner(`Generating "${description}"`)
   try {
-    const generated = await generateAndWait(
-      client,
-      { description, type: opts.type, access: opts.access },
-      {
-        onProgress: (msg) => {
-          spinner.text = msg
-        },
+    const started = await startGeneration(client, {
+      description,
+      type: opts.type,
+      access: opts.access,
+    })
+    // Fast providers finish inline; slow, job-based ones return a job to poll.
+    if (started.status === 'completed') {
+      await installGenerated(client, started, { cwd: opts.cwd, baseUrl, spinner })
+      return
+    }
+    // Block-and-poll for a bounded window (the command waits, not the agent). If the job outlasts the
+    // window, hand back its id so the caller continues with `generate install`.
+    const outcome = await waitForGeneration(client, started.jobId, {
+      onProgress: (message) => {
+        spinner.text = message
       },
-    )
-
-    spinner.text = `Resolving ${generated.assetName}@${generated.version}`
-    const resolution = await resolve(client.asset, [
-      { name: generated.assetName, range: generated.version },
-    ])
-
-    await runInstall(client, resolution, {
+    })
+    await finishGeneration(client, outcome, {
       cwd: opts.cwd,
       baseUrl,
-      onProgress: (msg) => {
-        spinner.text = msg
-      },
+      spinner,
+      stillRunning: generationStartedResult(started.jobId),
     })
-
-    spinner.stop()
-    console.log(generatedInstallResult(generated.assetName, generated.version))
   } catch (err) {
     spinner.stop()
     throw err
   }
 }
+
+interface InstallContext {
+  cwd?: string
+  baseUrl: string
+  spinner: Ora
+}
+
+// Finish a poll outcome: install on completion, fail loudly on error, or print the still-running note
+// (the job keeps running server-side). Shared by `generate` and `generate install`.
+export async function finishGeneration(
+  client: MarketClient,
+  outcome: PollOutcome,
+  ctx: InstallContext & { stillRunning: string },
+): Promise<void> {
+  if (outcome.status === 'completed') {
+    await installGenerated(client, outcome, ctx)
+    return
+  }
+  ctx.spinner.stop()
+  if (outcome.status === 'failed') {
+    throw new GenerateError(outcome.error)
+  }
+  console.log(ctx.stillRunning)
+}
+
+// Install a finished generation into the project and report it.
+async function installGenerated(
+  client: MarketClient,
+  generated: { assetName: string; version: string },
+  ctx: InstallContext,
+): Promise<void> {
+  ctx.spinner.text = `Resolving ${generated.assetName}@${generated.version}`
+  const resolution = await resolve(client.asset, [
+    { name: generated.assetName, range: generated.version },
+  ])
+  await runInstall(client, resolution, {
+    cwd: ctx.cwd,
+    baseUrl: ctx.baseUrl,
+    onProgress: (message) => {
+      ctx.spinner.text = message
+    },
+  })
+  ctx.spinner.stop()
+  console.log(generatedInstallResult(generated.assetName, generated.version))
+}
+
+export function startSpinner(text: string): Ora {
+  return ora({
+    text,
+    isEnabled: Boolean(process.stderr.isTTY),
+    isSilent: !process.stderr.isTTY,
+  }).start()
+}

package/src/commands/pack.ts

@@ -0,0 +1,55 @@
+import * as fs from 'fs/promises'
+import * as path from 'path'
+import ora from 'ora'
+import { packAsset, packPolicyForType, parsePackDependencies } from '../pack.js'
+import { packResult } from '../output.js'
+import type { AssetType } from '../schemas.js'
+
+export interface PackCommandOptions {
+  type?: AssetType
+  cwd?: string
+  out?: string
+  /** npm dependencies, each `name@range` (range defaults to `*`). */
+  npm?: string[]
+  /** asset dependencies, each `name@range` (range defaults to `*`). */
+  asset?: string[]
+  /** skill dependencies, each `label=source` passed to `skills add`. */
+  skill?: string[]
+}
+
+export async function packCommand(zipFilter: string, opts: PackCommandOptions): Promise<void> {
+  const spinner = ora({
+    text: 'Preparing pack',
+    isEnabled: Boolean(process.stderr.isTTY),
+    isSilent: !process.stderr.isTTY,
+  }).start()
+
+  try {
+    const cwd = opts.cwd ?? process.cwd()
+    const packed = await packAsset(zipFilter, {
+      cwd,
+      dependencies: parsePackDependencies({
+        npm: opts.npm,
+        asset: opts.asset,
+        skill: opts.skill,
+      }),
+      policy: packPolicyForType(opts.type),
+    })
+    const out = opts.out ? path.resolve(cwd, opts.out) : defaultPackPath(packed.sourcePath)
+
+    spinner.text = `Writing ${path.relative(cwd, out) || out}`
+    await fs.mkdir(path.dirname(out), { recursive: true })
+    await fs.writeFile(out, packed.zip)
+
+    spinner.stop()
+    console.log(packResult(out, packed))
+  } catch (err) {
+    spinner.stop()
+    throw err
+  }
+}
+
+function defaultPackPath(sourcePath: string): string {
+  const ext = path.extname(sourcePath)
+  return path.join(path.dirname(sourcePath), `${path.basename(sourcePath, ext)}.packed${ext}`)
+}

package/src/commands/upload.ts

@@ -1,21 +1,13 @@
-import * as fs from 'fs/promises'
-import * as path from 'path'
-import { unzipSync, zipSync } from 'fflate'
 import ora from 'ora'
 import semver from 'semver'
 import { getCliClient } from '../cli-client.js'
-import { findInstallRoot } from '../install.js'
-import { readMarketLock, sha256 } from '../market-lock.js'
-import {
-  packageJsonAssetDependenciesFromFiles,
-  packageJsonNpmDependenciesFromFiles,
-} from '../package-json.js'
-import { MAX_ASSET_DESCRIPTION_LENGTH, MAX_UPLOAD_ZIP_SIZE_BYTES } from '../schemas.js'
+import { packAsset, packPolicyFromInstallMetadata, parsePackDependencies } from '../pack.js'
 import { unchangedUploadResult, uploadResult } from '../output.js'
 import {
   assetDescriptionSchema,
   assetNameSchema,
   semverSchema,
+  MAX_ASSET_DESCRIPTION_LENGTH,
   type AssetAccess,
   type AssetType,
 } from '../schemas.js'
@@ -52,37 +44,21 @@ export async function uploadCommand(
     const parsedVersion = opts.version ? semverSchema.parse(opts.version) : undefined
     const parsedDescription = parseUploadDescription(description)
     // Parse dep flags before any network work so a malformed spec fails fast.
-    const explicitNpmDependencies = parseVersionedDeps(opts.npm ?? [], 'npm')
-    const explicitAssetDependencies = parseVersionedDeps(opts.asset ?? [], 'asset')
-    const skillDependencies = parseSkillDeps(opts.skill ?? [])
+    const dependencies = parsePackDependencies({
+      npm: opts.npm,
+      asset: opts.asset,
+      skill: opts.skill,
+    })
     const cwd = opts.cwd ?? process.cwd()
     const type = opts.type
-    const zipFile = await resolveOneZipFile(cwd, zipFilter)
-    const zipStat = await fs.stat(zipFile)
-    if (zipStat.size >= MAX_UPLOAD_ZIP_SIZE_BYTES) {
-      throw new Error('Upload zip must be smaller than 1 GB')
-    }
-    const sourceZip = new Uint8Array(await fs.readFile(zipFile))
     const { client } = await getCliClient({ baseUrl: opts.baseUrl, requireAuth: true })
     const installMetadata = await client.asset.installMetadata()
-    const typeMetadata = installMetadata[type]
-    const sourceFiles = unzipSync(sourceZip)
-    const packageJsonAssetDependencies = typeMetadata?.readAssetDependenciesFromPackageJson
-      ? packageJsonAssetDependenciesFromFiles(sourceFiles)
-      : {}
-    const assetDependencies = mergeAssetDependencies(
-      packageJsonAssetDependencies,
-      explicitAssetDependencies,
-    )
-    // A type that ships a package.json (templates) also carries its npm dependencies there; capture
-    // them so an install over an existing package.json brings them. Explicit --npm wins on conflict.
-    const packageJsonNpmDependencies = typeMetadata?.readAssetDependenciesFromPackageJson
-      ? packageJsonNpmDependenciesFromFiles(sourceFiles)
-      : {}
-    const npmDependencies = { ...packageJsonNpmDependencies, ...explicitNpmDependencies }
-    const zip = typeMetadata?.omitUnchangedInstalledFilesOnUpload
-      ? await omitUnchangedInstalledFiles(sourceZip, sourceFiles, assetDependencies, cwd)
-      : sourceZip
+    const packed = await packAsset(zipFilter, {
+      cwd,
+      dependencies,
+      policy: packPolicyFromInstallMetadata(installMetadata[type]),
+    })
+    const { zip, npmDependencies, assetDependencies, skillDependencies } = packed
     const profile = await client.user.getProfile()
     if (!profile) {
       throw new Error('Not logged in. Run `market login` first.')
@@ -151,118 +127,6 @@ export function parseUploadDescription(description: string): string {
   }
 }
 
-/**
- * Parse `name@range` specs (npm or asset deps) into a name→range record. The
- * range is optional and defaults to `*`. A leading `@` is treated as a scope
- * marker, so `@scope/pkg@^1.0.0` splits into `@scope/pkg` and `^1.0.0`.
- */
-export function parseVersionedDeps(specs: string[], kind: 'npm' | 'asset'): Record<string, string> {
-  const out: Record<string, string> = {}
-  for (const spec of specs) {
-    const at = spec.lastIndexOf('@')
-    const hasRange = at > 0
-    const name = hasRange ? spec.slice(0, at) : spec
-    const range = hasRange ? spec.slice(at + 1) : '*'
-    if (!name || !range) {
-      throw new Error(`Invalid ${kind} dependency "${spec}". Use name@range (e.g. three@^0.178.0).`)
-    }
-    if (name in out) {
-      throw new Error(`Duplicate ${kind} dependency "${name}".`)
-    }
-    out[name] = range
-  }
-  return out
-}
-
-/**
- * Parse `label=source` specs into a label→source record. The source is passed
- * verbatim to `skills add` (a GitHub/git ref or a local path), so only the
- * first `=` is treated as the separator.
- */
-export function parseSkillDeps(specs: string[]): Record<string, string> {
-  const out: Record<string, string> = {}
-  for (const spec of specs) {
-    const eq = spec.indexOf('=')
-    if (eq <= 0 || eq === spec.length - 1) {
-      throw new Error(
-        `Invalid skill dependency "${spec}". Use label=source ` +
-          `(e.g. web-design=vercel-labs/agent-skills).`,
-      )
-    }
-    const label = spec.slice(0, eq)
-    if (label in out) {
-      throw new Error(`Duplicate skill dependency "${label}".`)
-    }
-    out[label] = spec.slice(eq + 1)
-  }
-  return out
-}
-
-function mergeAssetDependencies(
-  fromPackageJson: Record<string, string>,
-  explicit: Record<string, string>,
-): Record<string, string> {
-  const merged = { ...fromPackageJson }
-  for (const [name, range] of Object.entries(explicit)) {
-    if (name in merged && merged[name] !== range) {
-      throw new Error(
-        `Conflicting asset dependency "${name}": package.json has ${merged[name]}, --asset has ${range}`,
-      )
-    }
-    merged[name] = range
-  }
-  return merged
-}
-
-async function omitUnchangedInstalledFiles(
-  sourceZip: Uint8Array,
-  sourceFiles: Record<string, Uint8Array>,
-  assetDependencies: Record<string, string>,
-  cwd: string,
-): Promise<Uint8Array> {
-  const dependencyNames = new Set(Object.keys(assetDependencies))
-  if (dependencyNames.size === 0) return sourceZip
-
-  const installRoot = await findInstallRoot(cwd)
-  const lock = await readMarketLock(installRoot)
-  const hashesByPath = new Map<string, string>()
-
-  for (const [name, asset] of Object.entries(lock.assets)) {
-    if (!dependencyNames.has(name)) continue
-    for (const [file, metadata] of Object.entries(asset.files)) {
-      hashesByPath.set(file, metadata.sha256)
-    }
-  }
-
-  if (hashesByPath.size === 0) return sourceZip
-
-  let omitted = false
-  const filtered: Record<string, Uint8Array> = {}
-  for (const [file, content] of Object.entries(sourceFiles)) {
-    const normalizedPath = normalizedZipPath(file)
-    const lockedHash = normalizedPath ? hashesByPath.get(normalizedPath) : undefined
-    if (lockedHash && lockedHash === sha256(content)) {
-      omitted = true
-      continue
-    }
-    filtered[file] = content
-  }
-
-  return omitted ? zipSync(filtered) : sourceZip
-}
-
-function normalizedZipPath(file: string): string | null {
-  const zipPath = file.replace(/\\/g, '/')
-  if (
-    zipPath.split('/').includes('..') ||
-    path.posix.isAbsolute(zipPath) ||
-    path.win32.isAbsolute(zipPath)
-  ) {
-    return null
-  }
-  return path.posix.normalize(zipPath)
-}
-
 /** Compare a stored dependency JSON string against a freshly parsed record. */
 function depsEqual(storedJson: string, next: Record<string, string>): boolean {
   let stored: Record<string, string>
@@ -275,71 +139,6 @@ function depsEqual(storedJson: string, next: Record<string, string>): boolean {
   return keys.length === Object.keys(next).length && keys.every((key) => stored[key] === next[key])
 }
 
-async function resolveOneZipFile(cwd: string, zipFilter: string): Promise<string> {
-  const absolute = path.resolve(cwd, zipFilter)
-  const stat = await maybeStat(absolute)
-  if (stat?.isFile()) return assertZipFile(absolute)
-
-  const files = await listFiles(cwd)
-  const matches = files
-    .filter((file) => matchesFilter(path.relative(cwd, file), zipFilter))
-    .filter(isZipFile)
-    .sort()
-
-  if (matches.length === 0) {
-    throw new Error(`No .zip files matched "${zipFilter}"`)
-  }
-  if (matches.length > 1) {
-    throw new Error(`File filter matched ${matches.length} zips; upload one asset at a time`)
-  }
-
-  return matches[0]
-}
-
-function assertZipFile(file: string): string {
-  if (!isZipFile(file)) throw new Error(`Upload file must be a .zip: ${file}`)
-  return file
-}
-
-function isZipFile(file: string): boolean {
-  return /\.zip$/i.test(file)
-}
-
-async function maybeStat(file: string) {
-  try {
-    return await fs.stat(file)
-  } catch {
-    return null
-  }
-}
-
-async function listFiles(dir: string): Promise<string[]> {
-  const entries = await fs.readdir(dir, { withFileTypes: true })
-  const files: string[] = []
-  for (const entry of entries) {
-    if (entry.name === 'node_modules' || entry.name === '.git') continue
-    const fullPath = path.join(dir, entry.name)
-    if (entry.isDirectory()) {
-      files.push(...(await listFiles(fullPath)))
-    } else if (entry.isFile()) {
-      files.push(fullPath)
-    }
-  }
-  return files
-}
-
-function matchesFilter(file: string, filter: string): boolean {
-  const normalizedFile = file.split(path.sep).join('/')
-  const normalizedFilter = filter.split(path.sep).join('/')
-  const pattern =
-    '^' +
-    escapeRegExp(normalizedFilter)
-      .replace(/\\\*\\\*/g, '.*')
-      .replace(/\\\*/g, '[^/]*') +
-    '$'
-  return new RegExp(pattern).test(normalizedFile)
-}
-
 function nextVersion(latest?: string): string {
   if (!latest) return '1.0.0'
   const version = semver.inc(latest, 'patch')
@@ -356,9 +155,5 @@ function bytesEqual(a: Uint8Array, b: Uint8Array): boolean {
 }
 
 function toArrayBuffer(bytes: Uint8Array): ArrayBuffer {
-  return new Uint8Array(bytes).buffer
-}
-
-function escapeRegExp(value: string): string {
-  return value.replace(/[|\\{}()[\]^$+?.*]/g, '\\$&')
+  return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer
 }

package/src/generate.ts

@@ -1,5 +1,5 @@
 import type { MarketClient } from './client.js'
-import type { AssetAccess, AssetType } from './schemas.js'
+import type { AssetAccess, AssetType, GenerateJobStatus, GenerateResponse } from './schemas.js'
 
 export class GenerateError extends Error {
   constructor(message: string) {
@@ -15,49 +15,61 @@ export interface GenerateInput {
   access?: AssetAccess
 }
 
-export interface GenerateResult {
-  assetName: string
-  version: string
+// How often the CLI polls a running job.
+const POLL_INTERVAL_MS = 4_000
+// How long one `generate` / `generate install` call blocks-and-polls before handing back control.
+// The command does the waiting (the agent never sleeps between checks). When the window elapses the
+// call returns and the caller just runs `generate install <jobId>` again, which picks up the SAME
+// server-side job where this left off — so no asset type needs to be flagged "slow" ahead of time;
+// anything that outlasts one window simply continues on the next call.
+const WINDOW_MS = 120_000
+
+// Start a generation. Fast providers answer `completed` inline; slow, job-based providers answer
+// `pending` with a jobId to poll (see `waitForGeneration` / `generate install`).
+export function startGeneration(
+  client: MarketClient,
+  input: GenerateInput,
+): Promise<GenerateResponse> {
+  return client.asset.generate(input)
 }
 
-export interface GenerateOptions {
-  onProgress?: (message: string) => void
+// One poll of a generation job.
+export function pollGeneration(client: MarketClient, jobId: string): Promise<GenerateJobStatus> {
+  return client.asset.generateStatus({ jobId })
 }
 
-// Poll cadence for a job-based generation. Each poll is a short, independently-retriable request;
-// the client sleeps between them, so no single request is held open for the length of the job (that
-// is exactly the client/proxy-timeout trap the async flow removes). The deadline sits above the
-// slowest provider's own budget (the humanoid model's upstream job runs up to ~10 min).
-const POLL_INTERVAL_MS = 4_000
-const POLL_DEADLINE_MS = 15 * 60_000
+// The bounded outcome of waiting on a job: it settled (completed/failed), or the window elapsed while
+// it was still running.
+export type PollOutcome =
+  | { status: 'completed'; assetName: string; version: string }
+  | { status: 'failed'; error: string }
+  | { status: 'running' }
 
-export async function generateAndWait(
+// Poll a job until it settles or `windowMs` elapses. Bounded so a single call stays under a harness
+// timeout; on timeout it returns `running` (the job keeps going server-side — run again to continue).
+export async function waitForGeneration(
   client: MarketClient,
-  input: GenerateInput,
-  opts: GenerateOptions = {},
-): Promise<GenerateResult> {
+  jobId: string,
+  opts: { windowMs?: number; onProgress?: (message: string) => void } = {},
+): Promise<PollOutcome> {
+  const windowMs = opts.windowMs ?? WINDOW_MS
   const report = opts.onProgress ?? (() => {})
-
-  report('Generating asset')
-  const started = await client.asset.generate(input)
-  if (started.status === 'completed') {
-    return { assetName: started.assetName, version: started.version }
-  }
-
   const startedAt = Date.now()
   for (;;) {
-    if (Date.now() - startedAt > POLL_DEADLINE_MS) {
-      throw new GenerateError('Generation timed out.')
-    }
-    await delay(POLL_INTERVAL_MS)
-    report('Generating asset')
-    const status = await client.asset.generateStatus({ jobId: started.jobId })
+    const status = await pollGeneration(client, jobId)
     if (status.status === 'completed') {
-      return { assetName: status.assetName, version: status.version }
+      return { status: 'completed', assetName: status.assetName, version: status.version }
     }
     if (status.status === 'failed') {
-      throw new GenerateError(status.error)
+      return { status: 'failed', error: status.error }
+    }
+    const remaining = windowMs - (Date.now() - startedAt)
+    if (remaining <= 0) {
+      return { status: 'running' }
     }
+    report('Generating asset')
+    // Trim the last sleep so the call returns as soon as the window passes, not a poll-interval later.
+    await delay(Math.min(POLL_INTERVAL_MS, remaining))
   }
 }
 

package/src/index.ts

@@ -40,8 +40,8 @@ export { resolve, ResolutionError } from './resolve.js'
 export type { ResolvedAsset, ResolveResult } from './resolve.js'
 
 // Generate
-export { generateAndWait, GenerateError } from './generate.js'
-export type { GenerateInput, GenerateResult, GenerateOptions } from './generate.js'
+export { startGeneration, pollGeneration, GenerateError } from './generate.js'
+export type { GenerateInput } from './generate.js'
 
 // Agent
 export { agentAndWait, AgentError } from './agent.js'

package/src/output.ts

@@ -1,6 +1,7 @@
 import type { AssetInstallMetadata, AssetSearchResult } from './contract.js'
 import type { ListInstalledAssetsResult } from './commands/list.js'
 import type { InstallResult } from './install.js'
+import type { PackedAsset } from './pack.js'
 
 export function assetVersionRef(name: string, version?: string): string {
   return version ? `${name}@${version}` : name
@@ -125,6 +126,20 @@ export function generatedInstallResult(name: string, version: string): string {
   return `Generated and installed ${assetVersionRef(name, version)}`
 }
 
+// Printed when `generate` returns before the job finished: it keeps running server-side and the
+// caller continues it with `generate install`, which installs the asset once it is ready.
+export function generationStartedResult(jobId: string): string {
+  return [
+    'Still generating after a couple of minutes — it continues in the background.',
+    `Run this to pick up where it left off and install when ready:  market generate install ${jobId}`,
+  ].join('\n')
+}
+
+// Printed by `generate install` when the job is still running after its wait window.
+export function generationRunningResult(jobId: string): string {
+  return `Still generating. Run again to continue:  market generate install ${jobId}`
+}
+
 export function uploadResult(name: string, version: string): string {
   return `Uploaded ${assetVersionRef(name, version)}`
 }
@@ -133,6 +148,20 @@ export function unchangedUploadResult(name: string, version: string): string {
   return `No upload needed: ${assetVersionRef(name, version)} is unchanged`
 }
 
+export function packResult(out: string, packed: PackedAsset): string {
+  const lines = [`Packed: ${out}`]
+  if (packed.gitignoredFiles > 0) {
+    lines.push(`Excluded ${packed.gitignoredFiles} .gitignore'd file(s).`)
+  }
+  if (packed.omittedUnchangedInstalledFiles) {
+    lines.push('Omitted unchanged installed dependency files.')
+  }
+  lines.push(...dependencyLines('npm dependencies', packed.npmDependencies))
+  lines.push(...dependencyLines('asset dependencies', packed.assetDependencies))
+  lines.push(...dependencyLines('skill dependencies', packed.skillDependencies))
+  return lines.join('\n')
+}
+
 export function previewResult(name: string, version: string, out: string): string {
   return `Saved preview for ${assetVersionRef(name, version)}: ${out}`
 }
@@ -204,3 +233,12 @@ function unique<T>(values: T[]): T[] {
 function indent(lines: string[], prefix: string): string[] {
   return lines.map((line) => `${prefix}${line}`)
 }
+
+function dependencyLines(label: string, deps: Record<string, string>): string[] {
+  const entries = Object.entries(deps)
+  if (entries.length === 0) return []
+  return [
+    `${label}:`,
+    ...entries.sort(([a], [b]) => a.localeCompare(b)).map(([name, value]) => `- ${name}: ${value}`),
+  ]
+}