@drawcall/market 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/list.d.ts +4 -2
- package/dist/commands/list.d.ts.map +1 -1
- package/dist/commands/list.js +1 -1
- package/dist/commands/list.js.map +1 -1
- package/dist/commands/upload.d.ts.map +1 -1
- package/dist/commands/upload.js +4 -1
- package/dist/commands/upload.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/install.d.ts +2 -1
- package/dist/install.d.ts.map +1 -1
- package/dist/install.js +148 -35
- package/dist/install.js.map +1 -1
- package/dist/output.d.ts +6 -0
- package/dist/output.d.ts.map +1 -1
- package/dist/output.js +24 -2
- package/dist/output.js.map +1 -1
- package/dist/pack.d.ts +49 -9
- package/dist/pack.d.ts.map +1 -1
- package/dist/pack.js +82 -77
- package/dist/pack.js.map +1 -1
- package/dist/project-walk.d.ts +18 -0
- package/dist/project-walk.d.ts.map +1 -0
- package/dist/project-walk.js +83 -0
- package/dist/project-walk.js.map +1 -0
- package/dist/resolve.d.ts +2 -2
- package/dist/resolve.d.ts.map +1 -1
- package/dist/resolve.js.map +1 -1
- package/dist/schemas.d.ts +16 -14
- package/dist/schemas.d.ts.map +1 -1
- package/dist/schemas.js +9 -3
- package/dist/schemas.js.map +1 -1
- package/dist/v1/contract.d.ts +5 -5
- package/dist/v1/contract.d.ts.map +1 -1
- package/dist/v1/contract.js +6 -4
- package/dist/v1/contract.js.map +1 -1
- package/package.json +1 -1
- package/src/commands/list.ts +8 -3
- package/src/commands/upload.ts +3 -1
- package/src/index.ts +4 -2
- package/src/install.ts +218 -47
- package/src/output.ts +33 -10
- package/src/pack.ts +125 -95
- package/src/project-walk.ts +95 -0
- package/src/resolve.ts +4 -3
- package/src/schemas.ts +24 -11
- package/src/v1/contract.ts +8 -6
package/src/pack.ts
CHANGED
|
@@ -3,9 +3,17 @@ import * as fs from 'fs/promises'
|
|
|
3
3
|
import * as path from 'path'
|
|
4
4
|
import { unzipSync, zipSync } from 'fflate'
|
|
5
5
|
import ignore from 'ignore'
|
|
6
|
+
import semver from 'semver'
|
|
7
|
+
import {
|
|
8
|
+
ALWAYS_IGNORED_DIRS,
|
|
9
|
+
GITIGNORE_FILENAME,
|
|
10
|
+
isGitignored,
|
|
11
|
+
findInstallRoot,
|
|
12
|
+
walkWithGitignore,
|
|
13
|
+
type GitignoreMatcher,
|
|
14
|
+
} from './project-walk.js'
|
|
6
15
|
import type { MarketClient } from './client.js'
|
|
7
16
|
import type { AssetInstallMetadata } from './contract.js'
|
|
8
|
-
import { findInstallRoot } from './install.js'
|
|
9
17
|
import {
|
|
10
18
|
packageJsonAssetDependencies,
|
|
11
19
|
packageJsonAssetDependenciesFromFiles,
|
|
@@ -19,6 +27,7 @@ import {
|
|
|
19
27
|
assetDependencyValue,
|
|
20
28
|
semverSchema,
|
|
21
29
|
type AssetDependencies,
|
|
30
|
+
type AssetDependencyAlias,
|
|
22
31
|
type AssetType,
|
|
23
32
|
} from './schemas.js'
|
|
24
33
|
|
|
@@ -46,10 +55,10 @@ export interface PackAssetOptions {
|
|
|
46
55
|
dependencies: ParsedPackDependencies
|
|
47
56
|
policy?: PackPolicy
|
|
48
57
|
/**
|
|
49
|
-
* Fetch
|
|
50
|
-
*
|
|
51
|
-
* path, so renamed installed files are omitted too and recorded as aliases.
|
|
52
|
-
* failing fetch) and the dependency's files are kept.
|
|
58
|
+
* Fetch `path → etag` (md5) for a dependency's files given the declared range (see
|
|
59
|
+
* `fileManifestForRange`). Pack matches local files against these hashes by content —
|
|
60
|
+
* independent of path, so renamed installed files are omitted too and recorded as aliases.
|
|
61
|
+
* Omit this (or a failing fetch) and the dependency's files are kept.
|
|
53
62
|
*/
|
|
54
63
|
fetchFileManifest?: FetchFileManifest
|
|
55
64
|
}
|
|
@@ -63,36 +72,85 @@ export interface PackedAsset {
|
|
|
63
72
|
omittedUnchangedInstalledFiles: boolean
|
|
64
73
|
/** Number of zip entries dropped because a `.gitignore` inside the zip excluded them. */
|
|
65
74
|
gitignoredFiles: number
|
|
75
|
+
/** Dependencies whose file index could not be fetched: files kept, aliases untouched. */
|
|
76
|
+
skippedDependencies: string[]
|
|
77
|
+
/** Dependency files whose canonical bytes were found nowhere (locally modified or deleted). */
|
|
78
|
+
missingDependencyFiles: Array<{ name: string; path: string }>
|
|
66
79
|
}
|
|
67
80
|
|
|
68
|
-
|
|
69
|
-
|
|
81
|
+
/** md5 hex — the same digest R2 serves as every file's etag, so local bytes match server hashes. */
|
|
82
|
+
export function md5(bytes: Uint8Array): string {
|
|
83
|
+
return createHash('md5').update(bytes).digest('hex')
|
|
70
84
|
}
|
|
71
85
|
|
|
72
86
|
/**
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
*
|
|
87
|
+
* Cap on how many in-range versions contribute hashes to one dependency's manifest. High enough
|
|
88
|
+
* that any real project's installed version is covered; bounds the API calls for assets with long
|
|
89
|
+
* histories (an out-of-cap version's files simply stay unmatched and ship — the fail-open
|
|
90
|
+
* direction).
|
|
91
|
+
*/
|
|
92
|
+
const MANIFEST_VERSION_CAP = 20
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* A `FetchFileManifest` backed by the Market API: `path → etag` (md5) per dependency file. Without
|
|
96
|
+
* a local lockfile the installed version is unknown, so a non-exact range merges the file indexes
|
|
97
|
+
* of every published version satisfying it (newest wins per path) — a hash match identifies the
|
|
98
|
+
* file regardless of which version it came from. Content addressing keeps mistakes safe: a stale
|
|
99
|
+
* hash simply fails to match and the file is kept.
|
|
77
100
|
*/
|
|
78
101
|
export function fileManifestForRange(asset: MarketClient['asset']): FetchFileManifest {
|
|
79
102
|
return async (name, range) => {
|
|
80
103
|
const exact = exactRangeVersion(range)
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
104
|
+
if (exact) return fileIndexHashes(asset, name, exact)
|
|
105
|
+
|
|
106
|
+
const { versions } = await asset.versions({ name })
|
|
107
|
+
const inRange = versions
|
|
108
|
+
.map((entry) => entry.version)
|
|
109
|
+
.filter((version) => semver.satisfies(version, range))
|
|
110
|
+
.sort(semver.compare)
|
|
111
|
+
.slice(-MANIFEST_VERSION_CAP)
|
|
112
|
+
|
|
113
|
+
// Oldest first, so a newer version's hash wins when a path recurs across versions.
|
|
114
|
+
const merged: Record<string, string> = {}
|
|
115
|
+
for (const hashes of await mapWithConcurrency(inRange, 4, (version) =>
|
|
116
|
+
fileIndexHashes(asset, name, version),
|
|
117
|
+
)) {
|
|
118
|
+
Object.assign(merged, hashes)
|
|
119
|
+
}
|
|
120
|
+
return merged
|
|
88
121
|
}
|
|
89
122
|
}
|
|
90
123
|
|
|
124
|
+
async function fileIndexHashes(
|
|
125
|
+
asset: MarketClient['asset'],
|
|
126
|
+
name: string,
|
|
127
|
+
version: string,
|
|
128
|
+
): Promise<Record<string, string>> {
|
|
129
|
+
const index = await asset.files({ name, version })
|
|
130
|
+
return Object.fromEntries(index.files.map((file) => [file.path, file.etag]))
|
|
131
|
+
}
|
|
132
|
+
|
|
91
133
|
function exactRangeVersion(range: string): string | null {
|
|
92
134
|
const candidate = range.trim().replace(/^=/u, '')
|
|
93
135
|
return semverSchema.safeParse(candidate).success ? candidate : null
|
|
94
136
|
}
|
|
95
137
|
|
|
138
|
+
async function mapWithConcurrency<T, R>(
|
|
139
|
+
items: readonly T[],
|
|
140
|
+
concurrency: number,
|
|
141
|
+
fn: (item: T) => Promise<R>,
|
|
142
|
+
): Promise<R[]> {
|
|
143
|
+
const results = new Array<R>(items.length)
|
|
144
|
+
let next = 0
|
|
145
|
+
async function worker(): Promise<void> {
|
|
146
|
+
for (let index = next++; index < items.length; index = next++) {
|
|
147
|
+
results[index] = await fn(items[index])
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker()))
|
|
151
|
+
return results
|
|
152
|
+
}
|
|
153
|
+
|
|
96
154
|
export async function packAsset(zipFilter: string, opts: PackAssetOptions): Promise<PackedAsset> {
|
|
97
155
|
const cwd = opts.cwd ?? process.cwd()
|
|
98
156
|
|
|
@@ -131,6 +189,8 @@ export async function packAsset(zipFilter: string, opts: PackAssetOptions): Prom
|
|
|
131
189
|
const { kept: withoutIgnored, gitignored } = applyGitignore(sourceFiles)
|
|
132
190
|
let files = withoutIgnored
|
|
133
191
|
let finalAssetDependencies = assetDependencies
|
|
192
|
+
let skippedDependencies: string[] = []
|
|
193
|
+
let missingDependencyFiles: Array<{ name: string; path: string }> = []
|
|
134
194
|
if (policy.omitUnchangedInstalledFiles) {
|
|
135
195
|
const omitted = await withoutInstalledDependencyFiles(
|
|
136
196
|
withoutIgnored,
|
|
@@ -139,6 +199,8 @@ export async function packAsset(zipFilter: string, opts: PackAssetOptions): Prom
|
|
|
139
199
|
)
|
|
140
200
|
files = omitted.kept
|
|
141
201
|
finalAssetDependencies = omitted.assetDependencies
|
|
202
|
+
skippedDependencies = omitted.skippedDependencies
|
|
203
|
+
missingDependencyFiles = omitted.missingDependencyFiles
|
|
142
204
|
}
|
|
143
205
|
|
|
144
206
|
const originalCount = Object.keys(sourceFiles).length
|
|
@@ -153,6 +215,8 @@ export async function packAsset(zipFilter: string, opts: PackAssetOptions): Prom
|
|
|
153
215
|
skillDependencies: opts.dependencies.skillDependencies,
|
|
154
216
|
omittedUnchangedInstalledFiles: finalCount < afterIgnoreCount,
|
|
155
217
|
gitignoredFiles: gitignored,
|
|
218
|
+
skippedDependencies,
|
|
219
|
+
missingDependencyFiles,
|
|
156
220
|
}
|
|
157
221
|
}
|
|
158
222
|
|
|
@@ -207,7 +271,7 @@ async function packDirectory(dir: string, opts: PackAssetOptions): Promise<Packe
|
|
|
207
271
|
files[relative] = content
|
|
208
272
|
continue
|
|
209
273
|
}
|
|
210
|
-
const hash =
|
|
274
|
+
const hash = md5(content)
|
|
211
275
|
hashesByPath.set(relative, hash)
|
|
212
276
|
if (dependencyHashes.has(hash)) {
|
|
213
277
|
deferred.push(relative)
|
|
@@ -240,57 +304,16 @@ async function packDirectory(dir: string, opts: PackAssetOptions): Promise<Packe
|
|
|
240
304
|
skillDependencies: opts.dependencies.skillDependencies,
|
|
241
305
|
omittedUnchangedInstalledFiles: layout.omit.size > 0,
|
|
242
306
|
gitignoredFiles: walk.ignored,
|
|
307
|
+
skippedDependencies: policy.omitUnchangedInstalledFiles
|
|
308
|
+
? Object.keys(assetDependencies).filter((name) => !index.fetched.has(name))
|
|
309
|
+
: [],
|
|
310
|
+
missingDependencyFiles: layout.missing.map((file) => ({
|
|
311
|
+
name: file.name,
|
|
312
|
+
path: file.canonicalPath,
|
|
313
|
+
})),
|
|
243
314
|
}
|
|
244
315
|
}
|
|
245
316
|
|
|
246
|
-
// Walk a directory applying git semantics as we descend: each directory's `.gitignore` governs its
|
|
247
|
-
// own subtree, an ignored directory is pruned without entering it, and `.git`/`node_modules` never
|
|
248
|
-
// ship. Returns kept files as posix paths relative to the root, plus the count of ignored entries.
|
|
249
|
-
async function walkWithGitignore(root: string): Promise<{ kept: string[]; ignored: number }> {
|
|
250
|
-
const kept: string[] = []
|
|
251
|
-
let ignored = 0
|
|
252
|
-
|
|
253
|
-
async function descend(dir: string, matchers: GitignoreMatcher[]): Promise<void> {
|
|
254
|
-
const scoped = [...matchers]
|
|
255
|
-
const gitignore = await maybeStat(path.join(dir, GITIGNORE_FILENAME))
|
|
256
|
-
if (gitignore?.isFile()) {
|
|
257
|
-
const prefix = toPosix(path.relative(root, dir))
|
|
258
|
-
scoped.push({
|
|
259
|
-
dir: prefix ? `${prefix}/` : '',
|
|
260
|
-
filter: ignore().add(await fs.readFile(path.join(dir, GITIGNORE_FILENAME), 'utf8')),
|
|
261
|
-
})
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
const entries = await fs.readdir(dir, { withFileTypes: true })
|
|
265
|
-
for (const entry of entries) {
|
|
266
|
-
if (ALWAYS_IGNORED_DIRS.has(entry.name)) continue
|
|
267
|
-
const relative = toPosix(path.relative(root, path.join(dir, entry.name)))
|
|
268
|
-
const candidate = entry.isDirectory() ? `${relative}/` : relative
|
|
269
|
-
if (isGitignored(candidate, scoped)) {
|
|
270
|
-
ignored += 1
|
|
271
|
-
continue
|
|
272
|
-
}
|
|
273
|
-
if (entry.isDirectory()) await descend(path.join(dir, entry.name), scoped)
|
|
274
|
-
else if (entry.isFile()) kept.push(relative)
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
await descend(root, [])
|
|
279
|
-
return { kept, ignored }
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
function toPosix(p: string): string {
|
|
283
|
-
return p.split(path.sep).join('/')
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
const GITIGNORE_FILENAME = '.gitignore'
|
|
287
|
-
|
|
288
|
-
// Directories that never belong in a published asset — VCS/tooling metadata, reinstallable
|
|
289
|
-
// dependencies, and the market's own state dir (`.drawcall`, holding the install mutex). Excluded
|
|
290
|
-
// structurally in every pack path, regardless of any `.gitignore`, so a re-uploaded asset never
|
|
291
|
-
// ships them.
|
|
292
|
-
const ALWAYS_IGNORED_DIRS = new Set(['.git', 'node_modules', '.drawcall'])
|
|
293
|
-
|
|
294
317
|
function hasAlwaysIgnoredSegment(posixPath: string): boolean {
|
|
295
318
|
return posixPath.split('/').some((segment) => ALWAYS_IGNORED_DIRS.has(segment))
|
|
296
319
|
}
|
|
@@ -318,12 +341,6 @@ function applyGitignore(files: Record<string, Uint8Array>): {
|
|
|
318
341
|
return { kept, gitignored }
|
|
319
342
|
}
|
|
320
343
|
|
|
321
|
-
interface GitignoreMatcher {
|
|
322
|
-
/** Directory the `.gitignore` governs: '' for the zip root, or e.g. 'sub/'. */
|
|
323
|
-
dir: string
|
|
324
|
-
filter: ReturnType<typeof ignore>
|
|
325
|
-
}
|
|
326
|
-
|
|
327
344
|
function gitignoreMatchers(files: Record<string, Uint8Array>): GitignoreMatcher[] {
|
|
328
345
|
const matchers: GitignoreMatcher[] = []
|
|
329
346
|
for (const [name, content] of Object.entries(files)) {
|
|
@@ -334,16 +351,6 @@ function gitignoreMatchers(files: Record<string, Uint8Array>): GitignoreMatcher[
|
|
|
334
351
|
return matchers
|
|
335
352
|
}
|
|
336
353
|
|
|
337
|
-
function isGitignored(name: string, matchers: GitignoreMatcher[]): boolean {
|
|
338
|
-
// fflate keeps directory entries with a trailing slash; the `ignore` lib matches directory patterns
|
|
339
|
-
// (`dist/`) against exactly that form, so the path is passed through unchanged.
|
|
340
|
-
return matchers.some((matcher) => {
|
|
341
|
-
if (!name.startsWith(matcher.dir)) return false
|
|
342
|
-
const relative = name.slice(matcher.dir.length)
|
|
343
|
-
return relative.length > 0 && matcher.filter.ignores(relative)
|
|
344
|
-
})
|
|
345
|
-
}
|
|
346
|
-
|
|
347
354
|
export function parsePackDependencies(specs: PackDependencySpecs): ParsedPackDependencies {
|
|
348
355
|
return {
|
|
349
356
|
npmDependencies: parseVersionedDeps(specs.npm ?? [], 'npm'),
|
|
@@ -451,16 +458,26 @@ async function withoutInstalledDependencyFiles(
|
|
|
451
458
|
files: Record<string, Uint8Array>,
|
|
452
459
|
assetDependencies: AssetDependencies,
|
|
453
460
|
fetchFileManifest: FetchFileManifest | undefined,
|
|
454
|
-
): Promise<{
|
|
461
|
+
): Promise<{
|
|
462
|
+
kept: Record<string, Uint8Array>
|
|
463
|
+
assetDependencies: AssetDependencies
|
|
464
|
+
skippedDependencies: string[]
|
|
465
|
+
missingDependencyFiles: Array<{ name: string; path: string }>
|
|
466
|
+
}> {
|
|
455
467
|
const index = await dependencyFileIndex(assetDependencies, fetchFileManifest)
|
|
456
|
-
|
|
468
|
+
const skippedDependencies = Object.keys(assetDependencies).filter(
|
|
469
|
+
(name) => !index.fetched.has(name),
|
|
470
|
+
)
|
|
471
|
+
if (index.files.length === 0) {
|
|
472
|
+
return { kept: files, assetDependencies, skippedDependencies, missingDependencyFiles: [] }
|
|
473
|
+
}
|
|
457
474
|
|
|
458
475
|
const hashesByPath = new Map<string, string>()
|
|
459
476
|
const entryByNormalized = new Map<string, string>()
|
|
460
477
|
for (const [file, content] of Object.entries(files)) {
|
|
461
478
|
const normalizedPath = normalizedZipPath(file)
|
|
462
479
|
if (!normalizedPath || normalizedPath.endsWith('/')) continue
|
|
463
|
-
hashesByPath.set(normalizedPath,
|
|
480
|
+
hashesByPath.set(normalizedPath, md5(content))
|
|
464
481
|
entryByNormalized.set(normalizedPath, file)
|
|
465
482
|
}
|
|
466
483
|
|
|
@@ -471,20 +488,28 @@ async function withoutInstalledDependencyFiles(
|
|
|
471
488
|
const kept = Object.fromEntries(
|
|
472
489
|
Object.entries(files).filter(([file]) => !omittedEntries.has(file)),
|
|
473
490
|
)
|
|
474
|
-
return {
|
|
491
|
+
return {
|
|
492
|
+
kept,
|
|
493
|
+
assetDependencies: layout.assetDependencies,
|
|
494
|
+
skippedDependencies,
|
|
495
|
+
missingDependencyFiles: layout.missing.map((file) => ({
|
|
496
|
+
name: file.name,
|
|
497
|
+
path: file.canonicalPath,
|
|
498
|
+
})),
|
|
499
|
+
}
|
|
475
500
|
}
|
|
476
501
|
|
|
477
|
-
interface DependencyFile {
|
|
502
|
+
export interface DependencyFile {
|
|
478
503
|
name: string
|
|
479
504
|
canonicalPath: string
|
|
480
505
|
}
|
|
481
506
|
|
|
482
|
-
interface DependencyFileEntry extends DependencyFile {
|
|
483
|
-
/**
|
|
507
|
+
export interface DependencyFileEntry extends DependencyFile {
|
|
508
|
+
/** md5 hex of the file's canonical content — its R2 etag, from the file index. */
|
|
484
509
|
hash: string
|
|
485
510
|
}
|
|
486
511
|
|
|
487
|
-
interface DependencyFileIndex {
|
|
512
|
+
export interface DependencyFileIndex {
|
|
488
513
|
files: DependencyFileEntry[]
|
|
489
514
|
/** Dependencies whose manifest was fetched — only their aliases are re-derived. */
|
|
490
515
|
fetched: Set<string>
|
|
@@ -496,7 +521,7 @@ function emptyDependencyFileIndex(): DependencyFileIndex {
|
|
|
496
521
|
|
|
497
522
|
/**
|
|
498
523
|
* The canonical content hash of every file belonging to a declared asset dependency, fetched from
|
|
499
|
-
* the server (
|
|
524
|
+
* the server (the version file indexes' etags). Returns an empty index when there's nothing to omit or no way
|
|
500
525
|
* to fetch (offline / no client) — pack then keeps all files rather than guessing.
|
|
501
526
|
*/
|
|
502
527
|
async function dependencyFileIndex(
|
|
@@ -547,7 +572,7 @@ interface DependencyLayout {
|
|
|
547
572
|
* user's own content. Stale aliases drop: a renamed file that was since modified ships as user
|
|
548
573
|
* content and reinstall restores the canonical path.
|
|
549
574
|
*/
|
|
550
|
-
function resolveDependencyLayout(
|
|
575
|
+
export function resolveDependencyLayout(
|
|
551
576
|
declared: AssetDependencies,
|
|
552
577
|
index: DependencyFileIndex,
|
|
553
578
|
hashesByPath: Map<string, string>,
|
|
@@ -560,13 +585,18 @@ function resolveDependencyLayout(
|
|
|
560
585
|
|
|
561
586
|
const omit = new Set<string>()
|
|
562
587
|
const missing: DependencyFile[] = []
|
|
563
|
-
const aliases: Record<string,
|
|
588
|
+
const aliases: Record<string, AssetDependencyAlias> = {}
|
|
564
589
|
for (const file of index.files) {
|
|
565
590
|
const declaredValue = declared[file.name]
|
|
566
591
|
const declaredAlias =
|
|
567
592
|
declaredValue === undefined
|
|
568
593
|
? undefined
|
|
569
594
|
: assetDependencyAlias(declaredValue)[file.canonicalPath]
|
|
595
|
+
// A tombstone is a deliberate deletion: not missing, nothing to omit, and it survives as-is.
|
|
596
|
+
if (declaredAlias === false) {
|
|
597
|
+
aliases[file.name] = { ...(aliases[file.name] ?? {}), [file.canonicalPath]: false }
|
|
598
|
+
continue
|
|
599
|
+
}
|
|
570
600
|
const candidates = (pathsByHash.get(file.hash) ?? []).filter((p) => !omit.has(p))
|
|
571
601
|
if (candidates.length === 0) {
|
|
572
602
|
missing.push({ name: file.name, canonicalPath: file.canonicalPath })
|
|
@@ -632,7 +662,7 @@ export async function syncAssetDependencies(
|
|
|
632
662
|
for (const relative of walk.kept) {
|
|
633
663
|
hashesByPath.set(
|
|
634
664
|
relative,
|
|
635
|
-
|
|
665
|
+
md5(new Uint8Array(await fs.readFile(path.join(projectRoot, relative)))),
|
|
636
666
|
)
|
|
637
667
|
}
|
|
638
668
|
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Directory walking with git semantics, shared by pack/sync (dependency-layout resolution) and
|
|
3
|
+
* install (rename adoption): each directory's `.gitignore` governs its own subtree, an ignored
|
|
4
|
+
* directory is pruned without entering it, and `.git`/`node_modules`/`.drawcall` never count.
|
|
5
|
+
*/
|
|
6
|
+
import * as fs from 'fs/promises'
|
|
7
|
+
import * as path from 'path'
|
|
8
|
+
import ignore from 'ignore'
|
|
9
|
+
|
|
10
|
+
export const GITIGNORE_FILENAME = '.gitignore'
|
|
11
|
+
|
|
12
|
+
// Directories that never belong in a published asset — VCS/tooling metadata, reinstallable
|
|
13
|
+
// dependencies, and the market's own state dir (`.drawcall`, holding the install mutex). Excluded
|
|
14
|
+
// structurally in every walk, regardless of any `.gitignore`.
|
|
15
|
+
export const ALWAYS_IGNORED_DIRS = new Set(['.git', 'node_modules', '.drawcall'])
|
|
16
|
+
|
|
17
|
+
export interface GitignoreMatcher {
|
|
18
|
+
/** Directory the `.gitignore` governs: '' for the root, or e.g. 'sub/'. */
|
|
19
|
+
dir: string
|
|
20
|
+
filter: ReturnType<typeof ignore>
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function isGitignored(name: string, matchers: GitignoreMatcher[]): boolean {
|
|
24
|
+
// Directory candidates carry a trailing slash; the `ignore` lib matches directory patterns
|
|
25
|
+
// (`dist/`) against exactly that form, so the path is passed through unchanged.
|
|
26
|
+
return matchers.some((matcher) => {
|
|
27
|
+
if (!name.startsWith(matcher.dir)) return false
|
|
28
|
+
const relative = name.slice(matcher.dir.length)
|
|
29
|
+
return relative.length > 0 && matcher.filter.ignores(relative)
|
|
30
|
+
})
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Walk a directory tree; returns kept files as posix paths relative to the root, plus the count
|
|
34
|
+
* of ignored entries. */
|
|
35
|
+
export async function walkWithGitignore(
|
|
36
|
+
root: string,
|
|
37
|
+
): Promise<{ kept: string[]; ignored: number }> {
|
|
38
|
+
const kept: string[] = []
|
|
39
|
+
let ignored = 0
|
|
40
|
+
|
|
41
|
+
async function descend(dir: string, matchers: GitignoreMatcher[]): Promise<void> {
|
|
42
|
+
const scoped = [...matchers]
|
|
43
|
+
const gitignore = await maybeStat(path.join(dir, GITIGNORE_FILENAME))
|
|
44
|
+
if (gitignore?.isFile()) {
|
|
45
|
+
const prefix = toPosix(path.relative(root, dir))
|
|
46
|
+
scoped.push({
|
|
47
|
+
dir: prefix ? `${prefix}/` : '',
|
|
48
|
+
filter: ignore().add(await fs.readFile(path.join(dir, GITIGNORE_FILENAME), 'utf8')),
|
|
49
|
+
})
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const entries = await fs.readdir(dir, { withFileTypes: true })
|
|
53
|
+
for (const entry of entries) {
|
|
54
|
+
if (ALWAYS_IGNORED_DIRS.has(entry.name)) continue
|
|
55
|
+
const relative = toPosix(path.relative(root, path.join(dir, entry.name)))
|
|
56
|
+
const candidate = entry.isDirectory() ? `${relative}/` : relative
|
|
57
|
+
if (isGitignored(candidate, scoped)) {
|
|
58
|
+
ignored += 1
|
|
59
|
+
continue
|
|
60
|
+
}
|
|
61
|
+
if (entry.isDirectory()) await descend(path.join(dir, entry.name), scoped)
|
|
62
|
+
else if (entry.isFile()) kept.push(relative)
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
await descend(root, [])
|
|
67
|
+
return { kept, ignored }
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** The nearest ancestor (including start) holding a package.json — the project root. */
|
|
71
|
+
export async function findInstallRoot(cwd: string = process.cwd()): Promise<string> {
|
|
72
|
+
const start = path.resolve(cwd)
|
|
73
|
+
|
|
74
|
+
for (let dir = start; ; dir = path.dirname(dir)) {
|
|
75
|
+
if ((await maybeStat(path.join(dir, 'package.json')))?.isFile()) {
|
|
76
|
+
return dir
|
|
77
|
+
}
|
|
78
|
+
if (path.dirname(dir) === dir) {
|
|
79
|
+
return start
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function toPosix(p: string): string {
|
|
85
|
+
return p.split(path.sep).join('/')
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
async function maybeStat(file: string) {
|
|
89
|
+
try {
|
|
90
|
+
return await fs.stat(file)
|
|
91
|
+
} catch (error) {
|
|
92
|
+
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return null
|
|
93
|
+
throw error
|
|
94
|
+
}
|
|
95
|
+
}
|
package/src/resolve.ts
CHANGED
|
@@ -14,6 +14,7 @@ import type { MarketClient } from './client.js'
|
|
|
14
14
|
import type { AssetExactResult } from './contract.js'
|
|
15
15
|
import {
|
|
16
16
|
assetDependencyAlias,
|
|
17
|
+
type AssetDependencyAlias,
|
|
17
18
|
assetDependencyRange,
|
|
18
19
|
semverSchema,
|
|
19
20
|
type AssetDependencies,
|
|
@@ -38,7 +39,7 @@ export interface ResolveResult {
|
|
|
38
39
|
* path the dependent's project keeps the file at. Install writes files through this map (a
|
|
39
40
|
* template whose author renamed a dependency's files reinstalls them at the renamed locations).
|
|
40
41
|
*/
|
|
41
|
-
assetAliases?: Record<string,
|
|
42
|
+
assetAliases?: Record<string, AssetDependencyAlias>
|
|
42
43
|
}
|
|
43
44
|
|
|
44
45
|
interface AssetRequest {
|
|
@@ -221,8 +222,8 @@ function mergeSkillDependencies(assets: ResolvedAsset[]): Record<string, string>
|
|
|
221
222
|
* dependents rarely alias the same file; when they do, the first resolved asset wins — a
|
|
222
223
|
* deterministic choice that keeps installs reproducible.
|
|
223
224
|
*/
|
|
224
|
-
function mergeAssetAliases(assets: ResolvedAsset[]): Record<string,
|
|
225
|
-
const merged: Record<string,
|
|
225
|
+
function mergeAssetAliases(assets: ResolvedAsset[]): Record<string, AssetDependencyAlias> {
|
|
226
|
+
const merged: Record<string, AssetDependencyAlias> = {}
|
|
226
227
|
for (const asset of assets) {
|
|
227
228
|
for (const [dependencyName, value] of Object.entries(asset.assetDependencies)) {
|
|
228
229
|
const alias = assetDependencyAlias(value)
|
package/src/schemas.ts
CHANGED
|
@@ -44,7 +44,15 @@ export const npmDependenciesSchema = z.record(z.string(), z.string()).default({}
|
|
|
44
44
|
// after renaming/moving it. Install writes each file at `alias[path] ?? path`; pack records the
|
|
45
45
|
// aliases it detects by content hash, so renamed installed files are omitted from packs and
|
|
46
46
|
// restored to their renamed locations on reinstall.
|
|
47
|
-
|
|
47
|
+
//
|
|
48
|
+
// `false` is a deletion tombstone (precedent: package.json's `browser` field): "do not install
|
|
49
|
+
// this file here". Install skips the file, sync/pack treat it as intentionally absent, and the
|
|
50
|
+
// entry survives untouched. Tombstones are authored, never inferred — a missing file is ambiguous
|
|
51
|
+
// (it also covers in-place edits), so nothing auto-tombstones.
|
|
52
|
+
export const assetDependencyAliasSchema = z.record(
|
|
53
|
+
z.string(),
|
|
54
|
+
z.union([z.string(), z.literal(false)]),
|
|
55
|
+
)
|
|
48
56
|
|
|
49
57
|
export const assetDependencyValueSchema = z.union([
|
|
50
58
|
z.string(),
|
|
@@ -59,14 +67,16 @@ export function assetDependencyRange(value: AssetDependencyValue): string {
|
|
|
59
67
|
return typeof value === 'string' ? value : value.version
|
|
60
68
|
}
|
|
61
69
|
|
|
62
|
-
export
|
|
70
|
+
export type AssetDependencyAlias = Record<string, string | false>
|
|
71
|
+
|
|
72
|
+
export function assetDependencyAlias(value: AssetDependencyValue): AssetDependencyAlias {
|
|
63
73
|
return typeof value === 'string' ? {} : (value.alias ?? {})
|
|
64
74
|
}
|
|
65
75
|
|
|
66
76
|
/** The slimmest value expressing `range` + `alias`: a plain string when there are no aliases. */
|
|
67
77
|
export function assetDependencyValue(
|
|
68
78
|
range: string,
|
|
69
|
-
alias:
|
|
79
|
+
alias: AssetDependencyAlias,
|
|
70
80
|
): AssetDependencyValue {
|
|
71
81
|
return Object.keys(alias).length === 0 ? range : { version: range, alias }
|
|
72
82
|
}
|
|
@@ -133,17 +143,20 @@ export const assetFilesSchema = z.object({
|
|
|
133
143
|
key: accessKeySchema.optional(),
|
|
134
144
|
})
|
|
135
145
|
|
|
136
|
-
|
|
146
|
+
// Input of `asset.versions`: every published version of an asset (same credential rules as reads).
|
|
147
|
+
export const assetVersionsSchema = z.object({
|
|
137
148
|
name: assetNameSchema,
|
|
138
|
-
|
|
149
|
+
key: accessKeySchema.optional(),
|
|
139
150
|
})
|
|
140
151
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
export
|
|
152
|
+
export interface AssetVersionListing {
|
|
153
|
+
version: string
|
|
154
|
+
approved: boolean
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export interface AssetVersionsResult {
|
|
158
|
+
versions: AssetVersionListing[]
|
|
159
|
+
}
|
|
147
160
|
|
|
148
161
|
// A phone-photo data URI is a few MB; this bounds worker and model payloads while staying far
|
|
149
162
|
// above what providers keep (images are downscaled upstream).
|
package/src/v1/contract.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { oc } from '@orpc/contract'
|
|
2
2
|
import { z } from 'zod'
|
|
3
|
-
import type {
|
|
3
|
+
import type { AssetVersionsResult } from '../schemas.js'
|
|
4
4
|
import type {
|
|
5
5
|
AssetExactResult,
|
|
6
6
|
AssetFileEntry,
|
|
@@ -16,8 +16,8 @@ import {
|
|
|
16
16
|
agentStartSchema,
|
|
17
17
|
agentStatusSchema,
|
|
18
18
|
assetFilesSchema,
|
|
19
|
+
assetVersionsSchema,
|
|
19
20
|
exactAssetSchema,
|
|
20
|
-
fileManifestSchema,
|
|
21
21
|
generateAssetSchema,
|
|
22
22
|
generateJobStatusSchema,
|
|
23
23
|
generateResponseSchema,
|
|
@@ -76,10 +76,12 @@ export const contract = {
|
|
|
76
76
|
.input(assetFilesSchema)
|
|
77
77
|
.output(z.custom<{ files: AssetFileEntry[] }>()),
|
|
78
78
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
.
|
|
79
|
+
// Every published version of an asset. Feeds multi-version content matching: without a local
|
|
80
|
+
// lockfile the installed version is unknown, so clients merge hashes across the range.
|
|
81
|
+
versions: oc
|
|
82
|
+
.route({ method: 'GET', path: '/assets/{name}/versions' })
|
|
83
|
+
.input(assetVersionsSchema)
|
|
84
|
+
.output(z.custom<AssetVersionsResult>()),
|
|
83
85
|
|
|
84
86
|
uploadZip: oc
|
|
85
87
|
.route({ method: 'PUT', path: '/assets/{name}/{version}' })
|