@series-inc/stowkit-cli 0.6.16 → 0.6.18

package/dist/publish.js

@@ -0,0 +1,399 @@
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import { readProjectConfig, scanDirectory } from './node-fs.js';
+import { readAssetsPackage, initAssetsPackage, createEmptyRegistry, } from './assets-package.js';
+import { createGCSClient } from './gcs.js';
+import { readStowmeta } from './app/stowmeta-io.js';
+// ─── Build full dependency graph from srcArtDir ──────────────────────────────
+async function scanAssets(srcArtDir, scan, verbose) {
+    const assets = [];
+    // Index source files by relative path for size lookup
+    const sourceByPath = new Map();
+    for (const f of scan.sourceFiles)
+        sourceByPath.set(f.relativePath, f);
+    // Pass 1: read all stowmeta files, collect assets with raw file-path deps
+    for (const metaFile of scan.metaFiles) {
+        const assetPath = metaFile.relativePath.replace(/\.stowmeta$/, '');
+        const meta = await readStowmeta(srcArtDir, assetPath);
+        if (!meta)
+            continue;
+        if (meta.type === 'glbContainer') {
+            const container = meta;
+            for (const child of container.children) {
+                if (child.excluded)
+                    continue;
+                const childFiles = [assetPath, metaFile.relativePath];
+                const rawDeps = [];
+                if (child.childType === 'staticMesh' || child.childType === 'skinnedMesh') {
+                    if (child.materialOverrides) {
+                        for (const ref of Object.values(child.materialOverrides)) {
+                            if (ref)
+                                rawDeps.push(ref);
+                        }
+                    }
+                }
+                if (child.childType === 'materialSchema' && child.materialConfig) {
+                    for (const prop of child.materialConfig.properties) {
+                        if (prop.textureAsset)
+                            rawDeps.push(prop.textureAsset);
+                    }
+                }
+                if (child.childType === 'animationClip' && child.targetMeshId) {
+                    rawDeps.push(child.targetMeshId);
+                }
+                assets.push({
+                    stringId: child.stringId,
+                    type: child.childType,
+                    sourceFile: assetPath,
+                    files: childFiles,
+                    size: sourceByPath.get(assetPath)?.size ?? 0,
+                    tags: child.tags ?? [],
+                    dependencies: [], // resolved in pass 2
+                    rawDeps,
+                });
+            }
+            continue;
+        }
+        if (meta.excluded)
+            continue;
+        const files = [assetPath, metaFile.relativePath];
+        const rawDeps = [];
+        // Mesh → material refs (file paths like "Materials/M_Sea_Floor.stowmat")
+        if (meta.type === 'staticMesh' || meta.type === 'skinnedMesh') {
+            for (const ref of Object.values(meta.materialOverrides)) {
+                if (ref)
+                    rawDeps.push(ref);
+            }
+        }
+        // Animation → target mesh (stringId or file path)
+        if (meta.type === 'animationClip' && meta.targetMeshId) {
+            rawDeps.push(meta.targetMeshId);
+        }
+        // Material schema (.stowmat is the source file itself) → texture refs
+        // The source file IS the .stowmat, and it's already a source file in the scan.
+        // We need to read it to find texture dependencies.
+        if (meta.type === 'materialSchema' && assetPath.endsWith('.stowmat')) {
+            try {
+                const matText = await fs.readFile(path.join(srcArtDir, assetPath), 'utf-8');
+                const mat = JSON.parse(matText);
+                if (mat.properties) {
+                    for (const prop of mat.properties) {
+                        if (prop.textureAsset)
+                            rawDeps.push(prop.textureAsset);
+                    }
+                }
+            }
+            catch { /* not a valid stowmat */ }
+        }
+        assets.push({
+            stringId: meta.stringId,
+            type: meta.type,
+            sourceFile: assetPath,
+            files,
+            size: sourceByPath.get(assetPath)?.size ?? 0,
+            tags: meta.tags ?? [],
+            dependencies: [], // resolved in pass 2
+            rawDeps,
+            ...(meta.type === 'texture' && meta.filtering ? { filtering: meta.filtering } : {}),
+        });
+    }
+    // Pass 2: resolve raw file-path deps → stringIds
+    // Build lookup: source file path → stringId
+    const pathToStringId = new Map();
+    for (const asset of assets) {
+        pathToStringId.set(asset.sourceFile, asset.stringId);
+    }
+    for (const asset of assets) {
+        for (const ref of asset.rawDeps) {
+            // Try as file path first
+            const resolved = pathToStringId.get(ref);
+            if (resolved) {
+                asset.dependencies.push(resolved);
+                continue;
+            }
+            // Maybe it's already a stringId
+            const byStringId = assets.find(a => a.stringId === ref);
+            if (byStringId) {
+                asset.dependencies.push(ref);
+                continue;
+            }
+            // Unresolved — keep the raw ref so warning still fires
+            asset.dependencies.push(ref);
+        }
+        if (asset.dependencies.length > 0) {
+            verbose(`${asset.stringId} [${asset.type}] → ${asset.dependencies.join(', ')}`);
+        }
+    }
+    return assets;
+}
+// ─── Main Publish Function ───────────────────────────────────────────────────
+export async function publishPackage(projectDir, opts = {}) {
+    const log = (msg) => console.log(msg);
+    const vlog = (msg) => { if (opts.verbose)
+        console.log(`  ${msg}`); };
+    // Step 1: Read project config
+    const project = await readProjectConfig(projectDir);
+    log(`Project: ${project.projectName}`);
+    // Step 2: Read or auto-create assets-package.json
+    let pkg = await readAssetsPackage(projectDir);
+    if (!pkg) {
+        log('No assets-package.json found — creating default...');
+        pkg = await initAssetsPackage(projectDir, { name: project.projectName });
+        log(`Created assets-package.json (name: ${pkg.name}, version: ${pkg.version})`);
+    }
+    // Step 3: Resolve bucket
+    const bucketUri = opts.bucket
+        ?? pkg.bucket
+        ?? process.env.STOWKIT_BUCKET
+        ?? 'gs://venus-shared-assets-test';
+    log(`Bucket: ${bucketUri}`);
+    // Step 4: Scan srcArtDir
+    const scan = await scanDirectory(project.srcArtDir);
+    if (scan.sourceFiles.length === 0) {
+        throw new Error(`No source assets found in ${project.srcArtDir}. Nothing to publish.`);
+    }
+    // Step 5: Build asset list with full dependency graph
+    const scannedAssets = await scanAssets(project.srcArtDir, scan, vlog);
+    log(`Assets: ${scannedAssets.length}`);
+    // Validate: reject duplicate stringIds
+    const idCounts = new Map();
+    for (const asset of scannedAssets) {
+        const files = idCounts.get(asset.stringId) ?? [];
+        files.push(asset.sourceFile);
+        idCounts.set(asset.stringId, files);
+    }
+    const duplicates = [...idCounts.entries()].filter(([, files]) => files.length > 1);
+    if (duplicates.length > 0) {
+        const lines = duplicates.map(([id, files]) => `  "${id}" used by:\n${files.map(f => `    - ${f}`).join('\n')}`);
+        throw new Error(`Cannot publish: ${duplicates.length} duplicate stringId(s) found. Each asset must have a unique stringId.\n${lines.join('\n')}`);
+    }
+    // Validate: warn about unresolved dependencies
+    const knownIds = new Set(scannedAssets.map(a => a.stringId));
+    for (const asset of scannedAssets) {
+        for (const dep of asset.dependencies) {
+            if (!knownIds.has(dep)) {
+                console.warn(`  Warning: ${asset.stringId} depends on "${dep}" which was not found`);
+            }
+        }
+    }
+    // Collect all unique files to upload
+    const allFiles = new Set();
+    for (const asset of scannedAssets) {
+        for (const f of asset.files)
+            allFiles.add(f);
+    }
+    for (const f of scan.sourceFiles)
+        allFiles.add(f.relativePath);
+    for (const f of scan.metaFiles)
+        allFiles.add(f.relativePath);
+    for (const f of scan.matFiles)
+        allFiles.add(f.relativePath);
+    const filesToUpload = [...allFiles].sort();
+    let totalSize = 0;
+    for (const rel of filesToUpload) {
+        const stat = await fs.stat(path.join(project.srcArtDir, rel)).catch(() => null);
+        totalSize += stat?.size ?? 0;
+    }
+    log(`Files to upload: ${filesToUpload.length} (${formatBytes(totalSize)})`);
+    log(`  ${scan.sourceFiles.length} source, ${scan.metaFiles.length} stowmeta, ${scan.matFiles.length} stowmat`);
+    // Step 6: Build registry assets
+    const thumbMap = opts.thumbnails ?? {};
+    const registryAssets = scannedAssets.map(a => {
+        const thumb = thumbMap[a.stringId];
+        return {
+            stringId: a.stringId,
+            type: a.type,
+            file: a.sourceFile,
+            files: a.files,
+            size: a.size,
+            tags: a.tags,
+            dependencies: a.dependencies,
+            ...(thumb ? { thumbnail: true, thumbnailFormat: thumb.format } : {}),
+            ...(a.filtering && a.filtering !== 'linear' ? { filtering: a.filtering } : {}),
+        };
+    });
+    const versionEntry = {
+        publishedAt: new Date().toISOString(),
+        fileCount: filesToUpload.length,
+        totalSize,
+        assets: registryAssets,
+    };
+    // Dry run — print summary and exit
+    if (opts.dryRun) {
+        log('\n--- DRY RUN ---');
+        log(`Would publish ${pkg.name}@${pkg.version} to ${bucketUri}`);
+        log(`Would upload ${filesToUpload.length} files (${formatBytes(totalSize)})`);
+        log(`\nAssets (${registryAssets.length}):`);
+        for (const a of registryAssets) {
+            const deps = a.dependencies.length > 0 ? ` → [${a.dependencies.join(', ')}]` : '';
+            const tags = a.tags.length > 0 ? ` [${a.tags.join(', ')}]` : '';
+            log(`  [${a.type}] ${a.stringId} (${a.file}, ${formatBytes(a.size)})${tags}${deps}`);
+        }
+        // Demo transitive resolution
+        const { resolveTransitiveDeps } = await import('./assets-package.js');
+        const example = registryAssets.find(a => a.dependencies.length > 0);
+        if (example) {
+            log(`\nExample: pulling "${example.stringId}" would also grab:`);
+            const resolved = resolveTransitiveDeps([example.stringId], registryAssets);
+            for (const id of resolved) {
+                const asset = registryAssets.find(a => a.stringId === id);
+                log(`  ${id === example.stringId ? '→' : '  +'} ${id} [${asset?.type ?? '?'}]`);
+            }
+        }
+        log('\nNo files were uploaded.');
+        return {
+            ok: true,
+            packageName: pkg.name,
+            version: pkg.version,
+            assetCount: registryAssets.length,
+            fileCount: filesToUpload.length,
+            thumbnailCount: Object.keys(thumbMap).length,
+        };
+    }
+    // Step 7: Create GCS client and auth
+    const emitProgress = opts.onProgress ?? (() => { });
+    const totalUploads = filesToUpload.length + Object.keys(thumbMap).length + 2; // +2 for assets-package.json and registry.json
+    let totalDone = 0;
+    log('Authenticating with GCS...');
+    emitProgress({ phase: 'auth', done: 0, total: totalUploads, message: 'Authenticating with GCS...' });
+    const gcs = await createGCSClient(projectDir, bucketUri);
+    // Step 8: Download current registry
+    log('Fetching registry...');
+    const registryResult = await gcs.downloadWithGeneration('registry.json');
+    let registry;
+    let generation;
+    if (registryResult) {
+        registry = JSON.parse(registryResult.data);
+        generation = registryResult.generation;
+        vlog(`Registry loaded (generation: ${generation})`);
+    }
+    else {
+        registry = createEmptyRegistry();
+        generation = null;
+        vlog('No existing registry — will create new one');
+    }
+    // Step 9: Check version not already published
+    const existingPkg = registry.packages[pkg.name];
+    if (existingPkg?.versions[pkg.version] && !opts.force) {
+        throw new Error(`Version ${pkg.version} of "${pkg.name}" is already published. ` +
+            `Bump the version in assets-package.json or use --force to overwrite.`);
+    }
+    // Step 10: Upload all files (concurrent)
+    const uploadPrefix = `packages/${pkg.name}/${pkg.version}`;
+    let uploaded = 0;
+    const UPLOAD_CONCURRENCY = 10;
+    emitProgress({ phase: 'files', done: 0, total: totalUploads, message: `Uploading ${filesToUpload.length} files...` });
+    await runConcurrent(filesToUpload, UPLOAD_CONCURRENCY, async (relativePath) => {
+        const objectPath = `${uploadPrefix}/${relativePath}`;
+        const absPath = path.join(project.srcArtDir, relativePath);
+        const data = new Uint8Array(await fs.readFile(absPath));
+        await gcs.upload(objectPath, data, guessContentType(relativePath));
+        uploaded++;
+        totalDone++;
+        emitProgress({ phase: 'files', done: totalDone, total: totalUploads, message: `Uploading files: ${uploaded}/${filesToUpload.length}` });
+        if (opts.verbose) {
+            vlog(`[${uploaded}/${filesToUpload.length}] ${relativePath} (${formatBytes(data.length)})`);
+        }
+        else if (uploaded % 10 === 0 || uploaded === filesToUpload.length) {
+            log(`  Uploaded ${uploaded}/${filesToUpload.length} files...`);
+        }
+    });
+    // Step 11: Upload thumbnails concurrently (if provided by packer GUI)
+    let thumbnailCount = 0;
+    const thumbEntries = opts.thumbnails ? Object.entries(opts.thumbnails) : [];
+    if (thumbEntries.length > 0) {
+        emitProgress({ phase: 'thumbnails', done: totalDone, total: totalUploads, message: `Uploading thumbnails...` });
+    }
+    await runConcurrent(thumbEntries, UPLOAD_CONCURRENCY, async ([stringId, thumb]) => {
+        const binaryData = Buffer.from(thumb.data, 'base64');
+        const ext = thumb.format === 'webm' ? 'webm' : thumb.format === 'webp' ? 'webp' : 'png';
+        const mime = thumb.format === 'webm' ? 'video/webm' : thumb.format === 'webp' ? 'image/webp' : 'image/png';
+        const objectPath = `${uploadPrefix}/thumbnails/${stringId}.${ext}`;
+        await gcs.upload(objectPath, new Uint8Array(binaryData), mime);
+        thumbnailCount++;
+        totalDone++;
+        emitProgress({ phase: 'thumbnails', done: totalDone, total: totalUploads, message: `Uploading thumbnails: ${thumbnailCount}/${thumbEntries.length}` });
+    });
+    if (thumbnailCount > 0)
+        log(`  Uploaded ${thumbnailCount} thumbnails`);
+    // Step 12: Upload assets-package.json receipt copy
+    emitProgress({ phase: 'registry', done: totalDone, total: totalUploads, message: 'Uploading package manifest...' });
+    const receiptPath = `${uploadPrefix}/assets-package.json`;
+    await gcs.upload(receiptPath, JSON.stringify(pkg, null, 2), 'application/json');
+    totalDone++;
+    // Step 13: Update and upload registry
+    log('Updating registry...');
+    if (!registry.packages[pkg.name]) {
+        registry.packages[pkg.name] = {
+            description: pkg.description,
+            author: pkg.author,
+            tags: pkg.tags ?? [],
+            latest: pkg.version,
+            versions: {},
+        };
+    }
+    const regPkg = registry.packages[pkg.name];
+    regPkg.versions[pkg.version] = versionEntry;
+    regPkg.latest = pkg.version;
+    regPkg.description = pkg.description;
+    regPkg.author = pkg.author;
+    regPkg.tags = pkg.tags ?? [];
+    emitProgress({ phase: 'registry', done: totalDone, total: totalUploads, message: 'Updating registry...' });
+    await gcs.uploadWithGeneration('registry.json', JSON.stringify(registry, null, 2), generation);
+    totalDone++;
+    emitProgress({ phase: 'registry', done: totalDone, total: totalUploads, message: 'Done' });
+    log(`\nPublished ${pkg.name}@${pkg.version} successfully!`);
+    log(`  ${filesToUpload.length} files, ${registryAssets.length} assets, ${formatBytes(totalSize)}`);
+    return {
+        ok: true,
+        packageName: pkg.name,
+        version: pkg.version,
+        assetCount: registryAssets.length,
+        fileCount: filesToUpload.length,
+        thumbnailCount,
+    };
+}
+// ─── Helpers ─────────────────────────────────────────────────────────────────
+function formatBytes(bytes) {
+    if (bytes < 1024)
+        return `${bytes} B`;
+    if (bytes < 1024 * 1024)
+        return `${(bytes / 1024).toFixed(1)} KB`;
+    return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}
+const CONTENT_TYPES = {
+    png: 'image/png',
+    jpg: 'image/jpeg',
+    jpeg: 'image/jpeg',
+    webp: 'image/webp',
+    gif: 'image/gif',
+    bmp: 'image/bmp',
+    wav: 'audio/wav',
+    mp3: 'audio/mpeg',
+    ogg: 'audio/ogg',
+    flac: 'audio/flac',
+    aac: 'audio/aac',
+    m4a: 'audio/mp4',
+    fbx: 'application/octet-stream',
+    obj: 'text/plain',
+    glb: 'model/gltf-binary',
+    gltf: 'model/gltf+json',
+    json: 'application/json',
+    stowmeta: 'application/json',
+    stowmat: 'application/json',
+};
+function guessContentType(filePath) {
+    const ext = filePath.split('.').pop()?.toLowerCase() ?? '';
+    return CONTENT_TYPES[ext] ?? 'application/octet-stream';
+}
+/** Run async tasks with a concurrency limit. */
+async function runConcurrent(items, limit, fn) {
+    let i = 0;
+    const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
+        while (i < items.length) {
+            const idx = i++;
+            await fn(items[idx]);
+        }
+    });
+    await Promise.all(workers);
+}