@lazycatcloud/lzc-cli 2.0.0-pre.0 → 2.0.0-pre.2

package/lib/app/lpk_build_images_pack_local.js

@@ -0,0 +1,409 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import crypto from 'node:crypto';
+import zlib from 'node:zlib';
+import { Transform } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import * as tar from 'tar';
+
+const SHA256_PREFIX = 'sha256:';
+const MAX_INLINE_CONFIG_BLOB_SIZE = 2 * 1024 * 1024;
+
+function normalizeSha256Digest(value, fieldName) {
+	const raw = String(value ?? '').trim();
+	if (!raw.startsWith(SHA256_PREFIX)) {
+		throw new Error(`${fieldName} must start with sha256:, got: ${raw}`);
+	}
+	const hex = raw.slice(SHA256_PREFIX.length);
+	if (!/^[0-9a-f]{64}$/i.test(hex)) {
+		throw new Error(`${fieldName} is not a valid sha256 digest: ${raw}`);
+	}
+	return `${SHA256_PREFIX}${hex.toLowerCase()}`;
+}
+
+function normalizeArchivePath(name) {
+	return String(name ?? '').trim().replace(/^\.\//, '');
+}
+
+function isRegularEntry(entry) {
+	return entry && (entry.type === 'File' || entry.type === 'OldFile' || entry.type === 'ContiguousFile');
+}
+
+async function readEntryBuffer(entry) {
+	const chunks = [];
+	for await (const chunk of entry) {
+		chunks.push(chunk);
+	}
+	return Buffer.concat(chunks);
+}
+
+async function walkDockerArchive(archivePath, onEntry) {
+	const tasks = [];
+	await tar.t({
+		file: archivePath,
+		onReadEntry: (entry) => {
+			const task = (async () => {
+				if (!isRegularEntry(entry)) {
+					entry.resume();
+					return;
+				}
+				await onEntry(entry);
+			})();
+			tasks.push(task);
+		},
+	});
+	await Promise.all(tasks);
+}
+
+async function collectArchiveManifestAndEntrySizes(archivePath) {
+	const entrySizeByPath = {};
+	const jsonFileByPath = {};
+	let manifestRaw = null;
+
+	await walkDockerArchive(archivePath, async (entry) => {
+		const name = normalizeArchivePath(entry.path);
+		entrySizeByPath[name] = Number(entry.size ?? 0);
+		const shouldRead = name === 'manifest.json' || name.endsWith('.json') || (name.startsWith('blobs/sha256/') && Number(entry.size ?? 0) <= MAX_INLINE_CONFIG_BLOB_SIZE);
+		if (!shouldRead) {
+			entry.resume();
+			return;
+		}
+		const raw = await readEntryBuffer(entry);
+		jsonFileByPath[name] = raw;
+		if (name === 'manifest.json') {
+			manifestRaw = raw;
+		}
+	});
+
+	if (!manifestRaw) {
+		throw new Error('manifest.json not found in docker archive');
+	}
+	return { manifestRaw, entrySizeByPath, jsonFileByPath };
+}
+
+async function readArchiveFiles(archivePath, wantedSet) {
+	const files = {};
+	if (!wantedSet || wantedSet.size === 0) {
+		return files;
+	}
+	await walkDockerArchive(archivePath, async (entry) => {
+		const name = normalizeArchivePath(entry.path);
+		if (!wantedSet.has(name)) {
+			entry.resume();
+			return;
+		}
+		files[name] = await readEntryBuffer(entry);
+	});
+	for (const name of wantedSet) {
+		if (!Object.prototype.hasOwnProperty.call(files, name)) {
+			throw new Error(`entry ${name} not found in docker archive`);
+		}
+	}
+	return files;
+}
+
+function copyBytesToBlob(content, blobsDir) {
+	const buffer = Buffer.isBuffer(content) ? content : Buffer.from(content);
+	const digest = crypto.createHash('sha256').update(buffer).digest('hex');
+	const targetPath = path.join(blobsDir, digest);
+	if (!fs.existsSync(targetPath)) {
+		fs.writeFileSync(targetPath, buffer);
+	}
+	return {
+		digest,
+		size: buffer.length,
+	};
+}
+
+async function gzipEntryToBlob(entry, blobsDir) {
+	const tmpDir = fs.mkdtempSync(path.join(blobsDir, '.lzc-cli-embed-layer-'));
+	const tmpPath = path.join(tmpDir, 'layer.tar.gz');
+	const out = fs.createWriteStream(tmpPath);
+	const hash = crypto.createHash('sha256');
+	let size = 0;
+	const tee = new Transform({
+		transform(chunk, _encoding, callback) {
+			hash.update(chunk);
+			size += chunk.length;
+			callback(null, chunk);
+		},
+	});
+	try {
+		await pipeline(entry, zlib.createGzip({ mtime: 0 }), tee, out);
+		const digest = hash.digest('hex');
+		const targetPath = path.join(blobsDir, digest);
+		if (!fs.existsSync(targetPath)) {
+			fs.renameSync(tmpPath, targetPath);
+		}
+		return {
+			digest,
+			size,
+		};
+	} finally {
+		fs.rmSync(tmpDir, { recursive: true, force: true });
+	}
+}
+
+function compactLockLayers(rawLayers) {
+	if (!Array.isArray(rawLayers)) {
+		return [];
+	}
+	return rawLayers
+		.map((item) => ({
+			digest: String(item?.digest ?? '').trim(),
+			source: String(item?.source ?? '').trim().toLowerCase(),
+		}))
+		.filter((item) => item.digest);
+}
+
+function writeOciLayout(stageDir, state) {
+	const imageRootDir = path.join(stageDir, 'images');
+	const blobsDir = path.join(imageRootDir, 'blobs', 'sha256');
+	fs.mkdirSync(blobsDir, { recursive: true });
+	state.indexDescriptors.sort((a, b) => String(a?.annotations?.['org.opencontainers.image.ref.name'] ?? '').localeCompare(String(b?.annotations?.['org.opencontainers.image.ref.name'] ?? '')));
+	fs.writeFileSync(path.join(imageRootDir, 'oci-layout'), JSON.stringify({ imageLayoutVersion: '1.0.0' }));
+	fs.writeFileSync(
+		path.join(imageRootDir, 'index.json'),
+		JSON.stringify(
+			{
+				schemaVersion: 2,
+				manifests: state.indexDescriptors,
+			},
+			null,
+			2,
+		),
+	);
+	fs.writeFileSync(
+		path.join(stageDir, 'pack-result.json'),
+		JSON.stringify({
+			lockImages: state.lockImages,
+			embeddedLayerBytes: state.embeddedLayerBytes,
+			embeddedLayerCount: state.embeddedDigestSet.size,
+		}),
+	);
+}
+
+function finalizeSelectedPackImage(selected, blobsDir, state) {
+	const layerDescriptors = [];
+	const lockLayers = [];
+	for (const layerPlan of selected.layerPlans) {
+		let finalDigest = layerPlan.rawDigest;
+		let finalSize = layerPlan.layerSize;
+		let mediaType = 'application/vnd.oci.image.layer.v1.tar';
+		let source = 'upstream';
+
+		if (layerPlan.shouldEmbed) {
+			const compressedBlob = state.compressedByRaw.get(layerPlan.rawDigest);
+			if (!compressedBlob) {
+				throw new Error(`compressed layer not found for ${layerPlan.rawDigest}`);
+			}
+			finalDigest = `${SHA256_PREFIX}${compressedBlob.digest}`;
+			finalSize = compressedBlob.size;
+			mediaType = 'application/vnd.oci.image.layer.v1.tar+gzip';
+			source = 'embed';
+			if (!state.embeddedDigestSet.has(finalDigest)) {
+				state.embeddedDigestSet.add(finalDigest);
+				state.embeddedLayerBytes += compressedBlob.size;
+			}
+		}
+
+		layerDescriptors.push({
+			mediaType,
+			digest: finalDigest,
+			size: finalSize,
+		});
+		lockLayers.push({
+			digest: finalDigest,
+			source,
+		});
+	}
+
+	const manifestContent = Buffer.from(
+		JSON.stringify({
+			schemaVersion: 2,
+			mediaType: 'application/vnd.oci.image.manifest.v1+json',
+			config: {
+				mediaType: 'application/vnd.oci.image.config.v1+json',
+				digest: `${SHA256_PREFIX}${selected.configBlob.digest}`,
+				size: selected.configBlob.size,
+			},
+			layers: layerDescriptors,
+		}),
+	);
+	const manifestBlob = copyBytesToBlob(manifestContent, blobsDir);
+	state.indexDescriptors.push({
+		mediaType: 'application/vnd.oci.image.manifest.v1+json',
+		digest: `${SHA256_PREFIX}${manifestBlob.digest}`,
+		size: manifestBlob.size,
+		annotations: {
+			'org.opencontainers.image.ref.name': selected.spec.alias,
+		},
+	});
+	state.lockImages[selected.spec.alias] = {
+		image_id: selected.normalizedImageID,
+		upstream: String(selected.spec.upstream ?? '').trim(),
+		layers: compactLockLayers(lockLayers),
+	};
+	state.matchedAlias.add(selected.spec.alias);
+}
+
+export async function packLocalArchiveToStageDir(archivePath, packSpecs, stageDir) {
+	if (!Array.isArray(packSpecs) || packSpecs.length === 0) {
+		throw new Error('packSpecs cannot be empty');
+	}
+	const imageRootDir = path.join(stageDir, 'images');
+	const blobsDir = path.join(imageRootDir, 'blobs', 'sha256');
+	fs.mkdirSync(blobsDir, { recursive: true });
+
+	const { manifestRaw, entrySizeByPath, jsonFileByPath } = await collectArchiveManifestAndEntrySizes(archivePath);
+	const manifests = JSON.parse(String(manifestRaw));
+	if (!Array.isArray(manifests) || manifests.length === 0) {
+		throw new Error('manifest.json is empty in docker archive');
+	}
+
+	const specByRef = new Map();
+	for (const item of packSpecs) {
+		const ref = String(item?.ref ?? '').trim();
+		if (!specByRef.has(ref)) {
+			specByRef.set(ref, []);
+		}
+		specByRef.get(ref).push(item);
+	}
+
+	const selectedByAlias = new Map();
+	for (const manifestItem of manifests) {
+		const repoTags = Array.isArray(manifestItem?.RepoTags) ? manifestItem.RepoTags.map((item) => String(item ?? '').trim()).filter((item) => item) : [];
+		const normalizedManifest = {
+			config: normalizeArchivePath(manifestItem?.Config),
+			layers: Array.isArray(manifestItem?.Layers) ? manifestItem.Layers.map((item) => normalizeArchivePath(item)) : [],
+			repoTags,
+		};
+		for (const repoTag of repoTags) {
+			for (const candidate of specByRef.get(repoTag) ?? []) {
+				if (!selectedByAlias.has(candidate.alias)) {
+					selectedByAlias.set(candidate.alias, {
+						spec: candidate,
+						manifest: normalizedManifest,
+					});
+				}
+			}
+		}
+	}
+
+	for (const item of packSpecs) {
+		if (!selectedByAlias.has(item.alias)) {
+			throw new Error(`alias ${item.alias} not found in archive ${archivePath}`);
+		}
+	}
+
+	const missingConfigs = new Set();
+	for (const selected of selectedByAlias.values()) {
+		if (!Object.prototype.hasOwnProperty.call(jsonFileByPath, selected.manifest.config)) {
+			missingConfigs.add(selected.manifest.config);
+		}
+	}
+	if (missingConfigs.size > 0) {
+		const extraConfigByPath = await readArchiveFiles(archivePath, missingConfigs);
+		for (const [name, content] of Object.entries(extraConfigByPath)) {
+			jsonFileByPath[name] = content;
+		}
+	}
+
+	const embedLayerPathToRawDigest = new Map();
+	const requiredEmbedRawDigestSet = new Set();
+	for (const [alias, selected] of selectedByAlias.entries()) {
+		const configRaw = jsonFileByPath[selected.manifest.config];
+		if (!configRaw) {
+			throw new Error(`config ${selected.manifest.config} not found for alias ${alias}`);
+		}
+		selected.configBlob = copyBytesToBlob(configRaw, blobsDir);
+		selected.normalizedImageID = normalizeSha256Digest(String(selected.spec.imageID ?? ''), `imageID for alias ${selected.spec.alias}`);
+		const actualImageID = `${SHA256_PREFIX}${selected.configBlob.digest}`;
+		if (actualImageID !== selected.normalizedImageID) {
+			throw new Error(`image id mismatch for alias ${selected.spec.alias}, got ${actualImageID} want ${selected.normalizedImageID}`);
+		}
+		const config = JSON.parse(String(configRaw));
+		const diffIDs = Array.isArray(config?.rootfs?.diff_ids) ? config.rootfs.diff_ids : [];
+		if (diffIDs.length !== selected.manifest.layers.length) {
+			throw new Error(`layer count mismatch for alias ${selected.spec.alias}`);
+		}
+		const embeddedSet = new Set((selected.spec.embeddedDiffIDs ?? []).map((item, index) => normalizeSha256Digest(item, `embeddedDiffID for alias ${selected.spec.alias}[${index}]`)));
+		selected.layerPlans = selected.manifest.layers.map((layerPath, index) => {
+			const layerSize = Number(entrySizeByPath[layerPath] ?? -1);
+			if (layerSize < 0) {
+				throw new Error(`layer file missing in docker archive: ${layerPath}`);
+			}
+			const rawDigest = normalizeSha256Digest(diffIDs[index], `diffID for alias ${selected.spec.alias}[${index}]`);
+			const shouldEmbed = embeddedSet.has(rawDigest);
+			if (!shouldEmbed && !String(selected.spec.upstream ?? '').trim()) {
+				throw new Error(`alias ${selected.spec.alias} has upstream layer ${rawDigest} but upstream is empty`);
+			}
+			if (shouldEmbed) {
+				requiredEmbedRawDigestSet.add(rawDigest);
+				const existed = embedLayerPathToRawDigest.get(layerPath);
+				if (existed && existed !== rawDigest) {
+					throw new Error(`layer ${layerPath} maps to different diff ids: ${existed} vs ${rawDigest}`);
+				}
+				embedLayerPathToRawDigest.set(layerPath, rawDigest);
+			}
+			return {
+				layerPath,
+				rawDigest,
+				layerSize,
+				shouldEmbed,
+			};
+		});
+	}
+
+	const state = {
+		lockImages: {},
+		indexDescriptors: [],
+		matchedAlias: new Set(),
+		compressedByRaw: new Map(),
+		embeddedDigestSet: new Set(),
+		embeddedLayerBytes: 0,
+	};
+
+	if (embedLayerPathToRawDigest.size > 0) {
+		await walkDockerArchive(archivePath, async (entry) => {
+			const name = normalizeArchivePath(entry.path);
+			const rawDigest = embedLayerPathToRawDigest.get(name);
+			if (!rawDigest) {
+				entry.resume();
+				return;
+			}
+			if (state.compressedByRaw.has(rawDigest)) {
+				entry.resume();
+				return;
+			}
+			const compressedBlob = await gzipEntryToBlob(entry, blobsDir);
+			state.compressedByRaw.set(rawDigest, compressedBlob);
+		});
+	}
+
+	for (const rawDigest of requiredEmbedRawDigestSet) {
+		if (!state.compressedByRaw.has(rawDigest)) {
+			throw new Error(`embed layer digest ${rawDigest} not found in docker archive`);
+		}
+	}
+
+	for (const item of packSpecs) {
+		const selected = selectedByAlias.get(item.alias);
+		if (!selected) {
+			throw new Error(`alias ${item.alias} not found in selected image set`);
+		}
+		finalizeSelectedPackImage(selected, blobsDir, state);
+	}
+
+	writeOciLayout(stageDir, state);
+	return {
+		lockImages: state.lockImages,
+		embeddedLayerBytes: state.embeddedLayerBytes,
+		embeddedLayerCount: state.embeddedDigestSet.size,
+	};
+}
+
+export const __test__ = {
+	normalizeSha256Digest,
+};