@arkebcacy/beacon-cli-temp 0.1.1

package/src/schema/assets/lib/NamingConvention.ts

@@ -0,0 +1,93 @@
+import type { Dirent } from 'node:fs';
+import path from 'node:path';
+
+export function getBlobPath(absPathToOriginalFile: string): string;
+export function getBlobPath(assetsPath: string, itemPath: string): string;
+export function getBlobPath(
+	assetsPathOrAbsPath: string,
+	itemPath?: string,
+): string {
+	const absPath =
+		typeof itemPath === 'string'
+			? path.resolve(assetsPathOrAbsPath, itemPath)
+			: assetsPathOrAbsPath;
+
+	const ext = path.extname(absPath);
+	const baseName = path.basename(absPath, ext);
+	return path.resolve(path.dirname(absPath), `${baseName}.blob${ext}`);
+}
+
+export function getMetaPath(assetsPath: string, itemPath: string) {
+	const absPath = path.resolve(assetsPath, itemPath);
+	const ext = path.extname(absPath);
+	const baseName = path.basename(absPath, ext);
+	return path.join(path.dirname(absPath), `${baseName}.meta${ext}.yaml`);
+}
+
+export function getItemPath(assetsPath: string, absPath: string) {
+	// absPath supported input formats:
+	//   /full/schema/path/assets/whatever/some-file.blob.webp
+	//   /full/schema/path/assets/whatever/some-file.meta.webp.yaml
+	//
+	// Expected output format:
+	//   whatever/some-file.webp
+
+	const ext1 = path.extname(absPath);
+	const stripFirstExt = path.basename(absPath, ext1);
+	const ext2 = path.extname(stripFirstExt);
+	const stripSecondExt = path.basename(stripFirstExt, ext2);
+	const dirname = path.dirname(absPath);
+
+	if (ext2 === '.blob') {
+		const originalPath = path.resolve(dirname, `${stripSecondExt}${ext1}`);
+		return formatItemPath(assetsPath, originalPath);
+	}
+
+	const ext3 = path.extname(stripSecondExt);
+
+	if (ext1 === '.yaml' && ext3 === '.meta') {
+		const stripThirdExt = path.basename(stripSecondExt, ext3);
+		const originalPath = path.resolve(dirname, `${stripThirdExt}${ext2}`);
+		return formatItemPath(assetsPath, originalPath);
+	}
+
+	throw new Error(`Invalid asset path: ${absPath}`);
+}
+
+export function* assetPaths(assetsPath: string, files: Iterable<Dirent>) {
+	for (const file of files) {
+		if (!file.isFile()) {
+			continue;
+		}
+
+		const ext = path.extname(file.name);
+		if (ext !== '.yaml') {
+			continue;
+		}
+
+		const stripYaml = path.basename(file.name, '.yaml');
+		const originalExt = path.extname(stripYaml);
+		const stripOriginalExt = path.basename(stripYaml, originalExt);
+		const metaExt = path.extname(stripOriginalExt);
+
+		if (metaExt !== '.meta') {
+			continue;
+		}
+
+		const orgBaseName = path.basename(stripOriginalExt, metaExt);
+		const orgName = `${orgBaseName}${originalExt}`;
+		const originalPath = path.resolve(file.parentPath, orgName);
+		const blobName = `${orgBaseName}.blob${originalExt}`;
+
+		yield {
+			blobPath: path.resolve(file.parentPath, blobName),
+			itemPath: formatItemPath(assetsPath, originalPath),
+			metaPath: path.resolve(file.parentPath, file.name),
+		};
+	}
+}
+
+export function formatItemPath(assetsPath: string, itemPath: string) {
+	const result = path.relative(assetsPath, itemPath);
+	return path.sep === '/' ? result : result.replaceAll(path.sep, '/');
+}

package/src/schema/assets/lib/loadMetaCs.ts

@@ -0,0 +1,19 @@
+import type { ReadonlyAssets } from '#cli/cs/assets/Assets.js';
+import { isRawAsset } from '#cli/cs/assets/Types.js';
+import type AssetMeta from '../AssetMeta.js';
+import { fromRawAsset } from '../AssetMeta.js';
+import resolveItemPath from './resolveItemPath.js';
+
+export default function loadMetaCs(
+	assets: ReadonlyAssets,
+): ReadonlyMap<string, AssetMeta> {
+	const result = new Map<string, AssetMeta>();
+	const allAssets = [...assets.descendantOrSelf(null)].filter(isRawAsset);
+
+	for (const asset of allAssets) {
+		const itemPath = resolveItemPath(assets.byUid, asset);
+		result.set(itemPath, fromRawAsset(asset, itemPath));
+	}
+
+	return result;
+}

package/src/schema/assets/lib/planPull.test.ts

@@ -0,0 +1,54 @@
+import TestLogContext from '#test/integration/lib/TestLogContext.js';
+import TestPushUiContext from '#test/integration/lib/TestPushUiContext.js';
+import mockMergePlan from '#test/models/filters/mockMergePlan.js';
+import { arrange, Theory } from '#test/models/filters/Theory.js';
+import { inspect } from 'node:util';
+import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
+import type planPull from './planPull.js';
+
+const logs = new TestLogContext();
+const ui = new TestPushUiContext('fixtures/asset-filters', logs);
+vi.doMock(import('../../lib/SchemaUi.js'), () => ({ default: () => ui }));
+
+describe('Assets: planPull', () => {
+	let sut: typeof planPull;
+
+	beforeAll(async () => {
+		sut = (await import('./planPull.js')).default;
+	});
+
+	afterEach(() => logs.clear());
+
+	const theories: readonly Theory[] = [
+		new Theory(true, true, true, true, 'skip'),
+		new Theory(true, true, true, false, 'update'),
+		new Theory(true, true, false, true, 'skip'),
+		new Theory(true, true, false, false, 'skip'),
+		new Theory(true, false, true, true, 'create'),
+		new Theory(true, false, true, false, 'create'),
+		new Theory(true, false, false, true, 'skip'),
+		new Theory(true, false, false, false, 'skip'),
+		new Theory(false, true, true, true, 'delete'),
+		new Theory(false, true, true, false, 'delete'),
+		new Theory(false, true, false, true, 'skip'),
+		new Theory(false, true, false, false, 'skip'),
+	];
+
+	theories.forEach((theory) => {
+		it(inspect(theory, { colors: true, compact: true }), () => {
+			// Arrange
+			const { cs, isIncluded, fs, itemPath } = arrange(theory);
+			ui.options.schema.assets = { isIncluded };
+
+			// Act
+			const actual = sut(cs, fs);
+
+			// Assert
+			expect(actual).toEqual(mockMergePlan(theory, itemPath, cs, fs));
+
+			expect(logs.warnings).not.toContainEqual(
+				expect.stringContaining(itemPath),
+			);
+		});
+	});
+});

package/src/schema/assets/lib/planPull.ts

@@ -0,0 +1,73 @@
+import type MergePlan from '#cli/schema/xfer/lib/MergePlan.js';
+import { isDeepStrictEqual } from 'node:util';
+import getUi from '../../lib/SchemaUi.js';
+import type AssetMeta from '../AssetMeta.js';
+
+export default function planPull(
+	cs: ReadonlyMap<string, AssetMeta>,
+	fs: ReadonlyMap<string, AssetMeta>,
+): MergePlan<AssetMeta> {
+	const ui = getUi();
+	const { isIncluded } = ui.options.schema.assets;
+
+	const { result, seen } = processCsAssets(cs, fs, isIncluded);
+	processFsAssets(fs, isIncluded, { result, seen });
+
+	return result;
+}
+
+function processCsAssets(
+	cs: ReadonlyMap<string, AssetMeta>,
+	fs: ReadonlyMap<string, AssetMeta>,
+	isIncluded: (path: string) => boolean,
+) {
+	const result = {
+		toCreate: new Map<string, AssetMeta>(),
+		toRemove: new Map<string, AssetMeta>(),
+		toSkip: new Set<string>(),
+		toUpdate: new Map<string, AssetMeta>(),
+	};
+
+	const seen = new Set<string>();
+
+	for (const [path, csMeta] of cs) {
+		seen.add(path);
+		const fsMeta = fs.get(path);
+
+		if (fsMeta) {
+			if (isIncluded(path)) {
+				if (isDeepStrictEqual(csMeta, fsMeta)) {
+					result.toSkip.add(path);
+				} else {
+					result.toUpdate.set(path, csMeta);
+				}
+			} else {
+				result.toSkip.add(path);
+			}
+		} else if (isIncluded(path)) {
+			result.toCreate.set(path, csMeta);
+		} else {
+			result.toSkip.add(path);
+		}
+	}
+
+	return { result, seen };
+}
+
+function processFsAssets(
+	fs: ReadonlyMap<string, AssetMeta>,
+	isIncluded: (path: string) => boolean,
+	{ result, seen }: ReturnType<typeof processCsAssets>,
+) {
+	for (const [path, fsMeta] of fs) {
+		if (seen.has(path)) {
+			continue;
+		}
+
+		if (isIncluded(path)) {
+			result.toRemove.set(path, fsMeta);
+		} else {
+			result.toSkip.add(path);
+		}
+	}
+}

package/src/schema/assets/lib/planPush.test.ts

@@ -0,0 +1,58 @@
+import TestLogContext from '#test/integration/lib/TestLogContext.js';
+import TestPushUiContext from '#test/integration/lib/TestPushUiContext.js';
+import mockMergePlan from '#test/models/filters/mockMergePlan.js';
+import { arrange, Theory } from '#test/models/filters/Theory.js';
+import { inspect } from 'node:util';
+import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
+import type planPush from './planPush.js';
+
+const logs = new TestLogContext();
+const ui = new TestPushUiContext('fixtures/asset-filters', logs);
+vi.doMock(import('../../lib/SchemaUi.js'), () => ({ default: () => ui }));
+
+describe('Assets: planPush', () => {
+	let sut: typeof planPush;
+
+	beforeAll(async () => {
+		sut = (await import('./planPush.js')).default;
+	});
+
+	afterEach(() => logs.clear());
+
+	const theories: readonly Theory[] = [
+		new Theory(true, true, true, true, 'skip'),
+		new Theory(true, true, true, false, 'update'),
+		new Theory(true, true, false, true, 'warning'),
+		new Theory(true, true, false, false, 'warning'),
+		new Theory(true, false, true, true, 'delete'),
+		new Theory(true, false, true, false, 'delete'),
+		new Theory(true, false, false, true, 'skip'),
+		new Theory(true, false, false, false, 'skip'),
+		new Theory(false, true, true, true, 'create'),
+		new Theory(false, true, true, false, 'create'),
+		new Theory(false, true, false, true, 'warning'),
+		new Theory(false, true, false, false, 'warning'),
+	];
+
+	theories.forEach((theory) => {
+		it(inspect(theory, { colors: true, compact: true }), () => {
+			// Arrange
+			const { cs, isIncluded, fs, itemPath } = arrange(theory);
+			ui.options.schema.assets = { isIncluded };
+
+			// Act
+			const actual = sut(cs, fs);
+
+			// Assert
+			expect(actual).toEqual(mockMergePlan(theory, itemPath, fs, cs));
+
+			if (theory.expected === 'warning') {
+				expect(logs.warnings).toContainEqual(expect.stringContaining(itemPath));
+			} else {
+				expect(logs.warnings).not.toContainEqual(
+					expect.stringContaining(itemPath),
+				);
+			}
+		});
+	});
+});

package/src/schema/assets/lib/planPush.ts

@@ -0,0 +1,88 @@
+import type MergePlan from '#cli/schema/xfer/lib/MergePlan.js';
+import type UiContext from '#cli/ui/UiContext.js';
+import { isDeepStrictEqual, styleText } from 'node:util';
+import getUi from '../../lib/SchemaUi.js';
+import type AssetMeta from '../AssetMeta.js';
+
+export default function planPush(
+	cs: ReadonlyMap<string, AssetMeta>,
+	fs: ReadonlyMap<string, AssetMeta>,
+): MergePlan<AssetMeta> {
+	const fsResult = processFsItems(cs, fs);
+	const csResult = processCsItems(cs, fsResult.seen);
+
+	return {
+		toCreate: fsResult.toCreate,
+		toRemove: csResult.toRemove,
+		toSkip: new Set([...fsResult.toSkip, ...csResult.toSkip]),
+		toUpdate: fsResult.toUpdate,
+	};
+}
+
+function warning(ui: UiContext, itemPath: string) {
+	const msg1 = 'Skipping asset which exists in the file system,';
+	const msg2 = 'but is not included by filters:';
+	const msg3 = styleText('yellowBright', itemPath);
+	ui.warn(msg1, msg2, msg3);
+}
+
+function processFsItems(
+	cs: ReadonlyMap<string, AssetMeta>,
+	fs: ReadonlyMap<string, AssetMeta>,
+) {
+	const seen = new Set<string>();
+	const ui = getUi();
+	const { isIncluded } = ui.options.schema.assets;
+	const toCreate = new Map<string, AssetMeta>();
+	const toSkip = new Set<string>();
+	const toUpdate = new Map<string, AssetMeta>();
+
+	for (const [itemPath, fsMeta] of fs) {
+		seen.add(itemPath);
+		const csMeta = cs.get(itemPath);
+
+		if (csMeta) {
+			if (isIncluded(itemPath)) {
+				if (isDeepStrictEqual(csMeta, fsMeta)) {
+					toSkip.add(itemPath);
+				} else {
+					toUpdate.set(itemPath, fsMeta);
+				}
+			} else {
+				warning(ui, itemPath);
+				toSkip.add(itemPath);
+			}
+		} else if (isIncluded(itemPath)) {
+			toCreate.set(itemPath, fsMeta);
+		} else {
+			warning(ui, itemPath);
+			toSkip.add(itemPath);
+		}
+	}
+
+	return { seen, toCreate, toSkip, toUpdate };
+}
+
+function processCsItems(
+	cs: ReadonlyMap<string, AssetMeta>,
+	seenItems: ReadonlySet<string>,
+) {
+	const ui = getUi();
+	const { isIncluded } = ui.options.schema.assets;
+	const toRemove = new Map<string, AssetMeta>();
+	const toSkip = new Set<string>();
+
+	for (const [itemPath, csMeta] of cs) {
+		if (seenItems.has(itemPath)) {
+			continue;
+		}
+
+		if (isIncluded(itemPath)) {
+			toRemove.set(itemPath, csMeta);
+		} else {
+			toSkip.add(itemPath);
+		}
+	}
+
+	return { toRemove, toSkip };
+}

package/src/schema/assets/lib/resolveItemPath.ts

@@ -0,0 +1,26 @@
+import type { RawAssetItem } from '#cli/cs/assets/Types.js';
+import { isRawFolder } from '#cli/cs/assets/Types.js';
+
+export default function resolveItemPath(
+	byUid: ReadonlyMap<string, RawAssetItem>,
+	item: RawAssetItem,
+): string {
+	const { parent_uid: parentUid } = item;
+	const name = isRawFolder(item) ? item.name : item.filename;
+	if (!parentUid) {
+		return name;
+	}
+
+	const parent = byUid.get(parentUid);
+
+	if (!parent) {
+		throw new Error(`Parent UID not found: ${parentUid}`);
+	}
+
+	const parentPath = resolveItemPath(byUid, parent);
+
+	// Do not use `path.join` here because we depend on a consistent path
+	// separator across all platforms. `name` is already known to be a pure
+	// filename and will not contain any path separators.
+	return `${parentPath}/${name}`;
+}

package/src/schema/assets/lib/writeAssetToContentstack.ts

@@ -0,0 +1,30 @@
+import type Assets from '#cli/cs/assets/Assets.js';
+import resolveRawAssetItem from '#cli/cs/assets/lib/resolveRawAssetItem.js';
+import { resolve } from 'node:path';
+import type AssetMeta from '../AssetMeta.js';
+
+export default async function writeAssetToContentstack(
+	csState: Assets,
+	assetsPath: string,
+	meta: AssetMeta,
+) {
+	const parentPath = meta.itemPath.split('/').slice(0, -1).join('/');
+	const parent = resolveRawAssetItem(csState.byParentUid, parentPath);
+
+	if (parentPath && !parent) {
+		throw new Error('Parent folder not found: ' + parentPath);
+	}
+
+	const filePath = resolve(assetsPath, meta.itemPath);
+
+	return csState.createAsset({
+		filePath,
+		...(typeof meta.description === 'string'
+			? { description: meta.description }
+			: {}),
+
+		...(meta.tags.size > 0 ? { tags: [...meta.tags] } : {}),
+		...(typeof meta.title === 'string' ? { title: meta.title } : {}),
+		...(parent ? { parent_uid: parent.uid } : {}),
+	});
+}

package/src/schema/assets/lib/writeAssetToFileSystem.ts

@@ -0,0 +1,39 @@
+import type { ReadonlyAssets } from '#cli/cs/assets/Assets.js';
+import resolveRawAssetItem from '#cli/cs/assets/lib/resolveRawAssetItem.js';
+import { isRawAsset } from '#cli/cs/assets/Types.js';
+import { createWriteStream } from 'node:fs';
+import { mkdir } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { pipeline } from 'node:stream/promises';
+import type AssetMeta from '../AssetMeta.js';
+import { save } from './MetaSerialization.js';
+import { getBlobPath } from './NamingConvention.js';
+
+export default async function writeAssetToFileSystem(
+	assets: ReadonlyAssets,
+	assetsPath: string,
+	asset: AssetMeta,
+) {
+	const { itemPath } = asset;
+	const rawAsset = resolveRawAssetItem(assets.byParentUid, itemPath);
+	if (!isRawAsset(rawAsset)) {
+		throw new Error(`Could not find raw asset for ${itemPath}`);
+	}
+
+	await Promise.all([
+		downloadFile(rawAsset.url, getBlobPath(assetsPath, itemPath)),
+		save(assetsPath, asset),
+	]);
+}
+
+async function downloadFile(url: string, destination: string) {
+	const { body } = await fetch(url);
+
+	if (!body) {
+		throw new Error(`Failed to download ${url}`);
+	}
+
+	await mkdir(dirname(destination), { recursive: true });
+	const fileStream = createWriteStream(destination);
+	await pipeline(body, fileStream);
+}

package/src/schema/assets/schemaDirectory.ts

@@ -0,0 +1,12 @@
+import { resolve } from 'node:path';
+import getUi from '../lib/SchemaUi.js';
+
+export default function schemaDirectory() {
+	const {
+		options: {
+			schema: { schemaPath },
+		},
+	} = getUi();
+
+	return resolve(schemaPath, 'assets');
+}

package/src/schema/assets/toContentstack.ts

@@ -0,0 +1,65 @@
+import resolveRawAssetItem from '#cli/cs/assets/lib/resolveRawAssetItem.js';
+import updateAsset from '#cli/cs/assets/update.js';
+import { mkdir } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import type Ctx from '../ctx/Ctx.js';
+import createProgressBar from '../lib/createProgressBar.js';
+import getUi from '../lib/SchemaUi.js';
+import processPlan from '../xfer/lib/processPlan.js';
+import type AssetMeta from './AssetMeta.js';
+import loadMetaCs from './lib/loadMetaCs.js';
+import planPush from './lib/planPush.js';
+import writeAssetToContentstack from './lib/writeAssetToContentstack.js';
+import schemaDirectory from './schemaDirectory.js';
+
+export default async function toContentstack(ctx: Ctx) {
+	const ui = getUi();
+	const assetsPath = schemaDirectory();
+
+	await mkdir(assetsPath, { recursive: true });
+	const fsMeta = ctx.fs.assets.assetsByPath;
+	const csMeta = loadMetaCs(ctx.cs.assets);
+	using bar = createProgressBar('Assets', csMeta, fsMeta);
+
+	const create = async (meta: AssetMeta) =>
+		writeAssetToContentstack(ctx.cs.assets, assetsPath, meta);
+
+	return await processPlan({
+		create,
+		deletionStrategy: ui.options.schema.deletionStrategy,
+		plan: planPush(csMeta, fsMeta),
+		progress: bar,
+		remove: async (asset) => ctx.cs.assets.deleteAsset(asset.itemPath),
+		update: async (meta) => update(ctx, csMeta, assetsPath, meta),
+	});
+}
+
+async function update(
+	ctx: Ctx,
+	csMeta: ReadonlyMap<string, AssetMeta>,
+	assetsPath: string,
+	updatedMeta: AssetMeta,
+) {
+	const currentCsMeta = csMeta.get(updatedMeta.itemPath);
+
+	const currentRawItem = resolveRawAssetItem(
+		ctx.cs.assets.byParentUid,
+		updatedMeta.itemPath,
+	);
+
+	if (!currentCsMeta || !currentRawItem) {
+		throw new Error('Asset not found in Contentstack: ' + updatedMeta.itemPath);
+	}
+
+	const updatedItemPath = resolve(assetsPath, updatedMeta.itemPath);
+
+	return updateAsset(ctx.cs.client, {
+		description: updatedMeta.description,
+		tags: [...updatedMeta.tags],
+		title: updatedMeta.title,
+		uid: currentRawItem.uid,
+		...(currentCsMeta.fileSize === updatedMeta.fileSize
+			? {}
+			: { filePath: updatedItemPath }),
+	});
+}

package/src/schema/assets/toFilesystem.ts

@@ -0,0 +1,51 @@
+import { rm } from 'node:fs/promises';
+import type Ctx from '../ctx/Ctx.js';
+import createProgressBar from '../lib/createProgressBar.js';
+import getUi from '../lib/SchemaUi.js';
+import processPlan from '../xfer/lib/processPlan.js';
+import type AssetMeta from './AssetMeta.js';
+import loadMetaCs from './lib/loadMetaCs.js';
+import { getBlobPath, getMetaPath } from './lib/NamingConvention.js';
+import planPull from './lib/planPull.js';
+import writeAssetToFileSystem from './lib/writeAssetToFileSystem.js';
+import schemaDirectory from './schemaDirectory.js';
+
+export default async function toFilesystem(ctx: Ctx) {
+	const assetsPath = schemaDirectory();
+	const write = writeAssetToFileSystem.bind(null, ctx.cs.assets, assetsPath);
+	const fs = ctx.fs.assets.assetsByPath;
+	const cs = loadMetaCs(ctx.cs.assets);
+	using bar = createProgressBar('Assets', cs, fs);
+
+	return await processPlan<AssetMeta>({
+		create: write,
+		deletionStrategy: 'delete',
+		plan: planPull(cs, fs),
+		progress: bar,
+		remove: removeAsset.bind(null, assetsPath),
+		update: write,
+	});
+}
+
+async function removeAsset(assetsPath: string, { itemPath }: AssetMeta) {
+	const blobPath = getBlobPath(assetsPath, itemPath);
+	const metaPath = getMetaPath(assetsPath, itemPath);
+	const ui = getUi();
+
+	for (const path of [blobPath, metaPath]) {
+		try {
+			await rm(path);
+		} catch (ex) {
+			if (ex instanceof Error) {
+				if ('code' in ex && ex.code === 'ENOENT') {
+					continue;
+				}
+
+				ui.warn(`Failed to remove ${path}:`, ex.message);
+				continue;
+			}
+
+			ui.warn(`Failed to remove ${path}:`, ex);
+		}
+	}
+}