@arkebcacy/beacon-cli-temp 0.1.1

package/src/schema/assetFolders/lib/planPush.test.ts

@@ -0,0 +1,158 @@
+import TestLogContext from '#test/integration/lib/TestLogContext.js';
+import TestPushUiContext from '#test/integration/lib/TestPushUiContext.js';
+import { inspect } from 'node:util';
+import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
+import type AssetMeta from '../../assets/AssetMeta.js';
+import type FolderMeta from './FolderMeta.js';
+import type planPush from './planPush.js';
+
+const logs = new TestLogContext();
+const ui = new TestPushUiContext('fixtures/asset-filters', logs);
+vi.doMock(import('../../lib/SchemaUi.js'), () => ({ default: () => ui }));
+
+describe('Asset Folders: planPush', () => {
+	let sut: typeof planPush;
+
+	beforeAll(async () => {
+		sut = (await import('./planPush.js')).default;
+	});
+
+	afterEach(() => logs.clear());
+
+	// See test/integration/asset-filters/asset-filters.cs.test.md
+
+	// cs, fs, included, csChild, fsChild, expected
+	const theories: readonly Theory[] = [
+		new Theory(true, true, true, true, true, 'skip'),
+		new Theory(true, true, true, true, false, 'skip'),
+		new Theory(true, true, true, false, true, 'skip'),
+		new Theory(true, true, true, false, false, 'skip'),
+		new Theory(true, true, false, true, true, 'skip'),
+		new Theory(true, true, false, true, false, 'warning'),
+		new Theory(true, true, false, false, true, 'skip'),
+		new Theory(true, true, false, false, false, 'warning'),
+		new Theory(true, false, true, true, false, 'delete'),
+		new Theory(true, false, true, false, false, 'delete'),
+		new Theory(true, false, false, true, false, 'skip'),
+		new Theory(true, false, false, false, false, 'skip'),
+		new Theory(false, true, true, false, true, 'create'),
+		new Theory(false, true, true, false, false, 'create'),
+		new Theory(false, true, false, false, true, 'create'),
+		new Theory(false, true, false, false, false, 'warning'),
+	];
+
+	theories.forEach((theory) => {
+		it(inspect(theory, { colors: true, compact: true }), () => {
+			// Arrange
+			const { assets, cs, isIncluded, fs, itemPath } = arrange(theory);
+			ui.options.schema.assets = { isIncluded };
+
+			// Act
+			const actual = sut(cs, fs, assets);
+
+			// Assert
+			if (theory.expected === 'create') {
+				expect(actual.toCreate.keys()).toContain(itemPath);
+			} else {
+				expect(actual.toCreate.keys()).not.toContain(itemPath);
+			}
+
+			if (theory.expected === 'delete') {
+				expect(actual.toRemove.keys()).toContain(itemPath);
+			} else {
+				expect(actual.toRemove.keys()).not.toContain(itemPath);
+			}
+
+			if (theory.expected === 'skip' || theory.expected === 'warning') {
+				expect(actual.toSkip).toContain(itemPath);
+			} else {
+				expect(actual.toSkip).not.toContain(itemPath);
+			}
+
+			if (theory.expected === 'update') {
+				expect(actual.toUpdate.keys()).toContain(itemPath);
+			} else {
+				expect(actual.toUpdate.keys()).not.toContain(itemPath);
+			}
+
+			if (theory.expected === 'warning') {
+				expect(logs.warnings).toContainEqual(expect.stringContaining(itemPath));
+			} else {
+				expect(logs.warnings).not.toContainEqual(
+					expect.stringContaining(itemPath),
+				);
+			}
+		});
+	});
+});
+
+class Theory {
+	public constructor(
+		public readonly cs: boolean,
+		public readonly fs: boolean,
+		public readonly included: boolean,
+		public readonly csChild: boolean,
+		public readonly fsChild: boolean,
+		public readonly expected:
+			| 'create'
+			| 'delete'
+			| 'skip'
+			| 'update'
+			| 'warning',
+	) {}
+}
+
+function arrange(theory: Theory) {
+	const cs = new Map<string, FolderMeta>();
+	const fs = new Map<string, FolderMeta>();
+	const meta: FolderMeta = { itemPath: '/some/folder', name: 'folder' };
+	const assets = new Map<string, AssetMeta>();
+
+	if (theory.cs) {
+		cs.set(meta.itemPath, meta);
+	}
+
+	if (theory.fs) {
+		fs.set(meta.itemPath, meta);
+	}
+
+	if (theory.csChild) {
+		const child = createChild();
+		cs.set(child.itemPath, child);
+	}
+
+	if (theory.fsChild) {
+		const childFolder = createChild();
+		fs.set(childFolder.itemPath, childFolder);
+
+		const child: AssetMeta = {
+			fileSize: 123,
+			itemPath: `${childFolder.itemPath}/file.txt`,
+			tags: new Set(),
+			title: 'file.txt',
+		};
+
+		assets.set(child.itemPath, child);
+	}
+
+	const isIncluded = (path: string) => {
+		if (path === meta.itemPath) {
+			return theory.included;
+		}
+
+		if (theory.csChild || theory.fsChild) {
+			return path.startsWith('/some/folder/child/');
+		}
+
+		return false;
+	};
+
+	return { assets, cs, fs, isIncluded, itemPath: meta.itemPath };
+}
+
+function createChild(): FolderMeta {
+	return {
+		itemPath: '/some/folder/child/sub-folder',
+		name: 'sub-folder',
+	};
+}

package/src/schema/assetFolders/lib/planPush.ts

@@ -0,0 +1,132 @@
+import type MergePlan from '#cli/schema/xfer/lib/MergePlan.js';
+import type UiContext from '#cli/ui/UiContext.js';
+import { styleText } from 'node:util';
+import type AssetMeta from '../../assets/AssetMeta.js';
+import getUi from '../../lib/SchemaUi.js';
+import type FolderMeta from './FolderMeta.js';
+
+export default function planPush(
+	cs: ReadonlyMap<string, FolderMeta>,
+	fsFolders: ReadonlyMap<string, FolderMeta>,
+	fsAssets: ReadonlyMap<string, AssetMeta>,
+): MergePlan<FolderMeta> {
+	const fsResult = processFsItems(cs, fsFolders, fsAssets);
+	const csResult = processCsItems(cs, fsResult.seen);
+
+	return {
+		toCreate: fsResult.toCreate,
+		toRemove: csResult.toRemove,
+		toSkip: new Set([...fsResult.toSkip, ...csResult.toSkip]),
+		toUpdate: new Map(),
+	};
+}
+
+function hasIncludedChildren(
+	isIncluded: (path: string) => boolean,
+	keys: Iterable<string>,
+	itemPath: string,
+) {
+	for (const otherItemPath of keys) {
+		if (!otherItemPath.startsWith(itemPath + '/')) {
+			continue;
+		}
+
+		if (isIncluded(otherItemPath)) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
+function* allKeys(
+	fsFolders: ReadonlyMap<string, FolderMeta>,
+	fsAssets: ReadonlyMap<string, AssetMeta>,
+) {
+	for (const itemPath of fsFolders.keys()) {
+		yield itemPath;
+	}
+
+	for (const itemPath of fsAssets.keys()) {
+		if (fsFolders.has(itemPath)) {
+			continue;
+		}
+
+		yield itemPath;
+	}
+}
+
+function warning(ui: UiContext, itemPath: string) {
+	const msg1 = 'Skipping folder which exists in the file system,';
+	const msg2 = 'but is not included by filters, and contains no';
+	const msg3 = 'included children:';
+	const msg4 = styleText('yellowBright', itemPath);
+	ui.warn(msg1, msg2, msg3, msg4);
+}
+
+function processFsItems(
+	cs: ReadonlyMap<string, FolderMeta>,
+	fsFolders: ReadonlyMap<string, FolderMeta>,
+	fsAssets: ReadonlyMap<string, AssetMeta>,
+) {
+	const ui = getUi();
+	const { isIncluded } = ui.options.schema.assets;
+	const keys = allKeys.bind(null, fsFolders, fsAssets);
+	const hasChildren = (x: string) => hasIncludedChildren(isIncluded, keys(), x);
+	const seen = new Set<string>();
+	const toCreate = new Map<string, FolderMeta>();
+	const toSkip = new Set<string>();
+
+	for (const [itemPath, fsMeta] of fsFolders) {
+		seen.add(itemPath);
+		const csMeta = cs.get(itemPath);
+
+		if (csMeta) {
+			toSkip.add(itemPath);
+
+			if (!isIncluded(itemPath) && !hasChildren(itemPath)) {
+				warning(ui, itemPath);
+			}
+
+			continue;
+		}
+
+		if (isIncluded(itemPath)) {
+			toCreate.set(itemPath, fsMeta);
+			continue;
+		}
+
+		if (hasChildren(itemPath)) {
+			toCreate.set(itemPath, fsMeta);
+		} else {
+			toSkip.add(itemPath);
+			warning(ui, itemPath);
+		}
+	}
+
+	return { seen, toCreate, toSkip };
+}
+
+function processCsItems(
+	cs: ReadonlyMap<string, FolderMeta>,
+	seen: ReadonlySet<string>,
+) {
+	const ui = getUi();
+	const { isIncluded } = ui.options.schema.assets;
+	const toRemove = new Map<string, FolderMeta>();
+	const toSkip = new Set<string>();
+
+	for (const [itemPath, csMeta] of cs) {
+		if (seen.has(itemPath)) {
+			continue;
+		}
+
+		if (isIncluded(itemPath)) {
+			toRemove.set(itemPath, csMeta);
+		} else {
+			toSkip.add(itemPath);
+		}
+	}
+
+	return { toRemove, toSkip };
+}

package/src/schema/assetFolders/toContentstack.ts

@@ -0,0 +1,58 @@
+import type Assets from '#cli/cs/assets/Assets.js';
+import resolveRawAssetItem from '#cli/cs/assets/lib/resolveRawAssetItem.js';
+import { mkdir } from 'node:fs/promises';
+import schemaDirectory from '../assets/schemaDirectory.js';
+import type Ctx from '../ctx/Ctx.js';
+import createProgressBar from '../lib/createProgressBar.js';
+import getUi from '../lib/SchemaUi.js';
+import processPlan from '../xfer/lib/processPlan.js';
+import type FolderMeta from './lib/FolderMeta.js';
+import indexDestination from './lib/indexContentstack.js';
+import planPush from './lib/planPush.js';
+
+export default async function toContentstack(ctx: Ctx) {
+	const ui = getUi();
+	const assetsPath = schemaDirectory();
+	await mkdir(assetsPath, { recursive: true });
+
+	// Indexes of both source and destination must be in order. Depth-first or
+	// breadth-first doesn't matter, so long as no child ever comes before its
+	// parent.
+	const source = ctx.fs.assets.foldersByPath;
+	const destination = indexDestination(ctx.cs.assets);
+	using bar = createProgressBar('Asset Folders', source, destination);
+
+	return await processPlan<FolderMeta>({
+		create: create.bind(null, ctx.cs.assets),
+		deletionStrategy: ui.options.schema.deletionStrategy,
+		plan: planPush(destination, source, ctx.fs.assets.assetsByPath),
+		progress: bar,
+		remove: tryDeleteFolder.bind(null, ctx.cs.assets),
+		update: () => {
+			throw new Error('Not implemented');
+		},
+	});
+}
+
+async function create(assets: Assets, meta: FolderMeta) {
+	const parentPath = meta.itemPath.split('/').slice(0, -1).join('/');
+	const parent = resolveRawAssetItem(assets.byParentUid, parentPath);
+
+	if (parentPath && !parent) {
+		throw new Error(`Parent folder not found: ${parentPath}`);
+	}
+
+	await assets.createFolder(meta.name, parent?.uid ?? null);
+}
+
+async function tryDeleteFolder(assets: Assets, meta: FolderMeta) {
+	const item = resolveRawAssetItem(assets.byParentUid, meta.itemPath);
+
+	if (!item) {
+		// Probably already deleted because deleting a parent also deletes all
+		// descendants.
+		return;
+	}
+
+	await assets.deleteFolder(item.uid);
+}

package/src/schema/assets/AssetMeta.ts

@@ -0,0 +1,19 @@
+import type { RawAsset } from '#cli/cs/assets/Types.js';
+
+export default interface AssetMeta {
+	readonly description?: string;
+	readonly fileSize: number;
+	readonly itemPath: string;
+	readonly tags: ReadonlySet<string>;
+	readonly title: string;
+}
+
+export function fromRawAsset(item: RawAsset, itemPath: string): AssetMeta {
+	return {
+		...(item.description ? { description: item.description } : {}),
+		fileSize: parseInt(item.file_size, 10),
+		itemPath,
+		tags: new Set(item.tags),
+		title: item.title,
+	};
+}

package/src/schema/assets/assets.md

@@ -0,0 +1,138 @@
+# Dev Notes: Assets
+
+Q: Can I create a folder with a specific UID?
+A: No. There is a "Create Folder" endpoint but it does not accept a UID.
+
+As a result, UID values will be different for every stack.
+
+We must therefore match on name and not UID.
+
+This presents a difficulty because Contentstack allows duplicate names for
+both folders and files. This introduces ambiguity. Probably manageable with
+warnings and errors.
+
+Q: Can I create a folder using characters that are invalid in either Windows,
+Linux, or MacOS?
+
+A: No, an error message is returned:
+
+> Folder name can contain only alphanumeric characters ('A-Z', 'a-z', '0-9'),
+> hyphens ('-'), and/or underscores ('\_').`
+
+Therefore, folder names can map directly to file system names. Also, the
+documentation specifies that folder names may only nest five levels deep,
+so I think it will be ok to use the folder name as the path.
+
+Q: Can I create an asset and specify the UID?
+A: No, there is no API that allows this.
+
+We will need to match on names again.
+
+## Plan
+
+We can compare folder names with folder names present in the file system.
+
+Content types, entries, and RTE assets may all reference assets by UID.
+
+When we pull, when a reference to an Asset via UID is found, we must serialize
+it as a reference to the title / filename.
+
+We must issue errors for duplicate names.
+
+## File System Serialization
+
+Each asset is going to have two files: one for the metadata and one for
+the binary.
+
+Our serialization should be informed by the API used to create asset folders
+and files.
+
+The [Create Folder][1] API accepts this body:
+
+```json
+{
+  "asset": {
+    "name": "asset_folder_name",
+    "parent_uid": "asset_parent_folder_uid"
+  }
+}
+```
+
+There is an [Upload Asset][2] and a [Replace Asset][3] endpoint.
+
+Uploading and Replacing both accept _only_ these parameters:
+
+- `parent_uid` - Folder UID
+- `title`
+- `description`
+- `tags`
+
+Therefore, our serialization must contain _at least_ these fields.
+Except for `parent_uid`. We cannot trust the UIDs because we have no way
+to set them, so they will be different in every stack. Also, we shouldn't
+serialize them at all, because they will be derived from the name of the
+parent folder in the file system.
+
+### Thoughts about how these get named:
+
+- The sidecar and the blob need to sort together.
+
+- The blob's extension must be the "real" extension. This is to support tooling
+  that wishes to display the contents; i.e., doing a PR review and wanting to
+  see images.
+
+- The sidecar's extension must be YAML for a similar reason.
+
+- The blob's name should start with the actual name as reported by Contentstack.
+  I've already verified that Contentstack only permits valid characters in
+  the name.
+
+- Contentstack permits duplicate file names but I don't see an easy way to
+  support that. If we generate a requirement that file names must be unique, our
+  serialization problem becomes much easier to solve.
+
+- Therefore, the first naming solution is that the blob name should be as-is,
+  and the sidecar name should use the blob's name (including extension) with an
+  additional `.yaml` extension after it.
+
+- This introduces a possible ambiguity because it is possible for a blob to be
+  named such that it conflicts with a different blob's sidecar.
+  - I've considered separating the two into different directories but I reject
+    that because it makes PR reviews harder.
+
+  - I've considered attempting to use alternate data streams to tie the sidecar
+    directly to the file at a filesystem level but I reject that because Git
+    does not support it.
+
+  - I've considered a naming scheme that avoids the problem:
+    - I could use the Contentstack UID in the name. I reject this because the
+      UID is not human-readable, and because it changes on a per-developer
+      basis. It would only ever make sense for the dev who first serializes the
+      asset, then it is useless to everyone else.
+
+    - I could apply a prefix to both the blob and the sidecar file names:
+      - A simple prefix (like `asset-` or `metadata-`) would suffice, but I
+        reject that because it means the files no longer sort together.
+
+      - A hash of the file contents would be unique and would sort together, but
+        then we can no longer alpha-sort the files. It becomes much harder to
+        compare the source control contents with what remains in Contentstack.
+
+      - Oh, hey, I could just apply a suffix to both files. That would work.
+        Wish I could think of obvious solutions without going the long
+        way around.
+
+- So the final naming pattern is:
+  - Blob: `original-blob-name.blob.original-blob-extension`
+  - Sidecar: `original-blob-name.meta.original-blob-extension.yaml`
+
+## Change Detection
+
+See [Asset Modification Detection][6].
+
+[1]: https://www.contentstack.com/docs/developers/apis/content-management-api#create-a-folder 'Create a Folder'
+[2]: https://www.contentstack.com/docs/developers/apis/content-management-api#upload-asset 'Upload Asset'
+[3]: https://www.contentstack.com/docs/developers/apis/content-management-api#replace-asset 'Replace Asset'
+[4]: https://www.contentstack.com/docs/developers/apis/content-management-api#delete-asset 'Delete Asset'
+[5]: https://www.contentstack.com/docs/developers/apis/content-management-api#get-all-assets 'Get All Assets'
+[6]: ../../../../doc/lessons-learned/asset-modification-detection.md 'Asset Modification Detection'

package/src/schema/assets/lib/MetaSerialization.ts

@@ -0,0 +1,61 @@
+import readYaml from '#cli/fs/readYaml.js';
+import writeYaml from '#cli/fs/writeYaml.js';
+import isRecord from '#cli/util/isRecord.js';
+import { stat } from 'node:fs/promises';
+import type AssetMeta from '../AssetMeta.js';
+import { getMetaPath } from './NamingConvention.js';
+
+interface RawMeta {
+	readonly description?: string;
+	readonly tags?: readonly string[];
+	readonly title: string;
+}
+
+function isRawMeta(o: unknown): o is RawMeta & Record<string, unknown> {
+	return (
+		isRecord(o) &&
+		(!('description' in o) || typeof o.description === 'string') &&
+		(!('tags' in o) ||
+			(Array.isArray(o.tags) && o.tags.every((t) => typeof t === 'string'))) &&
+		typeof o.title === 'string'
+	);
+}
+
+export async function load(paths: {
+	readonly blobPath: string;
+	readonly itemPath: string;
+	readonly metaPath: string;
+}): Promise<AssetMeta> {
+	const [parsed, blobStats] = await Promise.all([
+		readYaml(paths.metaPath),
+		stat(paths.blobPath),
+	]);
+
+	if (!isRawMeta(parsed)) {
+		throw new Error(`Invalid asset metadata: ${paths.metaPath}`);
+	}
+
+	const { description, tags, title } = parsed;
+
+	return {
+		...(description ? { description } : {}),
+		fileSize: blobStats.size,
+		itemPath: paths.itemPath,
+		tags: new Set(tags ?? []),
+		title,
+	};
+}
+
+export async function save(assetsPath: string, asset: AssetMeta) {
+	const raw: RawMeta = {
+		...(typeof asset.description === 'string'
+			? { description: asset.description }
+			: {}),
+
+		...(asset.tags.size > 0 ? { tags: [...asset.tags].sort() } : {}),
+
+		title: asset.title,
+	};
+
+	return writeYaml(getMetaPath(assetsPath, asset.itemPath), raw);
+}

package/src/schema/assets/lib/NamingConvention.test.ts

@@ -0,0 +1,49 @@
+import { describe, expect, it, vi } from 'vitest';
+import { getBlobPath, getItemPath, getMetaPath } from './NamingConvention.js';
+
+vi.mock(import('node:path'), async () => {
+	const winPath = await import('node:path/win32');
+	return winPath;
+});
+
+describe('NamingConvention', () => {
+	it('can map an absolute path to a blob path on Windows', () => {
+		// Act
+		const result = getBlobPath('C:\\cs\\assets\\file.txt');
+
+		// Assert
+		expect(result).toBe('C:\\cs\\assets\\file.blob.txt');
+	});
+
+	it('can map an item path to a blob path on Windows', () => {
+		// Act
+		const assetsPath = 'C:\\cs\\assets';
+		const itemPath = 'events/some-file.txt';
+		const result = getBlobPath(assetsPath, itemPath);
+
+		// Assert
+		expect(result).toBe('C:\\cs\\assets\\events\\some-file.blob.txt');
+	});
+
+	it('can map an item path to a meta path on Windows', () => {
+		// Act
+		const assetsPath = 'C:\\cs\\assets';
+		const itemPath = 'events/some-file.txt';
+		const result = getMetaPath(assetsPath, itemPath);
+
+		// Assert
+		expect(result).toBe('C:\\cs\\assets\\events\\some-file.meta.txt.yaml');
+	});
+
+	it('can map an absolute path to an item path on Windows', () => {
+		// Arrange
+		const assetsPath = 'C:\\cs\\assets';
+		const absPath = 'C:\\cs\\assets\\events\\some-file.blob.txt';
+
+		// Act
+		const result = getItemPath(assetsPath, absPath);
+
+		// Assert
+		expect(result).toBe('events/some-file.txt');
+	});
+});