@arkebcacy/beacon-cli-temp 0.1.1

package/src/schema/xfer/lib/handleDeletions.ts

@@ -0,0 +1,65 @@
+import ProgressReporter from '#cli/ui/progress/ProgressReporter.js';
+import type { TransferContext } from './processPlan.js';
+
+interface Result {
+	readonly deleted?: ReadonlySet<string>;
+	readonly unmodified?: ReadonlySet<string>;
+}
+
+export default async function handleDeletions<TItem>(
+	ctx: TransferContext<TItem>,
+	collectError: (key: string, ex: unknown) => void,
+): Promise<Result> {
+	if (ctx.deletionStrategy === 'ignore') {
+		return handleIgnoredDeletions(ctx);
+	}
+
+	if (ctx.deletionStrategy === 'warn') {
+		return handleWarnings(ctx);
+	}
+
+	const deleted = new Set<string>();
+
+	for (const [key, item] of ctx.plan.toRemove) {
+		using reporter = new ProgressReporter(ctx.progress, 'deleting', key);
+
+		try {
+			await ctx.remove(item);
+			deleted.add(key);
+		} catch (ex: unknown) {
+			collectError(key, ex);
+		} finally {
+			ctx.progress.increment();
+			reporter.finish('deleted');
+		}
+	}
+
+	return { deleted };
+}
+
+function handleWarnings<TItem>(ctx: TransferContext<TItem>) {
+	const unmodified = new Set<string>();
+
+	for (const [key] of ctx.plan.toRemove) {
+		ctx.progress.update({
+			action: 'skipping',
+			key: `deletion of ${key}`,
+		});
+
+		ctx.progress.increment();
+		unmodified.add(key);
+	}
+
+	return { unmodified };
+}
+
+function handleIgnoredDeletions<TItem>(ctx: TransferContext<TItem>) {
+	ctx.progress.update({
+		action: 'skipping',
+		key: `${ctx.plan.toRemove.size.toLocaleString()} deletions`,
+	});
+
+	ctx.progress.increment(ctx.plan.toRemove.size);
+	const unmodified = new Set<string>([...ctx.plan.toRemove.keys()]);
+	return { unmodified };
+}

package/src/schema/xfer/lib/handleUnmodified.ts

@@ -0,0 +1,15 @@
+import type { TransferContext } from './processPlan.js';
+
+export default function handleUnmodified<TItem>(ctx: TransferContext<TItem>) {
+	const unmodified = new Set(ctx.plan.toSkip.keys());
+
+	if (unmodified.size > 0) {
+		const noun = unmodified.size === 1 ? 'item' : 'items';
+		const key = `${unmodified.size.toLocaleString()} unmodified ${noun}`;
+
+		ctx.progress.update({ action: 'skipping', key });
+		ctx.progress.increment(unmodified.size);
+	}
+
+	return unmodified;
+}

package/src/schema/xfer/lib/handleUpdates.ts

@@ -0,0 +1,25 @@
+import ProgressReporter from '#cli/ui/progress/ProgressReporter.js';
+import type { TransferContext } from './processPlan.js';
+
+export default async function handleUpdates<TItem>(
+	ctx: TransferContext<TItem>,
+	collectError: (key: string, ex: unknown) => void,
+) {
+	const updated = new Set<string>();
+
+	for (const [key, item] of ctx.plan.toUpdate) {
+		using reporter = new ProgressReporter(ctx.progress, 'updating', key);
+
+		try {
+			await ctx.update(item);
+			updated.add(key);
+		} catch (ex: unknown) {
+			collectError(key, ex);
+		} finally {
+			ctx.progress.increment();
+			reporter.finish('updated');
+		}
+	}
+
+	return updated;
+}

package/src/schema/xfer/lib/planMerge.ts

@@ -0,0 +1,31 @@
+import type MergePlan from './MergePlan.js';
+
+export default function planMerge<TItem>(
+	equality: (a: TItem, b: TItem) => boolean,
+	source: ReadonlyMap<string, TItem>,
+	destination: ReadonlyMap<string, TItem>,
+): MergePlan<TItem> {
+	const toCreate = new Map<string, TItem>();
+	const toUpdate = new Map<string, TItem>();
+	const toSkip = new Set<string>();
+
+	const toRemove = [...destination]
+		.filter(([key]) => !source.has(key))
+		.reduce((acc, [key, item]) => acc.set(key, item), new Map<string, TItem>());
+
+	for (const [key, item] of source) {
+		const existing = destination.get(key);
+
+		if (existing) {
+			if (equality(item, existing)) {
+				toSkip.add(key);
+			} else {
+				toUpdate.set(key, item);
+			}
+		} else {
+			toCreate.set(key, item);
+		}
+	}
+
+	return { toCreate, toRemove, toSkip, toUpdate };
+}

package/src/schema/xfer/lib/processPlan.ts

@@ -0,0 +1,58 @@
+import type ProgressBar from '#cli/ui/progress/ProgressBar.js';
+import type TransferResults from '../TransferResults.js';
+import { merge, MutableTransferResults } from '../TransferResults.js';
+import handleCreations from './handleCreations.js';
+import handleDeletions from './handleDeletions.js';
+import handleUnmodified from './handleUnmodified.js';
+import handleUpdates from './handleUpdates.js';
+import type MergePlan from './MergePlan.js';
+
+export interface TransferContext<TItem> {
+	readonly create: (item: TItem) => Promise<unknown>;
+	readonly deletionStrategy: 'delete' | 'ignore' | 'warn';
+	readonly plan: MergePlan<TItem>;
+	readonly progress: ProgressBar;
+	readonly remove: (item: TItem) => Promise<void>;
+	readonly update: (item: TItem) => Promise<unknown>;
+}
+
+export default async function processPlan<TItem>(
+	ctx: TransferContext<TItem>,
+): Promise<TransferResults> {
+	const total =
+		ctx.plan.toCreate.size + ctx.plan.toRemove.size + ctx.plan.toUpdate.size;
+
+	const unmodified = handleUnmodified(ctx);
+
+	if (total === 0) {
+		const result = new MutableTransferResults();
+		unmodified.forEach((key) => result.unmodified.add(key));
+		return result;
+	}
+
+	const errored = new Map<string, unknown>();
+	const collectError = (key: string, ex: unknown) => errored.set(key, ex);
+
+	// The order of deletions, creations, and updates is important:
+	//
+	// When transferring entries from Contentstack to the filesystem, we cannot
+	// use the UID as a filename because it is not stable as a key. Instead, we
+	// sanitize the title and use that. However, part of sanitizing means we must
+	// ensure the titles are unique filenames. During this process, titles are
+	// compared for uniqueness only against the files that _will eventually_ exist
+	// _after_ the transfer.
+	//
+	// Thus, there is a chance that a filename will _not_ be unique when compared
+	// to the files that _currently_ exist.
+	//
+	// To avoid this, we must remove any extra files before we create new ones.
+	//
+	// Also, when processing updates, if any updated schemas refer to content
+	// types that have not yet been created, Contentstack will throw an error.
+	// To help avoid this, we make sure creations are done before updates.
+	const deleted = await handleDeletions(ctx, collectError);
+	const created = await handleCreations(ctx, collectError);
+	const updated = await handleUpdates(ctx, collectError);
+
+	return merge({ created, errored, unmodified, updated }, deleted);
+}

package/src/tsconfig.json

@@ -0,0 +1,15 @@
+{
+	"compilerOptions": {
+		"baseUrl": ".",
+		"composite": true,
+		"outDir": "../dist",
+		"rootDir": ".",
+		"skipLibCheck": true,
+		"paths": {
+			"#cli/*": ["*"]
+		}
+	},
+	"exclude": ["**/*.test.ts"],
+	"extends": "../../tsconfig.base.json",
+	"include": ["**/*.ts"]
+}

package/src/ui/HandledError.ts

@@ -0,0 +1,24 @@
+import { stripVTControlCharacters } from 'node:util';
+
+// To be used when an error condition should halt execution and display an
+// error message, but not a stack trace. Please ensure the provided message
+// is descriptive enough to track down.
+export default class HandledError extends Error {
+	public constructor(public readonly humanized: string) {
+		super(stripVTControlCharacters(humanized));
+		this.name = 'HandledError';
+	}
+
+	public static async ExitIfThrown(fn: () => Promise<void>): Promise<void> {
+		try {
+			await fn();
+		} catch (ex: unknown) {
+			if (ex instanceof HandledError) {
+				console.error(ex.humanized);
+				process.exit(1);
+			}
+
+			throw ex;
+		}
+	}
+}

package/src/ui/LogContext.ts

@@ -0,0 +1,5 @@
+export default interface LogContext {
+	readonly error: Console['warn'];
+	readonly info: Console['info'];
+	readonly warn: Console['warn'];
+}

package/src/ui/Options.ts

@@ -0,0 +1,33 @@
+export default interface Options {
+	readonly client: {
+		readonly apiKey: string;
+		readonly baseUrl: URL;
+		readonly branch: string;
+		readonly managementToken: string;
+		readonly timeout: number;
+	};
+	readonly schema: {
+		readonly assets: {
+			readonly isIncluded: (path: string) => boolean;
+		};
+		readonly deletionStrategy: 'delete' | 'ignore' | 'warn';
+		readonly extension: {
+			readonly byName: ReadonlyMap<string, string>;
+			readonly byUid: ReadonlyMap<string, string>;
+		};
+		readonly jsonRtePlugin: {
+			readonly byName: ReadonlyMap<string, string>;
+			readonly byUid: ReadonlyMap<string, string>;
+		};
+		readonly schemaPath: string;
+		readonly taxonomies: TaxonomyStrategies;
+	};
+	readonly verbose: boolean;
+}
+
+export type TaxonomyStrategy = 'only taxonomy' | 'taxonomy and terms';
+export type TaxonomyStrategies = ReadonlyMap<string, TaxonomyStrategy>;
+
+export const DefaultTaxonomyStrategies: TaxonomyStrategies = new Map([
+	['*', 'taxonomy and terms'],
+]);

package/src/ui/PartialOptions.ts

@@ -0,0 +1,19 @@
+import type Options from './Options.js';
+
+// Justification: Use of Function is intentional because we wish it to apply to
+// all functions, not just those with a specific signature.
+/* eslint-disable @typescript-eslint/no-unsafe-function-type */
+
+type RecursivePartial<T> = {
+	[P in keyof T]?: T[P] extends
+		| Function
+		| ReadonlyMap<string, string>
+		| ReadonlySet<string>
+		| URL
+		? T[P] | undefined
+		: T[P] extends object
+			? RecursivePartial<T[P]> | undefined
+			: T[P] | undefined;
+};
+
+export type PartialOptions = RecursivePartial<Options>;

package/src/ui/UiContext.ts

@@ -0,0 +1,77 @@
+import { MultiBar } from 'cli-progress';
+import type LogContext from './LogContext.js';
+import type Options from './Options.js';
+import createEmitter from './progress/createEmitter.js';
+import formatProgress from './progress/formatProgress.js';
+import type ProgressBar from './progress/ProgressBar.js';
+import UiProgressBar from './progress/UiProgressBar.js';
+import VerboseProgressBar from './progress/VerboseProgressBar.js';
+
+export default interface UiContext extends LogContext {
+	readonly options: Options;
+	createProgressBar(name: string, total: number): ProgressBar;
+}
+
+export class ConsoleUiContext implements UiContext, Disposable {
+	readonly error: Console['warn'];
+	readonly info: Console['info'];
+	readonly warn: Console['warn'];
+
+	maxElapsedLength: number = '999 ms'.length;
+	#barContainer: MultiBar | undefined;
+	#maxNameLength = -Infinity;
+	#maxTotalLength = '999'.length;
+
+	public constructor(public readonly options: Options) {
+		this.error = createEmitter(this, 'error');
+		this.info = createEmitter(this, 'info');
+		this.warn = createEmitter(this, 'warn');
+	}
+
+	get barContainer() {
+		return this.#barContainer;
+	}
+
+	get maxProgressNameLength() {
+		return this.#maxNameLength;
+	}
+
+	get maxTotalLength() {
+		return this.#maxTotalLength;
+	}
+
+	createProgressBar(name: string, total: number): ProgressBar {
+		const totalLength = total.toLocaleString().length;
+
+		this.#maxNameLength = Math.max(this.#maxNameLength, name.length);
+		this.#maxTotalLength = Math.max(this.#maxTotalLength, totalLength);
+
+		if (this.options.verbose) {
+			return new VerboseProgressBar(this, name, total);
+		}
+
+		const container = this.#getOrCreateBarContainer();
+		return new UiProgressBar(name, container, total);
+	}
+
+	stopAllBars() {
+		this.#barContainer?.stop();
+		this.#barContainer = undefined;
+	}
+
+	[Symbol.dispose](): void {
+		this.stopAllBars();
+	}
+
+	#getOrCreateBarContainer(): MultiBar {
+		return (this.#barContainer ??= new MultiBar({
+			clearOnComplete: false,
+			forceRedraw: true,
+			format: formatProgress.bind(null, this),
+			gracefulExit: true,
+			hideCursor: true,
+			stopOnComplete: true,
+			stream: process.stdout,
+		}));
+	}
+}

package/src/ui/UiOptions.test.ts

@@ -0,0 +1,45 @@
+import { describe, expect, it } from 'vitest';
+import UiOptions from './UiOptions.js';
+
+describe('UiOptions', () => {
+	it('overrides client and schema with environment values', () => {
+		// Arrange
+		const defaultValues = {
+			client: { branch: 'main', timeout: 1000 },
+			schema: { deletionStrategy: 'warn' as const, schemaPath: 'default-path' },
+		};
+
+		const fromConfigFile = {
+			client: {
+				apiKey: 'dev-key',
+				baseUrl: new URL('https://dev.com'),
+				branch: 'dev-branch',
+				timeout: 5000,
+			},
+			schema: { schemaPath: 'dev-path' },
+		};
+
+		const fromCommandEnvironment = {
+			client: {
+				managementToken: 'cli-token',
+				timeout: 10000,
+			},
+			schema: { schemaPath: 'cli-path' },
+		};
+
+		// Act
+		const options = new UiOptions(
+			defaultValues,
+			fromConfigFile,
+			fromCommandEnvironment,
+		);
+
+		// Assert
+		expect(options.client.apiKey).toBe('dev-key');
+		expect(options.client.baseUrl).toEqual(new URL('https://dev.com'));
+		expect(options.client.branch).toBe('dev-branch');
+		expect(options.client.managementToken).toBe('cli-token');
+		expect(options.client.timeout).toBe(fromCommandEnvironment.client.timeout);
+		expect(options.schema.schemaPath).toBe('cli-path');
+	});
+});

package/src/ui/UiOptions.ts

@@ -0,0 +1,84 @@
+import type Options from './Options.js';
+import { DefaultTaxonomyStrategies } from './Options.js';
+import type { PartialOptions } from './PartialOptions.js';
+import { defaultValue as defaultTimeout } from './option/apiTimeout.js';
+import { defaultValue as defaultBranch } from './option/branch.js';
+import { defaultValue as defaultStrategy } from './option/deletionStrategy.js';
+import { defaultValue as defaultSchemaPath } from './option/schemaPath.js';
+
+export default class UiOptions implements Options {
+	public readonly client: Options['client'];
+	public readonly schema: Options['schema'];
+	public readonly verbose: Options['verbose'];
+
+	public constructor(...others: PartialOptions[]) {
+		this.client = client(...others.map((o) => o.client));
+		this.schema = schema(...others.map((o) => o.schema));
+
+		const top = topLevel(...others);
+		this.verbose = top.verbose;
+	}
+}
+
+function topLevel(...others: PartialOptions[]) {
+	const other = mergeExceptUndefined(...others);
+
+	return {
+		verbose: other.verbose ?? false,
+	};
+}
+
+function client(...others: PartialOptions['client'][]): Options['client'] {
+	const other = mergeExceptUndefined(...others);
+
+	return {
+		apiKey: other.apiKey ?? '',
+		baseUrl: other.baseUrl ?? new URL('http://localhost'),
+		branch: other.branch ?? defaultBranch,
+		managementToken: other.managementToken ?? '',
+		timeout: other.timeout ?? defaultTimeout,
+	};
+}
+
+function schema(...others: PartialOptions['schema'][]): Options['schema'] {
+	const other = mergeExceptUndefined(...others);
+
+	return {
+		assets: assets(...others.map((o) => o?.assets)),
+		deletionStrategy: other.deletionStrategy ?? defaultStrategy,
+		extension: maps(...others.map((o) => o?.extension)),
+		jsonRtePlugin: maps(...others.map((o) => o?.jsonRtePlugin)),
+		schemaPath: other.schemaPath ?? defaultSchemaPath,
+		taxonomies: other.taxonomies ?? DefaultTaxonomyStrategies,
+	};
+}
+
+type AssetOptions = NonNullable<PartialOptions['schema']>['assets'];
+function assets(...others: AssetOptions[]): Options['schema']['assets'] {
+	const other = mergeExceptUndefined(...others);
+	return { isIncluded: other.isIncluded ?? (() => true) };
+}
+
+type MappedOptions = NonNullable<PartialOptions['schema']>['extension'];
+function maps(...others: MappedOptions[]): Options['schema']['extension'] {
+	const other = mergeExceptUndefined(...others);
+
+	return {
+		byName: other.byName ?? new Map<string, string>(),
+		byUid: other.byUid ?? new Map<string, string>(),
+	};
+}
+
+function mergeExceptUndefined<T extends Record<string, unknown>>(
+	...others: (T | undefined)[]
+): T {
+	const valueIsDefined = ([, value]: [unknown, unknown]) => value !== undefined;
+
+	const definedEntries = (x: Record<string, unknown>) =>
+		Object.entries(x).filter(valueIsDefined);
+
+	return Object.assign(
+		{},
+		...others.map((x) => (x ? Object.fromEntries(definedEntries(x)) : {})),
+	) as T;
+}

package/src/ui/command/clear.ts

@@ -0,0 +1,78 @@
+import resolveConfig from '#cli/cfg/resolveConfig.js';
+import { createClient } from '#cli/cs/api/Client.js';
+import clearJob from '#cli/schema/clear.js';
+import { Store } from '#cli/schema/lib/SchemaUi.js';
+import createStylus from '#cli/ui/createStylus.js';
+import HandledError from '#cli/ui/HandledError.js';
+import logApiPerformance from '#cli/ui/logApiPerformance.js';
+import { ConsoleUiContext } from '#cli/ui/UiContext.js';
+import { Command } from 'commander';
+import logOptions from '../logOptions.js';
+import type { CommonOptions } from '../option/commonOptions.js';
+import { addCommonOptions } from '../option/commonOptions.js';
+
+const clear = new Command('clear');
+
+addCommonOptions(clear);
+clear.description('Empty all data from a stack.');
+
+clear.action(async (cliOptions: CommonOptions) =>
+	HandledError.ExitIfThrown(async () => {
+		const options = await mapOptions(cliOptions);
+		using ui = new ConsoleUiContext(options);
+		ui.info(logStart(cliOptions, options));
+
+		const histogram = await Store.run(ui, async () => {
+			await using client = createClient(ui);
+			await clearJob(client, ui);
+			return client.performance;
+		});
+
+		ui.stopAllBars();
+		logApiPerformance(ui, histogram);
+	}),
+);
+
+export default clear;
+
+async function mapOptions(options: CommonOptions) {
+	return resolveConfig({
+		client: {
+			apiKey: options.apiKey,
+			baseUrl: options.baseUrl,
+			branch: options.branch,
+			managementToken: options.managementToken,
+			timeout: options.apiTimeout,
+		},
+		configFile: options.configFile,
+		namedEnvironment: options.environment,
+		schema: {
+			deletionStrategy: 'delete',
+		},
+		verbose: options.verbose,
+	});
+}
+
+function logStart(
+	cliOptions: CommonOptions,
+	options: Awaited<ReturnType<typeof mapOptions>>,
+) {
+	const b = createStylus('bold');
+	const y = createStylus('yellowBright');
+	const parts = [b`\n${'Emptying'} `];
+
+	if (cliOptions.environment) {
+		parts.push(y`${cliOptions.environment} (${options.client.apiKey}) `);
+	} else {
+		parts.push(y`${options.client.apiKey} `);
+	}
+
+	parts.push(
+		b`${'stack on the'} `,
+		y`${options.client.branch}`,
+		b` ${'branch'}:\n`,
+		logOptions(options),
+	);
+
+	return parts.join('');
+}

package/src/ui/command/pull.ts

@@ -0,0 +1,95 @@
+import resolveConfig from '#cli/cfg/resolveConfig.js';
+import { createClient } from '#cli/cs/api/Client.js';
+import { Store } from '#cli/schema/lib/SchemaUi.js';
+import pullJob from '#cli/schema/pull.js';
+import SchemaOperationError from '#cli/schema/SchemaOperationError.js';
+import createStylus from '#cli/ui/createStylus.js';
+import HandledError from '#cli/ui/HandledError.js';
+import humanizePath from '#cli/ui/humanizePath.js';
+import logApiPerformance from '#cli/ui/logApiPerformance.js';
+import logResults from '#cli/ui/logResults.js';
+import * as Options from '#cli/ui/option/index.js';
+import { ConsoleUiContext } from '#cli/ui/UiContext.js';
+import { Command } from 'commander';
+import { resolve } from 'node:path';
+import logOptions from '../logOptions.js';
+import type { CommonOptions } from '../option/commonOptions.js';
+import { addCommonOptions } from '../option/commonOptions.js';
+
+const pull = new Command('pull');
+addCommonOptions(pull);
+
+pull
+	.addOption(Options.extension)
+	.addOption(Options.jsonRtePlugin)
+	.addOption(Options.schemaPath)
+	.description('Serialize data and schema from a stack into the file system.');
+
+type CommandOptions = CommonOptions &
+	Options.ExtensionOption &
+	Options.JsonRtePluginOption &
+	Options.SchemaPathOption;
+
+pull.action(async (cliOptions: CommandOptions) =>
+	HandledError.ExitIfThrown(async () => {
+		const options = await mapOptions(cliOptions);
+		using ui = new ConsoleUiContext(options);
+		ui.info(logStart(cliOptions, options));
+
+		const { histogram, results } = await Store.run(ui, async () => {
+			await using client = createClient(ui);
+			const r = await pullJob(client);
+			return { histogram: client.performance, results: r };
+		});
+
+		ui.stopAllBars();
+		logResults(ui, results);
+		logApiPerformance(ui, histogram);
+		SchemaOperationError.throwIfFailuresExist(results.values());
+	}),
+);
+
+export default pull;
+
+async function mapOptions(options: CommandOptions) {
+	return resolveConfig({
+		client: {
+			apiKey: options.apiKey,
+			baseUrl: options.baseUrl,
+			branch: options.branch,
+			managementToken: options.managementToken,
+			timeout: options.apiTimeout,
+		},
+		configFile: options.configFile,
+		namedEnvironment: options.environment,
+		schema: {
+			deletionStrategy: 'delete',
+			extension: options.extension,
+			jsonRtePlugin: options.jsonRtePlugin,
+			schemaPath: options.schemaPath,
+		},
+		verbose: options.verbose,
+	});
+}
+
+function logStart(
+	cliOptions: CommandOptions,
+	options: Awaited<ReturnType<typeof mapOptions>>,
+) {
+	const y = createStylus('yellowBright');
+	const parts = ['\nSerializing schema from'];
+
+	if (cliOptions.environment) {
+		parts.push(y` ${cliOptions.environment} (${options.client.apiKey})`);
+	} else {
+		parts.push(y` ${options.client.apiKey}`);
+	}
+
+	parts.push(
+		y` on the ${options.client.branch} branch into `,
+		y`${humanizePath(resolve(options.schema.schemaPath))}:\n`,
+		logOptions(options),
+	);
+
+	return parts.join('');
+}