@arkebcacy/beacon-cli-temp 0.1.1

package/src/schema/isEquivalentSchema.ts

@@ -0,0 +1,21 @@
+import type { Schema } from '#cli/cs/Types.js';
+import { isDeepStrictEqual } from 'node:util';
+import normalize from './normalize.js';
+
+// The updated_at and _version fields appear in Contentstack schema responses,
+// and these values change with every update to the schema. Sadly, this happens
+// even for importing/exporting schemas (i.e., our CI/CD pipeline). As a result,
+// we cannot depend on these fields to determine whether a schema has changed,
+// because the "change" may just be a re-import of the same schema.
+//
+// Instead, this function compares schemas using deep equality, while omitting
+// fields that we don't actually care about.
+//
+// This is also necessary because the "import/export" data structure differs
+// from the "read" data structure. We decide if a schema needs to be imported
+// or exported by comparing the contents of the "read" structure with the
+// contents of the most recent import/export. These structures are _mostly_
+// the same, but not entirely.
+export default function isEquivalentSchema(a: Schema, b: Schema) {
+	return isDeepStrictEqual(normalize(a), normalize(b));
+}

package/src/schema/lib/PushResults.ts

@@ -0,0 +1,17 @@
+import type TransferResults from '../xfer/TransferResults.js';
+
+export default class PushResults {
+	readonly #results = new Map<string, PromiseSettledResult<TransferResults>>();
+
+	public get value(): ReadonlyMap<
+		string,
+		PromiseSettledResult<TransferResults>
+	> {
+		return this.#results;
+	}
+
+	public async set(name: string, promise: Promise<TransferResults>) {
+		const [result] = await Promise.allSettled([promise]);
+		this.#results.set(name, result);
+	}
+}

package/src/schema/lib/SchemaUi.ts

@@ -0,0 +1,14 @@
+import type UiContext from '#cli/ui/UiContext.js';
+import { AsyncLocalStorage } from 'node:async_hooks';
+
+export const Store = new AsyncLocalStorage<UiContext>();
+
+export default function getUi(): UiContext {
+	const ui = Store.getStore();
+
+	if (!ui) {
+		throw new Error(`${getUi.name}() must be called within a UI context`);
+	}
+
+	return ui;
+}

package/src/schema/lib/createProgressBar.ts

@@ -0,0 +1,9 @@
+import getUi from './SchemaUi.js';
+
+export default function createProgressBar(
+	name: string,
+	...indicies: readonly ReadonlyMap<unknown, unknown>[]
+) {
+	const totalSize = new Set(indicies.flatMap((x) => [...x.keys()])).size;
+	return getUi().createProgressBar(name, totalSize);
+}

package/src/schema/lib/pullModules.ts

@@ -0,0 +1,53 @@
+import type { ContentType } from '#cli/cs/content-types/Types.js';
+import assets from '../assets/toFilesystem.js';
+import contentTypes from '../content-types/toFilesystem.js';
+import type Ctx from '../ctx/Ctx.js';
+import clean from '../entries/clean.js';
+import entries from '../entries/toFilesystem.js';
+import globalFields from '../global-fields/toFilesystem.js';
+import taxonomies from '../taxonomies/toFilesystem.js';
+import type TransferResults from '../xfer/TransferResults.js';
+import getUi from './SchemaUi.js';
+
+export default async function* pullModules(
+	ctx: Ctx,
+): AsyncGenerator<readonly [string, Promise<TransferResults>]> {
+	yield ['Assets', assets(ctx)];
+	yield ['Global Fields', globalFields(ctx)];
+	yield ['Taxonomies', taxonomies(ctx)];
+	yield ['Content Types', contentTypes(ctx)];
+
+	const summary = summarizeContentTypes(ctx);
+	const total = [...summary.values()].reduce((acc, count) => acc + count, 0);
+	const ui = getUi();
+
+	{
+		using bar = ui.createProgressBar('Entries', total);
+
+		for (const contentType of summary.keys()) {
+			const task = entries(ctx, contentType, bar);
+			yield [`${contentType.title} Entries`, task];
+			await Promise.allSettled([task]);
+		}
+	}
+
+	yield [
+		'Stale Entries',
+		clean(ui.options.schema.schemaPath, new Set(ctx.cs.contentTypes.keys())),
+	];
+}
+
+function summarizeContentTypes(ctx: Ctx): ReadonlyMap<ContentType, number> {
+	const sorter = new Intl.Collator();
+
+	return [...ctx.cs.contentTypes.values()]
+		.sort((a, b) => sorter.compare(a.title, b.title))
+		.reduce((acc, contentType) => {
+			const titles = new Set([
+				...ctx.cs.entries.byTitleFor(contentType.uid).keys(),
+				...ctx.fs.entries.byTitleFor(contentType.uid).keys(),
+			]);
+
+			return acc.set(contentType, titles.size);
+		}, new Map<ContentType, number>());
+}

package/src/schema/normalize.test.ts

@@ -0,0 +1,20 @@
+import type { Schema } from '#cli/cs/Types.js';
+import readFixture from '#test/fixtures/readFixture';
+import getSnapshotPath from '#test/snapshots/getSnapshotPath';
+import { expect, test } from 'vitest';
+import { parse, stringify } from 'yaml';
+import normalize from './normalize.js';
+
+test('Normalization of an exported schema should match snapshot', async () => {
+	// Arrange
+	const fixtureYaml = await readFixture('navigation-export.yaml');
+	const exportFixture = parse(fixtureYaml) as Schema;
+
+	// Act
+	const normalized = normalize(exportFixture);
+
+	// Assert
+	const output = stringify(normalized, { sortMapEntries: true });
+	const snapPath = getSnapshotPath('normalize-exported-schema.yaml');
+	await expect(output).toMatchFileSnapshot(snapPath);
+});

package/src/schema/normalize.ts

@@ -0,0 +1,69 @@
+import type { Schema } from '#cli/cs/Types.js';
+
+// Contentstack observed providing differing values for these fields based on
+// whether an "import" or a "read" operation was performed:
+//
+//   - DEFAULT_ACL
+//   - SYS_ACL
+//
+// Last observed: 2024-07-26.
+export default function normalize({
+	description,
+	options,
+	schema,
+	title,
+	uid,
+}: Schema) {
+	return {
+		description,
+		options,
+		schema: normalizeSchema(schema),
+		title,
+		uid,
+	};
+}
+
+function normalizeSchema(schema: unknown): unknown {
+	if (schema === undefined) {
+		return schema;
+	}
+
+	if (Array.isArray(schema)) {
+		return schema.map(normalizeSchema);
+	}
+
+	if (typeof schema !== 'object' || schema === null) {
+		throw new Error('Invalid schema');
+	}
+
+	const {
+		inbuilt_model,
+		indexed,
+		schema: nestedSchema,
+		...rest
+	} = schema as Record<string, unknown>;
+
+	return {
+		...rest,
+		...(nestedSchema ? { schema: normalizeSchema(nestedSchema) } : {}),
+		...normalizeReferenceTo(rest),
+	};
+}
+
+function normalizeReferenceTo(schema: Record<string, unknown>) {
+	if (schema.data_type !== 'reference') {
+		return {};
+	}
+
+	const { reference_to } = schema;
+
+	if (!reference_to) {
+		return {};
+	}
+
+	if (!Array.isArray(reference_to)) {
+		return { reference_to };
+	}
+
+	return { reference_to: [...new Set(reference_to)].sort() };
+}

package/src/schema/pull.ts

@@ -0,0 +1,28 @@
+import type Client from '#cli/cs/api/Client.js';
+import Ctx from './ctx/Ctx.js';
+import pullModules from './lib/pullModules.js';
+import type TransferResults from './xfer/TransferResults.js';
+
+type Result = ReadonlyMap<string, PromiseSettledResult<TransferResults>>;
+
+export default async function pull(client: Client): Promise<Result> {
+	const names: string[] = [];
+	const tasks = [];
+	const ctx = await Ctx.prepare(client);
+
+	for await (const [name, task] of pullModules(ctx)) {
+		names.push(name);
+		tasks.push(task);
+	}
+
+	const results = await Promise.allSettled(tasks);
+
+	return results.reduce((map, result, index) => {
+		const name = names[index];
+		if (!name) {
+			throw new Error('Could not resolve task name');
+		}
+
+		return map.set(name, result);
+	}, new Map<string, PromiseSettledResult<TransferResults>>());
+}

package/src/schema/push.ts

@@ -0,0 +1,77 @@
+import type Client from '#cli/cs/api/Client.js';
+import assetFolders from './assetFolders/toContentstack.js';
+import assets from './assets/toContentstack.js';
+import contentTypeShims from './content-types/shimsToContentstack.js';
+import contentTypes from './content-types/toContentstack.js';
+import Ctx from './ctx/Ctx.js';
+import entries from './entries/toContentstack.js';
+import updateMissedReferences from './entries/updateMissedReferences.js';
+import globalFields from './global-fields/toContentstack.js';
+import PushResults from './lib/PushResults.js';
+import getUi from './lib/SchemaUi.js';
+import taxonomies from './taxonomies/toContentstack.js';
+import terms from './terms/toContentstack.js';
+
+export default async function push(client: Client) {
+	const results = new PushResults();
+	const ctx = await Ctx.prepare(client);
+
+	// When pulling, we can pull all schema / entries at the same time. When
+	// pushing, we have to do them in order. The order is defined here:
+	//
+	// https://www.contentstack.com
+	//   /docs/developers/cli/import-content-using-the-cli#module-wise-import
+	//
+	//     locales > environments > assets > taxonomies > extensions >
+	//     marketplace-apps > webhooks > global-fields > content-types >
+	//     workflows > entries > labels > custom-roles
+	//
+	// 2024-10-08 - Update: The above advise appears to be incorrect. Global
+	// fields which reference content types may not be created until the
+	// referenced content types exist. The current solution is to break the
+	// "content type" portion into two parts: one for a series of shims, then
+	// the actual content types. The new order becomes:
+	//
+	//   content-type-shims > global-fields > content-types
+	//
+	// 2024-10-25 - Update: If we do asset folders first, before assets, we can
+	// simplify the logic for assets. We also benefit from the fact that deletion
+	// of asset folders implies deletion of all assets within them, saving us the
+	// trouble of deleting assets individually.
+	await results.set('Asset Folders', assetFolders(ctx));
+	await results.set('Assets', assets(ctx));
+	await results.set('Taxonomies', taxonomies(ctx));
+
+	for await (const [human, result] of terms(ctx)) {
+		await results.set(human, Promise.resolve(result));
+	}
+
+	await results.set('Content Type Shims', contentTypeShims(ctx));
+	await results.set('Global Fields', globalFields(ctx));
+	await results.set('Content Types', contentTypes(ctx));
+
+	const totalEntries = calculateTotalEntries(ctx);
+
+	if (totalEntries > 0) {
+		using bar = getUi().createProgressBar('Entries', totalEntries);
+
+		for (const contentType of ctx.fs.contentTypes.values()) {
+			const task = entries(ctx, contentType, bar);
+			await results.set(`${contentType.title} entries`, task);
+		}
+	}
+
+	await results.set('References', updateMissedReferences(ctx));
+	return results.value;
+}
+
+function calculateTotalEntries(ctx: Ctx) {
+	return [...ctx.fs.contentTypes.values()].reduce((acc, ct) => {
+		const titles = new Set([
+			...ctx.cs.entries.byTitleFor(ct.uid).keys(),
+			...ctx.fs.entries.byTitleFor(ct.uid).keys(),
+		]);
+
+		return acc + titles.size;
+	}, 0);
+}

package/src/schema/references/ReferenceMap.ts

@@ -0,0 +1,98 @@
+import type { ContentType } from '#cli/cs/content-types/Types.js';
+import parseReferencePath from '#cli/cs/entries/parseReferencePath.js';
+import type { Entry, ReferencePath } from '#cli/cs/entries/Types.js';
+import createStylus from '#cli/ui/createStylus.js';
+import HandledError from '#cli/ui/HandledError.js';
+
+export default interface ReadonlyReferenceMap {
+	readonly missed: number;
+	entriesWithMissedReferences(): Iterable<readonly [ContentType['uid'], Entry]>;
+	findReferencedUid(from: ReferencePath, to: ReferencePath): string;
+}
+
+// This is just a placeholder value that intentionally doesn't match any
+// valid entry. It's used to indicate that the reference is missing.
+const placeholderUid = 'blt3d92b0cffbed654c';
+
+export class ReferenceMap implements ReadonlyReferenceMap {
+	#missed = 0;
+	readonly #byPath = new Map<ReferencePath, Entry>();
+
+	readonly #byTypedUid = new Map<
+		`${ContentType['uid']}/${Entry['uid']}`,
+		ReferencePath
+	>();
+
+	readonly #missedReferences = new Map<ReferencePath, Set<ReferencePath>>();
+
+	public get missed() {
+		return this.#missed;
+	}
+
+	public *entriesWithMissedReferences() {
+		for (const [from, missed] of this.#missedReferences) {
+			if (missed.size === 0) {
+				continue;
+			}
+
+			const entry = this.#byPath.get(from);
+			if (!entry) {
+				continue;
+			}
+
+			const { contentTypeUid } = parseReferencePath(from);
+			yield [contentTypeUid, entry] as const;
+		}
+	}
+
+	public findReferencedUid(from: ReferencePath, to: ReferencePath) {
+		const existing = this.#byPath.get(to);
+
+		if (existing) {
+			return existing.uid;
+		}
+
+		this.#addMissedReference(from, to);
+		return placeholderUid;
+	}
+
+	public recordEntryForReferences(
+		contentTypeUid: ContentType['uid'],
+		entry: Entry,
+	) {
+		const entryPath: ReferencePath = `${contentTypeUid}/${entry.title}`;
+		this.#byPath.set(entryPath, entry);
+		this.#byTypedUid.set(`${contentTypeUid}/${entry.uid}`, entryPath);
+	}
+
+	public seal(): ReadonlyReferenceMap {
+		return {
+			entriesWithMissedReferences: this.entriesWithMissedReferences.bind(this),
+
+			findReferencedUid: (from: ReferencePath, to: ReferencePath) => {
+				const existing = this.#byPath.get(to);
+
+				if (!existing) {
+					const y = createStylus('yellowBright');
+					throw new HandledError(y`Missing reference: ${from} -> ${to}`);
+				}
+
+				return existing.uid;
+			},
+
+			missed: this.missed,
+		};
+	}
+
+	#addMissedReference(from: ReferencePath, to: ReferencePath) {
+		const missing = this.#missedReferences.get(from);
+
+		this.#missed += 1;
+
+		if (missing) {
+			missing.add(to);
+		} else {
+			this.#missedReferences.set(from, new Set([to]));
+		}
+	}
+}

package/src/schema/taxonomies/lib/indexCs.ts

@@ -0,0 +1,72 @@
+import type Client from '#cli/cs/api/Client.js';
+import exportTaxonomy from '#cli/cs/taxonomies/export.js';
+import indexTaxonomies from '#cli/cs/taxonomies/index.js';
+import type Taxonomy from '#cli/cs/taxonomies/Taxonomy.js';
+import fromCs from '#cli/dto/taxonomy/fromCs.js';
+import type NormalizedTaxonomy from '#cli/dto/taxonomy/NormalizedTaxonomy.js';
+import taxonomyStrategy from '#cli/dto/taxonomy/taxonomyStrategy.js';
+import getUi from '#cli/schema/lib/SchemaUi.js';
+import ProgressReporter from '#cli/ui/progress/ProgressReporter.js';
+
+export default async function indexCs(
+	client: Client,
+): Promise<ReadonlyMap<string, NormalizedTaxonomy>> {
+	const raw = await indexTaxonomies(client);
+	const transformed = new Map<string, NormalizedTaxonomy>();
+	const toExport = taxonomiesNeedingFullExport(raw);
+
+	for (const taxonomy of raw.values()) {
+		if (toExport.has(taxonomy)) {
+			continue;
+		}
+
+		transformed.set(taxonomy.uid, fromCs(taxonomy));
+	}
+
+	for await (const normalized of taxonomiesWithTerms(client, toExport)) {
+		transformed.set(normalized.taxonomy.uid, normalized);
+	}
+
+	return transformed;
+}
+
+async function* taxonomiesWithTerms(
+	client: Client,
+	toExport: ReadonlySet<Taxonomy>,
+) {
+	if (toExport.size === 0) {
+		return;
+	}
+
+	const ui = getUi();
+	using bar = ui.createProgressBar('Taxonomy terms', toExport.size);
+
+	for (const taxonomy of toExport) {
+		using reporter = new ProgressReporter(
+			bar,
+			'Loading',
+			`${taxonomy.name} terms`,
+		);
+
+		const exported = await exportTaxonomy(client, taxonomy.uid);
+		yield fromCs(taxonomy, exported.terms);
+		bar.increment();
+		reporter.finish('Loaded');
+	}
+}
+
+function taxonomiesNeedingFullExport(
+	taxonomies: ReadonlyMap<string, Taxonomy>,
+): ReadonlySet<Taxonomy> {
+	const result = new Set<Taxonomy>();
+
+	for (const [uid, taxonomy] of taxonomies) {
+		const strategy = taxonomyStrategy(uid);
+
+		if (strategy === 'taxonomy and terms') {
+			result.add(taxonomy);
+		}
+	}
+
+	return result;
+}

package/src/schema/taxonomies/lib/indexFs.ts

@@ -0,0 +1,13 @@
+import type NormalizedTaxonomy from '#cli/dto/taxonomy/NormalizedTaxonomy.js';
+import {
+	isNormalizedTaxonomy,
+	key,
+} from '#cli/dto/taxonomy/NormalizedTaxonomy.js';
+import indexFromFilesystem from '#cli/schema/xfer/indexFromFilesystem.js';
+import schemaDirectory from '../schemaDirectory.js';
+
+export default async function indexFs(): Promise<
+	ReadonlyMap<string, NormalizedTaxonomy>
+> {
+	return indexFromFilesystem(schemaDirectory(), isNormalizedTaxonomy, key);
+}

package/src/schema/taxonomies/schemaDirectory.ts

@@ -0,0 +1,12 @@
+import { resolve } from 'node:path';
+import getUi from '../lib/SchemaUi.js';
+
+export default function schemaDirectory() {
+	const {
+		options: {
+			schema: { schemaPath },
+		},
+	} = getUi();
+
+	return resolve(schemaPath, 'taxonomies');
+}

package/src/schema/taxonomies/toContentstack.ts

@@ -0,0 +1,34 @@
+import type NormalizedTaxonomy from '#cli/dto/taxonomy/NormalizedTaxonomy.js';
+import { isDeepStrictEqual } from 'node:util';
+import type Ctx from '../ctx/Ctx.js';
+import createProgressBar from '../lib/createProgressBar.js';
+import getUi from '../lib/SchemaUi.js';
+import planMerge from '../xfer/lib/planMerge.js';
+import processPlan from '../xfer/lib/processPlan.js';
+
+export default async function toContentstack(ctx: Ctx) {
+	using bar = createProgressBar(
+		'Taxonomies',
+		ctx.cs.taxonomies.byUid,
+		ctx.fs.taxonomies.byUid,
+	);
+
+	const plan = planMerge(
+		equality,
+		ctx.fs.taxonomies.byUid,
+		ctx.cs.taxonomies.byUid,
+	);
+
+	return await processPlan<NormalizedTaxonomy>({
+		create: async (x) => ctx.cs.taxonomies.create(x),
+		deletionStrategy: getUi().options.schema.deletionStrategy,
+		plan,
+		progress: bar,
+		remove: async (x) => ctx.cs.taxonomies.remove(x),
+		update: async (x) => ctx.cs.taxonomies.update(x),
+	});
+}
+
+function equality(a: NormalizedTaxonomy, b: NormalizedTaxonomy) {
+	return isDeepStrictEqual(a.taxonomy, b.taxonomy);
+}

package/src/schema/taxonomies/toFilesystem.ts

@@ -0,0 +1,42 @@
+import type NormalizedTaxonomy from '#cli/dto/taxonomy/NormalizedTaxonomy.js';
+import { isDeepStrictEqual } from 'node:util';
+import type Ctx from '../ctx/Ctx.js';
+import createProgressBar from '../lib/createProgressBar.js';
+import planMerge from '../xfer/lib/planMerge.js';
+import processPlan from '../xfer/lib/processPlan.js';
+
+export default async function toFilesystem(ctx: Ctx) {
+	using bar = createProgressBar(
+		'Taxonomies',
+		ctx.cs.taxonomies.byUid,
+		ctx.fs.taxonomies.byUid,
+	);
+
+	const plan = planMerge(
+		equality,
+		ctx.cs.taxonomies.byUid,
+		ctx.fs.taxonomies.byUid,
+	);
+
+	return await processPlan<NormalizedTaxonomy>({
+		create: async (x) => ctx.fs.taxonomies.create(x),
+		deletionStrategy: 'delete',
+		plan,
+		progress: bar,
+		remove: async (x) => ctx.fs.taxonomies.remove(x),
+		update: async (x) => ctx.fs.taxonomies.update(x),
+	});
+}
+
+function equality(a: NormalizedTaxonomy, b: NormalizedTaxonomy) {
+	return isDeepStrictEqual(
+		{
+			taxonomy: a.taxonomy,
+			terms: a.terms ?? [],
+		},
+		{
+			taxonomy: b.taxonomy,
+			terms: b.terms ?? [],
+		},
+	);
+}

package/src/schema/terms/collection/Collection.ts

@@ -0,0 +1,70 @@
+import type Client from '#cli/cs/api/Client.js';
+import type Term from '#cli/cs/terms/Term.js';
+import flatten from '#cli/dto/taxonomy/flatten.js';
+import type NormalizedTaxonomy from '#cli/dto/taxonomy/NormalizedTaxonomy.js';
+import create from './lib/create.js';
+import descendants from './lib/descendants.js';
+import getRequired from './lib/getRequired.js';
+import { getSiblingContext } from './lib/getSiblingContext.js';
+import insertTermIntoSiblings from './lib/insertTermIntoSiblings.js';
+import move from './lib/move.js';
+import type MutableTerm from './lib/MutableTerm.js';
+import remove from './lib/remove.js';
+import update from './lib/update.js';
+
+export default class Collection {
+	public readonly remove: typeof remove;
+	public readonly create: typeof create;
+	public readonly update: typeof update;
+	public readonly move: typeof move;
+
+	protected readonly byParentUid = new Map<string | null, MutableTerm[]>();
+	protected readonly byUid = new Map<string, MutableTerm>();
+	protected readonly descendants: typeof descendants;
+	protected readonly getRequired: typeof getRequired;
+	protected readonly getSiblingContext: typeof getSiblingContext;
+	protected readonly insertTermIntoSiblings: typeof insertTermIntoSiblings;
+	protected readonly taxonomy: NormalizedTaxonomy['taxonomy'];
+
+	public constructor(
+		protected readonly client: Client,
+		normalized: NormalizedTaxonomy,
+	) {
+		this.descendants = descendants.bind(this);
+		this.insertTermIntoSiblings = insertTermIntoSiblings.bind(this);
+		this.getRequired = getRequired.bind(this);
+		this.getSiblingContext = getSiblingContext.bind(this);
+		this.remove = remove.bind(this);
+		this.create = create.bind(this);
+		this.update = update.bind(this);
+		this.move = move.bind(this);
+
+		this.taxonomy = normalized.taxonomy;
+		const flat = flatten(normalized.terms ?? []);
+
+		for (const term of flat) {
+			this.byUid.set(term.uid, term);
+
+			const existing = this.byParentUid.get(term.parent_uid);
+			if (existing) {
+				existing.push(term);
+			} else {
+				this.byParentUid.set(term.parent_uid, [term]);
+			}
+		}
+	}
+
+	public *all(this: Collection): Generator<Term> {
+		for (const term of this.byUid.values()) {
+			yield term;
+		}
+	}
+
+	public children(parentUid: string | null): readonly Term[] {
+		return this.byParentUid.get(parentUid) ?? [];
+	}
+
+	public get(termUid: string): Term | undefined {
+		return this.byUid.get(termUid);
+	}
+}

package/src/schema/terms/collection/lib/MutableTerm.ts

@@ -0,0 +1,7 @@
+import type Term from '#cli/cs/terms/Term.js';
+
+export default interface MutableTerm {
+	readonly uid: Term['uid'];
+	name: Term['name'];
+	parent_uid: Term['parent_uid'];
+}