@datasworn-community/build-tools 0.2.1 → 0.2.3

package/schema-source/scripts/json-typedef/utils.ts

@@ -0,0 +1,349 @@
+import {
+	Hint,
+	KindGuard,
+	type TAnySchema,
+	type TArray,
+	type TBoolean,
+	type TInteger,
+	type TLiteral,
+	type TNull,
+	type TNumber,
+	type TObject,
+	type TOptional,
+	type TRecord,
+	type TRef,
+	type TSchema,
+	type TString,
+	type TThis,
+	Type,
+	TypeGuard
+} from '@sinclair/typebox'
+import type { JTDSchemaType } from 'ajv/dist/core.js'
+import type * as JTD from 'jtd'
+import {
+	cloneDeep,
+	isInteger,
+	isString,
+	isUndefined,
+	mapValues,
+	merge,
+	omit,
+	omitBy,
+	pick,
+	set
+} from 'lodash-es'
+import * as Generic from '../../schema/Generic.js'
+import type { TRoot } from '../../schema/root/Root.js'
+import { Discriminator, JsonTypeDef, Mapping } from '../../schema/Symbols.js'
+import * as Utils from '../../schema/Utils.js'
+import { DefsKey, rootSchemaName } from '../const.js'
+import Log from '../utils/Log.js'
+import JtdType, {
+	type Metadata,
+	type TFields,
+	type TStruct
+} from './typedef.js'
+
+/** Extract metadata from a JSON schema for use in a JTD schema's `metadata` property */
+export function extractMetadata<T extends TAnySchema>(jsonSchema: T) {
+	const metadataKeys = [
+		// general
+		'description',
+		'examples',
+		// string
+		'pattern',
+		'format',
+		// numeric
+		'exclusiveMaximum',
+		'exclusiveMinimum',
+		'maximum',
+		'minimum',
+		'multipleOf',
+		// array
+		'uniqueItems',
+		'minItems',
+		'maxItems'
+	]
+
+	let metadata = pick(cloneDeep(jsonSchema), ...metadataKeys) as Metadata
+
+	if (jsonSchema[JsonTypeDef]?.metadata)
+		metadata = merge(metadata, omit(jsonSchema[JsonTypeDef].metadata))
+
+	// @ts-expect-error
+	if (jsonSchema[Utils.EnumDescription]) {
+		// console.log(jsonSchema)
+		// @ts-expect-error
+		metadata.enumDescription = jsonSchema[Utils.EnumDescription]
+		// @ts-expect-error
+		metadata.description = jsonSchema[Utils.Description]
+	}
+
+	if (Object.keys(metadata)?.length === 0) return undefined
+
+	return metadata
+}
+
+export function setIdRef<T extends { _id: string }, R extends string>(
+	schema: JTDSchemaType<T>,
+	ref: R
+) {
+	type NewRef = { [P in R]: string }
+	type RefRecord =
+		typeof schema extends JTDSchemaType<T, infer U> ? U & NewRef : NewRef
+	return set(schema, 'properties._id.ref', ref) as unknown as JTDSchemaType<
+		T & { _id: R },
+		RefRecord
+	>
+}
+
+export function toJtdEnum<
+	U extends string[] | number[],
+	T extends Utils.TUnionEnum<U>
+>(schema: T) {
+	if (schema.enum.every(isString)) return JtdType.Enum(schema.enum)
+	if (schema.enum.every(isInteger)) return JtdType.Uint8()
+
+	throw new Error(`Unable to infer schema for enum: ${JSON.stringify(schema)}`)
+}
+
+/** Identifiers of all JTD schemas that are referenced */
+export const refTracker = new Set<string>()
+
+export function toJtdRef(schema: TRef | TThis): JTD.SchemaFormRef {
+	const ref = schema.$ref.replace(`#/${DefsKey}/`, '')
+
+	refTracker.add(ref)
+
+	return JtdType.Ref(ref)
+}
+
+/** Transforms a Typebox array schema into JTD elements */
+export function toJtdElements<U extends TSchema, T extends TArray<U>>(
+	schema: T
+) {
+	const { items } = schema
+	return JtdType.Array(toJtdForm(items as unknown as TConvertible) as TSchema)
+}
+
+/** Transform a Typebox object schema into JTD properties */
+export function toJtdProperties<T extends TObject>(schema: T) {
+	const fields = omitBy(
+		mapValues(schema.properties, (subschema) => {
+			if (Utils.TNullable(subschema))
+				return JtdType.Nullable(toJtdForm(subschema.anyOf[0] as any) as TSchema)
+
+			const base = toJtdForm(subschema as any) as TSchema
+
+			if (TypeGuard.IsOptional(subschema)) return Type.Optional(base)
+
+			return base
+		}),
+		isUndefined
+	) as TFields
+
+	const result = JtdType.Struct(fields)
+
+	return result
+}
+
+/** Transform a Typebox record schema into JTD values */
+export function toJtdValues<T extends TRecord<TString, U>, U extends TSchema>(
+	schema: T
+) {
+	const [propertyPattern, value] = Object.entries(schema.patternProperties)[0]
+	const unwrap = toJtdForm(value as unknown as TConvertible) as TSchema
+
+	if (!unwrap)
+		throw new Error(
+			`Couldn't unwrap Record value schema: ${JSON.stringify(schema)}`
+		)
+
+	return JtdType.Record(unwrap, { propertyPattern })
+}
+
+export function toJtdSingleEnum(schema: TLiteral<string>) {
+	if (typeof schema.const === 'number')
+		throw new Error(`Got a number literal from ${schema.$id}`)
+
+	return JtdType.Enum([schema.const])
+}
+
+export function toJtdDiscriminator<
+	M extends Utils.TDiscriminatorMap<Utils.TDiscriminableish>,
+	D extends string
+>(schema: Utils.TDiscriminatedUnion<M, D>) {
+	const discriminator = schema[Discriminator] as string
+
+	// console.log(schema.$id)
+	// console.log(schema[Mapping])
+
+	const oldMapping = schema[Mapping] as Record<string, TObject>
+	const jtdMapping: Record<string, TStruct<TFields>> = {}
+
+	for (const k in oldMapping)
+		if (Object.hasOwn(oldMapping, k)) {
+			const subschema = oldMapping[k] as TObject
+			jtdMapping[k] = {
+				...omit(toJtdProperties(subschema), `properties.${discriminator}`),
+				metadata: extractMetadata(subschema as TAnySchema)
+			} as unknown as TStruct<TFields>
+		}
+
+	// const mapping = Object.fromEntries(
+	// 	mapValues(oldMapping, (subschema) => [
+	// 		subschema.properties[discriminator].const,
+	// 		{
+	// 			...omit(toJtdProperties(subschema), `properties.${discriminator}`),
+	// 			metadata: extractMetadata(subschema)
+	// 		}
+	// 	])
+	// )
+
+	// console.log(discriminator, jtdMapping)
+	return JtdType.Union(jtdMapping, discriminator)
+}
+
+type TConvertible =
+	| TLiteral<string>
+	| TString
+	| TBoolean
+	| TInteger
+	| TObject
+	| TNumber
+	| Utils.TUnionEnum
+	| TArray
+
+function toJtdForm(
+	schema: TConvertible | Utils.TNullable<TConvertible> | TOptional<TConvertible>
+): JTD.Schema
+function toJtdForm(schema: TNull): null
+function toJtdForm(schema: TSchema): JTD.Schema | null {
+	const metadata = extractMetadata(schema)
+
+	if (schema[JsonTypeDef]?.schema != null)
+		return merge(cloneDeep(schema[JsonTypeDef].schema), {
+			metadata
+		})
+
+	let result: JTD.Schema
+
+	switch (true) {
+		case schema[JsonTypeDef]?.skip:
+		case KindGuard.IsNull(schema):
+			return null
+		case Utils.TUnionEnum(schema):
+			result = toJtdEnum(schema)
+			break
+		case KindGuard.IsAny(schema):
+			result = JtdType.Any()
+			break
+		case KindGuard.IsLiteralString(schema):
+			result = toJtdSingleEnum(schema as TLiteral<string>)
+			break
+		case KindGuard.IsString(schema):
+			result = JtdType.String()
+			break
+		case KindGuard.IsLiteralBoolean(schema):
+		case KindGuard.IsBoolean(schema):
+			result = JtdType.Boolean()
+			break
+		case KindGuard.IsLiteralNumber(schema):
+		case KindGuard.IsInteger(schema):
+			result = JtdType.Int16()
+			break
+		case KindGuard.IsNumber(schema):
+			Log.warn(
+				'Received a number schema. Consider making it an integer instead.',
+				schema
+			)
+			result = JtdType.Float32()
+			break
+		case KindGuard.IsThis(schema):
+		case KindGuard.IsRef(schema):
+			result = toJtdRef(schema as TThis | TRef)
+			break
+		case KindGuard.IsRecord(schema):
+		case Generic.DictionaryBrand in schema:
+			result = toJtdValues(schema as TRecord) as JTD.Schema
+			break
+		case KindGuard.IsArray(schema):
+			result = toJtdElements(schema as TArray) as JTD.Schema
+			break
+		case KindGuard.IsObject(schema):
+			result = toJtdProperties(schema as TObject)
+			break
+		case Utils.TDiscriminatedUnion(schema):
+			result = toJtdDiscriminator(
+				schema as Utils.TDiscriminatedUnion
+			) as JTD.Schema
+			break
+		case KindGuard.IsUnion(schema) && schema[Hint] === 'Enum':
+			result = JtdType.Enum(
+				(schema.anyOf as TLiteral[]).map((item) => item.const as string),
+				{
+					enumDescription: Object.fromEntries(
+						(schema.anyOf as TLiteral[]).map((item) => [
+							item.const,
+							item.description ?? ''
+						])
+					) as Record<string, string>
+				}
+			)
+			break
+		default:
+			console.log(schema)
+			throw new Error('No JTD transform available for JSON schema')
+	}
+
+	result = merge(result, { metadata: extractMetadata(schema) })
+
+	// if (result[Kind] === 'TypeDef:Enum' && !result.metadata?.typescriptType) {
+	// 	result.metadata ||= {}
+	// 	result.metadata.typescriptType = result.enum
+	// 		.map((item: string | number) => (isInteger(item) ? item : `'${item}'`))
+	// 		.join(' | ')
+	// }
+
+	return result as any
+}
+
+export function toJtdRoot<T extends TRoot>(schemaRoot: T) {
+	const defs = {} as Record<string, JTD.Schema | undefined>
+
+	for (const k in schemaRoot[DefsKey]) {
+		if (k === rootSchemaName) continue
+		try {
+			defs[k] = toJtdForm(schemaRoot[DefsKey][k] as TConvertible)
+		} catch (err: any) {
+			console.log(`${k}:`, schemaRoot[DefsKey][k])
+
+			throw new Error(`Couldn't convert ${k}. ${err?.message ?? err}`)
+		}
+	}
+
+	const base = toJtdForm(schemaRoot[DefsKey][rootSchemaName] as TConvertible)
+
+	if (typeof base === 'undefined')
+		throw new Error(
+			`Unable to infer JSON Typedef form for root schema "${rootSchemaName}".`
+		)
+
+	return {
+		...(base as object),
+		definitions: omitBy(defs, (v, k) => {
+			if (v === null) {
+				Log.info(`Skipping JTD for "${k}"`)
+				return true
+			}
+			// if (!refTracker.has(k)) {
+			// 	Log.info(`Orphaned definition "${k}" -- skipping`)
+			// 	return true
+			// }
+
+			return false
+		})
+	}
+}
+
+export { toJtdForm }

package/schema-source/scripts/schema/buildSchema.ts

@@ -0,0 +1,109 @@
+/**
+ * Regenerates the schemas and write them to file.
+ */
+
+import path from 'node:path'
+import type { JsonSchema } from 'json-schema-library'
+import { kebabCase } from 'lodash-es'
+import * as Schema from '../../schema/index.js'
+import type { TRoot } from '../../schema/root/Root.js'
+import {
+	DefsKey,
+	DIR_HISTORY_CURRENT,
+	SCHEMA_NAME,
+	SCHEMA_PATH,
+	SOURCE_SCHEMA_NAME,
+	SOURCE_SCHEMA_PATH,
+	SOURCEDATA_SCHEMA_PATH
+} from '../const.js'
+import { sortSchemaKeys } from '../datasworn/sort.js'
+import Log from '../utils/Log.js'
+import { writeJSON } from '../utils/readWrite.js'
+import AJV from '../validation/ajv.js'
+
+interface SchemaOptions {
+	name: string
+	rootSchema: TRoot
+	paths: string[]
+	messages: {
+		writeStart: string
+		writeFinish: string
+	}
+}
+
+const schemaOptions: SchemaOptions[] = [
+	{
+		name: SCHEMA_NAME,
+		rootSchema: Schema.DataswornSchema,
+		paths: [
+			SCHEMA_PATH,
+			path.join(DIR_HISTORY_CURRENT, `${kebabCase(SCHEMA_NAME)}.schema.json`)
+		],
+		messages: {
+			writeStart: '✏️  Writing schema for Datasworn',
+			writeFinish: '✅ Finished writing schema for Datasworn'
+		}
+	},
+	{
+		name: SOURCE_SCHEMA_NAME,
+		rootSchema: Schema.DataswornSourceSchema,
+		paths: [
+			SOURCEDATA_SCHEMA_PATH,
+			SOURCE_SCHEMA_PATH,
+			path.join(
+				DIR_HISTORY_CURRENT,
+				`${kebabCase(SOURCE_SCHEMA_NAME)}.schema.json`
+			)
+		],
+		messages: {
+			writeStart: '✏️  Writing schema for DataswornSource',
+			writeFinish: '✅ Finished writing schema for DataswornSource'
+		}
+	}
+]
+
+const metadataKeys: string[] = []
+
+function replacer(k: string, v: unknown) {
+	if (metadataKeys.includes(k)) return undefined
+
+	if (k === '$id' && typeof v === 'string' && !v.startsWith('http'))
+		// omit $ids that aren't the root URI, they're redundant and only there for TypeBox
+		return undefined
+	if (
+		k === '$ref' &&
+		typeof v === 'string' &&
+		!v.startsWith('http') &&
+		!v.startsWith(`#/${DefsKey}/`)
+	)
+		// adjust references for use with standard json validation
+		return `#/${DefsKey}/${v}`
+
+	return v
+}
+
+/** Pending operations to write the schema to disk */
+const writeOps: Promise<unknown>[] = []
+
+for (const { rootSchema, name, paths, messages } of schemaOptions) {
+	AJV.addSchema(rootSchema as JsonSchema, name)
+
+	try {
+		Log.info(messages.writeStart)
+
+		// Recursively sort all keys in the schema
+		const sortedSchema = sortSchemaKeys(JSON.parse(JSON.stringify(rootSchema)))
+
+		writeOps.push(
+			writeJSON(paths, sortedSchema, {
+				replacer
+			}).then(() => Log.info(messages.writeFinish))
+		)
+	} catch (error) {
+		Log.error(error)
+
+		await writeJSON(paths, rootSchema, { replacer })
+	}
+}
+
+await Promise.all(writeOps)

package/schema-source/scripts/utils/Log.ts

@@ -0,0 +1,15 @@
+/**
+ * Minimal console-based logger, standing in for upstream's winston/logform
+ * setup so generation doesn't pull in a logging framework.
+ */
+const Log = {
+	// `log` (rather than `info`) matches ajv's `Logger` interface, which this
+	// object is passed to directly.
+	log: (...args: unknown[]) => console.info(...args),
+	info: (...args: unknown[]) => console.info(...args),
+	warn: (...args: unknown[]) => console.warn(...args),
+	error: (...args: unknown[]) => console.error(...args),
+	verbose: (...args: unknown[]) => console.debug(...args)
+}
+
+export default Log

package/schema-source/scripts/utils/readWrite.ts

@@ -0,0 +1,180 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import { type BunFile } from 'bun'
+import type { Simplify } from 'type-fest'
+import yaml from 'yaml'
+import type { DataswornSource } from '@datasworn-community/core'
+
+const space = '\t'
+
+/**
+ * Reads source data from a YAML or JSON file.
+ * @return The deserialized object.
+ */
+export async function readDataswornSourceData<
+	T extends DataswornSource.RulesPackage
+>(
+	filePath: string | BunFile,
+	reviver?: (key: unknown, value: unknown) => unknown
+): Promise<T> {
+	const file = typeof filePath === 'string' ? Bun.file(filePath) : filePath
+
+	const pathStr =
+		typeof filePath === 'string' ? filePath : (filePath.name as string)
+	switch (path.extname(pathStr)) {
+		case '.yaml':
+		case '.yml':
+			return await readYAML<T>(file, { reviver })
+		case '.json':
+			return await readJSON<T>(file, reviver)
+		default:
+			throw new Error(`Unrecognized file extension in "${filePath}"`)
+	}
+}
+export async function readYAML<T>(
+	filePath: string | BunFile,
+	options: YAMLOptions = {
+		// ensures that dates are serialized as strings rather than Date objects (which prevents AJV from validating them)
+		schema: 'core',
+		merge: true,
+		maxAliasCount: 1000,
+		logLevel: 'debug'
+	}
+): Promise<T> {
+	const file = typeof filePath === 'string' ? Bun.file(filePath) : filePath
+
+	const str = await file.text()
+	return yaml.parse(str, options)
+}
+
+export async function ensureDir(dirPath: string) {
+	return await fs.mkdir(dirPath, { recursive: true })
+}
+
+export async function copyDir(from: string, to: string) {
+	return await fs.cp(from, to, { recursive: true })
+}
+
+export async function copyFile(from: string | BunFile, to: string | BunFile) {
+	const fromFile = typeof from === 'string' ? Bun.file(from) : from
+	const toFile = typeof to === 'string' ? Bun.file(to) : to
+
+	const op = await Bun.write(toFile, fromFile, { createPath: true })
+
+	return op
+}
+
+export async function emptyDir(dirPath: string) {
+	try {
+		await fs.rmdir(dirPath)
+	} catch {
+		// Directory might not exist, ignore
+	}
+	return await ensureDir(dirPath)
+}
+
+export async function readJSON<T>(
+	filePath: string | BunFile,
+	reviver?: (this: any, key: string, value: any) => any
+): Promise<T> {
+	const file =
+		typeof filePath === 'string'
+			? Bun.file(filePath.toString(), {
+					type: 'application/json'
+				})
+			: filePath
+
+	if (reviver == null) return await file.json()
+
+	return JSON.parse(await file.text(), reviver)
+}
+
+type WriteJsonOptions = {
+	skipCopyAwait?: boolean
+	replacer?: (this: any, key: string, value: any) => any
+}
+export async function writeJSON(
+	filePath: string | BunFile,
+	object: any,
+	options?: WriteJsonOptions
+): Promise<string>
+export async function writeJSON(
+	filePaths: (string | BunFile)[],
+	object: any,
+	options?: WriteJsonOptions
+): Promise<any>
+export async function writeJSON(
+	filePath: string | BunFile | (string | BunFile)[],
+	object: any,
+	{ skipCopyAwait = false, replacer }: WriteJsonOptions = {}
+): Promise<any> {
+	if (typeof object === 'string') throw new Error()
+
+	const allPaths = Array.isArray(filePath) ? filePath : [filePath]
+	// dedupe identical destinations (callers may pass the same path twice when
+	// two conceptually distinct outputs happen to coincide); otherwise the file
+	// gets copied onto itself, which can race with its own write and truncate it
+	const seen = new Set<string>()
+	const pathParams = allPaths.filter((destination) => {
+		const key = typeof destination === 'string' ? destination : destination.name
+		if (key != null && seen.has(key)) return false
+		if (key != null) seen.add(key)
+		return true
+	})
+
+	if (pathParams.length === 0)
+		// nothing to do
+		return
+
+	const [writeDestination, ...copyDestinations]: BunFile[] = pathParams.map(
+		(destination) =>
+			typeof destination === 'string' ? Bun.file(destination) : destination
+	)
+
+	const json = JSON.stringify(object, replacer, space)
+
+	// write to the first destination
+	await Bun.write(writeDestination, json, { createPath: true })
+
+	if (copyDestinations.length === 0)
+		// nothing left to do
+		return json
+
+	const copyOps: Promise<any>[] = []
+
+	for (const copyDestionation of copyDestinations)
+		copyOps.push(copyFile(writeDestination, copyDestionation))
+
+	if (skipCopyAwait) {
+		void Promise.all(copyOps)
+	} else {
+		await Promise.all(copyOps)
+	}
+
+	return json
+}
+
+type YAMLOptions = Simplify<
+	yaml.ParseOptions &
+		yaml.DocumentOptions &
+		yaml.SchemaOptions &
+		yaml.ToJSOptions
+>
+
+export async function writeCode(filePath: string | BunFile, content: string) {
+	const file = typeof filePath === 'string' ? Bun.file(filePath) : filePath
+
+	await Bun.write(file, content, { createPath: true })
+}
+export async function updateJSON<T>(
+	path: string | BunFile,
+	update: Partial<T> | ((data: T) => T)
+) {
+	let json = (await readJSON(path)) as T
+
+	// @ts-expect-error
+	if (typeof update !== 'function') Object.assign(json, update)
+	else json = update(json)
+
+	await writeJSON(path, json)
+}

package/schema-source/scripts/validation/ajv.ts

@@ -0,0 +1,29 @@
+import ajvPkg from 'ajv'
+import ajvFormatPkg from 'ajv-formats'
+import Log from '../utils/Log.js'
+import { FORMATS } from './formats.js'
+import { KEYWORDS } from './keywords.js'
+
+// workaround for https://github.com/ajv-validator/ajv/issues/2132
+const Ajv = ajvPkg.default
+const addFormats = ajvFormatPkg.default
+
+const AJV = new Ajv({
+	logger: Log,
+	strict: 'log',
+	strictSchema: 'log',
+	strictTypes: 'log',
+	useDefaults: 'empty',
+	validateFormats: true,
+	verbose: true
+})
+
+for (const format in FORMATS) AJV.addFormat(format, FORMATS[format])
+
+for (const keyword in KEYWORDS)
+	AJV.addKeyword({ keyword, ...KEYWORDS[keyword] })
+
+// Type assertion needed due to ajv-formats bundling its own ajv types
+addFormats(AJV as unknown as Parameters<typeof addFormats>[0])
+
+export default AJV

package/schema-source/scripts/validation/formats.ts

@@ -0,0 +1,6 @@
+import type { Format } from 'ajv'
+
+export const FORMATS: Record<string, Format> = {
+	markdown: true,
+	dice_notation: true
+}