@getodk/xforms-engine 0.4.0 → 0.6.0

package/src/parse/model/SecondaryInstance/sources/CSVExternalSecondaryInstance.ts

@@ -0,0 +1,288 @@
+import { XFORMS_NAMESPACE_URI } from '@getodk/common/constants/xmlns.ts';
+import type { JRResourceURL } from '@getodk/common/jr-resources/JRResourceURL.ts';
+import * as papa from 'papaparse';
+import { ErrorProductionDesignPendingError } from '../../../../error/ErrorProductionDesignPendingError.ts';
+import { StaticAttribute } from '../../../../integration/xpath/static-dom/StaticAttribute.ts';
+import { StaticElement } from '../../../../integration/xpath/static-dom/StaticElement.ts';
+import { StaticText } from '../../../../integration/xpath/static-dom/StaticText.ts';
+import { SecondaryInstanceDefinition } from '../SecondaryInstanceDefinition.ts';
+import { SecondaryInstanceRootDefinition } from '../SecondaryInstanceRootDefinition.ts';
+import { ExternalSecondaryInstanceSource } from './ExternalSecondaryInstanceSource.ts';
+
+type CSVColumn = string;
+type CSVRow = readonly CSVColumn[];
+
+type AssertCSVRow = (columns: unknown) => asserts columns is CSVRow;
+
+/**
+ * Based on {@link https://github.com/getodk/central-frontend/commit/29cebcc870c9be70ab0d222e3349e34639045d19}
+ *
+ * Central performs this check for header and rows. A comment is included there for the header check, but the logic is the same in both cases.
+ */
+const rejectNullCharacters = (cell: string) => {
+	if (cell.includes('\0')) {
+		throw new ErrorProductionDesignPendingError(`Failed to parse CSV: null character`);
+	}
+};
+
+const assertCSVRow: AssertCSVRow = (columns) => {
+	if (!Array.isArray(columns)) {
+		throw new ErrorProductionDesignPendingError('Failed to parse CSV columns');
+	}
+
+	for (const [index, column] of columns.entries()) {
+		if (typeof column !== 'string') {
+			throw new ErrorProductionDesignPendingError(`Failed to parse CSV column at index ${index}`);
+		}
+
+		rejectNullCharacters(column);
+	}
+};
+
+type AssertPapaparseSuccess = (
+	resourceURL: JRResourceURL,
+	errors: readonly papa.ParseError[]
+) => asserts errors is readonly [];
+
+const assertPapaparseSuccess: AssertPapaparseSuccess = (resourceURL, errors) => {
+	if (errors.length > 0) {
+		const cause = new AggregateError(errors);
+		throw new ErrorProductionDesignPendingError(
+			`Failed to parse CSV external secondary instance ${resourceURL.href}`,
+			{ cause }
+		);
+	}
+};
+
+interface ParsedCSVHeader {
+	readonly columns: CSVRow;
+	readonly errors: readonly [];
+	readonly meta: papa.ParseMeta;
+}
+
+interface ParseCSVOptions {
+	readonly columns: readonly string[];
+	readonly delimiter: string;
+}
+
+const stripTrailingEmptyCells = (columns: CSVRow, row: CSVRow): CSVRow => {
+	const result = row.slice();
+
+	while (result.length > columns.length && result.at(-1) === '') {
+		result.pop();
+	}
+
+	return result;
+};
+
+interface ParsedCSVRows {
+	readonly rows: readonly CSVRow[];
+	readonly errors: readonly [];
+	readonly meta: papa.ParseMeta;
+}
+
+interface CSVExternalSecondaryInstanceItemColumn {
+	readonly columnName: string;
+	readonly cellValue: string;
+}
+
+type CSVExternalSecondaryInstanceItem = readonly CSVExternalSecondaryInstanceItemColumn[];
+
+class CSVExternalSecondaryInstanceColumnElement extends StaticElement {
+	constructor(parent: StaticElement, column: CSVExternalSecondaryInstanceItemColumn) {
+		const { columnName, cellValue } = column;
+
+		super(
+			parent,
+			() => [],
+			(self) => [new StaticText(self, cellValue)],
+			{
+				namespaceURI: XFORMS_NAMESPACE_URI,
+				localName: columnName,
+			}
+		);
+	}
+}
+
+class CSVExternalSecondaryInstanceItemElement extends StaticElement {
+	constructor(parent: StaticElement, item: CSVExternalSecondaryInstanceItem) {
+		super(
+			parent,
+			() => [],
+			(self) => {
+				return item.map((column) => {
+					return new CSVExternalSecondaryInstanceColumnElement(self, column);
+				});
+			},
+			{
+				namespaceURI: XFORMS_NAMESPACE_URI,
+				localName: 'item',
+			}
+		);
+	}
+}
+
+class CSVExternalSecondaryInstanceRootElement extends StaticElement {
+	constructor(parent: StaticElement, items: readonly CSVExternalSecondaryInstanceItem[]) {
+		super(
+			parent,
+			() => [],
+			(self) => {
+				return items.map((item) => {
+					return new CSVExternalSecondaryInstanceItemElement(self, item);
+				});
+			},
+			{
+				namespaceURI: XFORMS_NAMESPACE_URI,
+				localName: 'root',
+			}
+		);
+	}
+}
+
+class CSVExternalSecondaryInstanceDocumentElement extends SecondaryInstanceRootDefinition {
+	constructor(
+		instanceId: string,
+		parent: CSVExternalSecondaryInstanceDefinition,
+		items: readonly CSVExternalSecondaryInstanceItem[]
+	) {
+		super(
+			parent,
+			(self) => [
+				new StaticAttribute(self, {
+					namespaceURI: XFORMS_NAMESPACE_URI,
+					localName: 'id',
+					value: instanceId,
+				}),
+			],
+			(self) => [new CSVExternalSecondaryInstanceRootElement(self, items)],
+			{
+				namespaceURI: XFORMS_NAMESPACE_URI,
+				localName: 'instance',
+			}
+		);
+	}
+}
+
+class CSVExternalSecondaryInstanceDefinition extends SecondaryInstanceDefinition {
+	constructor(instanceId: string, items: readonly CSVExternalSecondaryInstanceItem[]) {
+		super((self) => {
+			return new CSVExternalSecondaryInstanceDocumentElement(instanceId, self, items);
+		});
+	}
+}
+
+export class CSVExternalSecondaryInstanceSource extends ExternalSecondaryInstanceSource<'csv'> {
+	/**
+	 * Based on
+	 * {@link https://github.com/getodk/central-frontend/blob/42c9277709e593480d1462e28b4be5f1364532b7/src/util/csv.js#L79} (and {@link https://github.com/getodk/central-frontend/blob/42c9277709e593480d1462e28b4be5f1364532b7/src/util/csv.js#L13}).
+	 *
+	 * The most significant deviations at time of writing:
+	 *
+	 * - we have already retrieved the CSV resource, so we are parsing the raw CSV data directly.
+	 * - we have no need for asynchronous/streaming parsing at this point in the
+	 *   form initialization process, so we can dispense with those details of the
+	 *   {@link papa | papaparse} API/config.
+	 */
+	private parseCSVHeader(csvData: string): ParsedCSVHeader {
+		const { data, errors, meta } = papa.parse(csvData, {
+			delimitersToGuess: [',', ';', '\t', '|'],
+			download: false,
+			preview: 1,
+		});
+		const [columns = []] = data;
+
+		assertCSVRow(columns);
+		assertPapaparseSuccess(this.resourceURL, errors);
+
+		return {
+			errors,
+			meta,
+			columns,
+		};
+	}
+
+	/**
+	 * Largely based on {@link https://github.com/getodk/central-frontend/blob/42c9277709e593480d1462e28b4be5f1364532b7/src/util/csv.js#L170}
+	 */
+	private parseCSVRows(csvData: string, options: ParseCSVOptions): ParsedCSVRows {
+		const { columns, delimiter } = options;
+		const { data, errors, meta } = papa.parse(csvData, {
+			delimiter,
+			download: false,
+		});
+
+		assertPapaparseSuccess(this.resourceURL, errors);
+
+		const rowData = data.slice(1);
+		const lastRowIndex = rowData.length - 1;
+
+		let stripLastRow = false;
+
+		const rows = rowData.map((values, index) => {
+			assertCSVRow(values);
+
+			const rowIndex = index + 1;
+
+			// Central: Remove trailing empty cells.
+			const row = stripTrailingEmptyCells(columns, values);
+
+			// Central: Skip trailing empty rows and do not check them for warnings.
+			// Throw for an empty row that is not trailing.
+			if (row.every((cell) => cell === '')) {
+				if (index === lastRowIndex) {
+					stripLastRow = true;
+				} else {
+					throw new ErrorProductionDesignPendingError(
+						`Failed to parse CSV row ${rowIndex}: unexpected empty row`
+					);
+				}
+			}
+
+			// Central: Throw if there are too many cells.
+			if (row.length > columns.length) {
+				throw new ErrorProductionDesignPendingError(
+					`Failed to parse CSV row ${rowIndex}: expected ${columns.length} columns, got ${row.length}`
+				);
+			}
+
+			return row;
+		});
+
+		if (stripLastRow) {
+			rows.pop();
+		}
+
+		return {
+			errors,
+			meta,
+			rows,
+		};
+	}
+
+	private toItems(
+		columns: CSVRow,
+		rows: readonly CSVRow[]
+	): readonly CSVExternalSecondaryInstanceItem[] {
+		return rows.map((row) => {
+			return columns.map((columnName, index) => {
+				return {
+					columnName,
+					cellValue: row[index] ?? '',
+				};
+			});
+		});
+	}
+
+	parseDefinition(): SecondaryInstanceDefinition {
+		const csvData = this.resource.data.replace(/[\n\r]+$/, '');
+		const { columns, meta } = this.parseCSVHeader(csvData);
+		const { rows } = this.parseCSVRows(csvData, {
+			columns,
+			delimiter: meta.delimiter,
+		});
+		const items = this.toItems(columns, rows);
+
+		return new CSVExternalSecondaryInstanceDefinition(this.instanceId, items);
+	}
+}

package/src/parse/model/SecondaryInstance/sources/ExternalSecondaryInstanceResource.ts

@@ -0,0 +1,222 @@
+import type { JRResourceURL } from '@getodk/common/jr-resources/JRResourceURL.ts';
+import type { MissingResourceBehavior } from '../../../../client/constants.ts';
+import type { FetchResource, FetchResourceResponse } from '../../../../client/resources.ts';
+import { ErrorProductionDesignPendingError } from '../../../../error/ErrorProductionDesignPendingError.ts';
+import { FormAttachmentResource } from '../../../attachments/FormAttachmentResource.ts';
+import type { ExternalSecondaryInstanceSourceFormat } from './SecondaryInstanceSource.ts';
+
+const assertResponseSuccess = (resourceURL: JRResourceURL, response: FetchResourceResponse) => {
+	const { ok = true, status = 200 } = response;
+
+	if (!ok || status !== 200) {
+		throw new ErrorProductionDesignPendingError(`Failed to load ${resourceURL.href}`);
+	}
+};
+
+const stripContentTypeCharset = (contentType: string): string => {
+	return contentType.replace(/;charset=.*$/, '');
+};
+
+const getResponseContentType = (response: FetchResourceResponse): string | null => {
+	const { headers } = response;
+
+	if (headers == null) {
+		return null;
+	}
+
+	const contentType = headers.get('content-type');
+
+	if (contentType != null) {
+		return stripContentTypeCharset(contentType);
+	}
+
+	if (headers instanceof Headers) {
+		return contentType;
+	}
+
+	for (const [header, value] of headers) {
+		if (header.toLowerCase() === 'content-type') {
+			return stripContentTypeCharset(value);
+		}
+	}
+
+	return null;
+};
+
+interface ExternalSecondaryInstanceResourceMetadata<
+	Format extends ExternalSecondaryInstanceSourceFormat = ExternalSecondaryInstanceSourceFormat,
+> {
+	readonly contentType: string;
+	readonly format: Format;
+}
+
+const inferSecondaryInstanceResourceMetadata = (
+	resourceURL: JRResourceURL,
+	contentType: string | null,
+	data: string
+): ExternalSecondaryInstanceResourceMetadata => {
+	const url = resourceURL.href;
+
+	let format: ExternalSecondaryInstanceSourceFormat | null = null;
+
+	if (url.endsWith('.xml') && data.startsWith('<')) {
+		format = 'xml';
+	} else if (url.endsWith('.csv')) {
+		format = 'csv';
+	} else if (url.endsWith('.geojson') && data.startsWith('{')) {
+		format = 'geojson';
+	}
+
+	if (format == null) {
+		throw new ErrorProductionDesignPendingError(
+			`Failed to infer external secondary instance format/content type for resource ${url} (response content type: ${contentType}, data: ${data})`
+		);
+	}
+
+	return {
+		contentType: contentType ?? 'text/plain',
+		format,
+	};
+};
+
+const detectSecondaryInstanceResourceMetadata = (
+	resourceURL: JRResourceURL,
+	response: FetchResourceResponse,
+	data: string
+): ExternalSecondaryInstanceResourceMetadata => {
+	const contentType = getResponseContentType(response);
+
+	if (contentType == null || contentType === 'text/plain') {
+		return inferSecondaryInstanceResourceMetadata(resourceURL, contentType, data);
+	}
+
+	let format: ExternalSecondaryInstanceSourceFormat | null = null;
+
+	switch (contentType) {
+		case 'text/csv':
+			format = 'csv';
+			break;
+
+		case 'application/geo+json':
+			format = 'geojson';
+			break;
+
+		case 'text/xml':
+			format = 'xml';
+			break;
+	}
+
+	if (format == null) {
+		throw new ErrorProductionDesignPendingError(
+			`Failed to detect external secondary instance format for resource ${resourceURL.href} (response content type: ${contentType}, data: ${data})`
+		);
+	}
+
+	return {
+		contentType,
+		format,
+	};
+};
+
+interface MissingResourceResponse extends FetchResourceResponse {
+	readonly status: 404;
+}
+
+export interface ExternalSecondaryInstanceResourceLoadOptions {
+	readonly fetchResource: FetchResource<JRResourceURL>;
+	readonly missingResourceBehavior: MissingResourceBehavior;
+}
+
+type LoadedExternalSecondaryInstanceResource = {
+	[Format in ExternalSecondaryInstanceSourceFormat]: ExternalSecondaryInstanceResource<Format>;
+}[ExternalSecondaryInstanceSourceFormat];
+
+interface ExternalSecondaryInstanceResourceOptions {
+	readonly isExplicitlyBlank?: boolean;
+}
+
+export class ExternalSecondaryInstanceResource<
+	Format extends ExternalSecondaryInstanceSourceFormat = ExternalSecondaryInstanceSourceFormat,
+> extends FormAttachmentResource<'secondary-instance'> {
+	private static isMissingResource(
+		response: FetchResourceResponse
+	): response is MissingResourceResponse {
+		return response.status === 404;
+	}
+
+	private static createBlankResource(
+		instanceId: string,
+		resourceURL: JRResourceURL,
+		response: MissingResourceResponse,
+		options: ExternalSecondaryInstanceResourceLoadOptions
+	) {
+		if (options.missingResourceBehavior === 'BLANK') {
+			return new this(
+				response.status,
+				instanceId,
+				resourceURL,
+				{
+					format: 'xml',
+					contentType: 'text/xml',
+				},
+				'',
+				{ isExplicitlyBlank: true }
+			);
+		}
+
+		throw new ErrorProductionDesignPendingError(
+			`Failed to load resource: ${resourceURL.href}: resource is missing (status: ${response.status})`
+		);
+	}
+
+	static async load(
+		instanceId: string,
+		resourceURL: JRResourceURL,
+		options: ExternalSecondaryInstanceResourceLoadOptions
+	): Promise<LoadedExternalSecondaryInstanceResource> {
+		const response = await options.fetchResource(resourceURL);
+
+		if (this.isMissingResource(response)) {
+			return this.createBlankResource(instanceId, resourceURL, response, options);
+		}
+
+		assertResponseSuccess(resourceURL, response);
+
+		const data = await response.text();
+		const metadata = detectSecondaryInstanceResourceMetadata(resourceURL, response, data);
+
+		return new this(response.status ?? null, instanceId, resourceURL, metadata, data, {
+			isExplicitlyBlank: false,
+		}) satisfies ExternalSecondaryInstanceResource as LoadedExternalSecondaryInstanceResource;
+	}
+
+	readonly format: Format;
+	readonly isBlank: boolean;
+
+	private constructor(
+		readonly responseStatus: number | null,
+		readonly instanceId: string,
+		resourceURL: JRResourceURL,
+		metadata: ExternalSecondaryInstanceResourceMetadata<Format>,
+		data: string,
+		options: ExternalSecondaryInstanceResourceOptions
+	) {
+		const { contentType, format } = metadata;
+
+		super('secondary-instance', resourceURL, contentType, data);
+
+		this.format = format;
+
+		if (data === '') {
+			if (options.isExplicitlyBlank) {
+				this.isBlank = true;
+			} else {
+				throw new ErrorProductionDesignPendingError(
+					`Failed to load blank external secndary instance ${resourceURL.href}`
+				);
+			}
+		} else {
+			this.isBlank = false;
+		}
+	}
+}

package/src/parse/model/SecondaryInstance/sources/ExternalSecondaryInstanceSource.ts

@@ -0,0 +1,22 @@
+import type { JRResourceURL } from '@getodk/common/jr-resources/JRResourceURL.ts';
+import type { DOMSecondaryInstanceElement } from '../../../XFormDOM.ts';
+import type { ExternalSecondaryInstanceResource } from './ExternalSecondaryInstanceResource.ts';
+import type { ExternalSecondaryInstanceSourceFormat } from './SecondaryInstanceSource.ts';
+import { SecondaryInstanceSource } from './SecondaryInstanceSource.ts';
+
+export abstract class ExternalSecondaryInstanceSource<
+	Format extends ExternalSecondaryInstanceSourceFormat = ExternalSecondaryInstanceSourceFormat,
+> extends SecondaryInstanceSource<Format> {
+	override readonly resourceURL: JRResourceURL;
+
+	constructor(
+		domElement: DOMSecondaryInstanceElement,
+		protected readonly resource: ExternalSecondaryInstanceResource<Format>
+	) {
+		const { format, instanceId, resourceURL } = resource;
+
+		super(format, instanceId, resourceURL, domElement);
+
+		this.resourceURL = resourceURL;
+	}
+}