@ausecon/core 0.0.0 → 0.1.0

package/README.md

@@ -10,37 +10,95 @@ Bun, and Deno.
 
 ---
 
-## Status
+## Install
+
+```shell
+npm install @ausecon/core effect
+```
 
-The library is in development. No release exists.
+## Read ABS Time Series
 
-The first release reads ABS time series by catalogue number and table number,
-or by series ID. The package now exports these schemas and errors:
+`readAbs` reads ABS time series from the latest release. It reads every
+table of a catalogue number, chosen tables by table number, or a list of
+series IDs from any catalogue numbers.
 
-- `CatalogueNumber`, such as `6202.0` or `6291.0.55.001`
-- `SeriesId`, such as `A84423050A`
-- `TableNumber`, such as `1`, `5a`, or `X28`
-- `SeriesNotFound`, the error when the Time Series Directory has no record of
-  a series
+`readAbs` returns an `Effect` that needs an `HttpClient`. To run it without
+writing Effect code, provide Effect's fetch-based client and call
+`Effect.runPromise`:
 
 ```ts
-import { CatalogueNumber } from "@ausecon/core";
-import { Schema } from "effect";
+import { AbsRequest, readAbs } from "@ausecon/core";
+import { Effect } from "effect";
+import { FetchHttpClient } from "effect/http";
 
-Schema.is(CatalogueNumber)("6202.0"); // true
-Schema.is(CatalogueNumber)("6202"); // false
+// Table 1 of the Wage Price Index.
+const wages = await Effect.runPromise(
+  readAbs(AbsRequest.Catalogue({ catalogueNumber: "6345.0", tableNumbers: ["1"] })).pipe(
+    Effect.provide(FetchHttpClient.layer),
+  ),
+);
+
+console.log(wages[0]);
 ```
 
-## Install
+Each observation holds the 12 `readabs` columns in camelCase: `tableId`,
+`sheet`, `tableTitle`, `date`, `seriesDescription`, `value`, `seriesType`,
+`dataType`, `collectionMonth`, `frequency`, `seriesId`, and `unit`. Dates are
+ISO 8601 strings such as `1997-09-01`.
 
-```shell
-npm install @ausecon/core effect
+Leave out `tableNumbers` to read every table of the catalogue number. To read
+series IDs instead, pass a series request:
+
+```ts
+const series = await Effect.runPromise(
+  readAbs(AbsRequest.Series({ seriesIds: ["A2603039T", "A2454446F"] })).pipe(
+    Effect.provide(FetchHttpClient.layer),
+  ),
+);
 ```
 
+Pass `{ hasMetadata: false }` as the second argument to get only the series ID,
+date, and value of each observation.
+
+The read trims spaces from each catalogue number, table number, and series
+ID, and upper-cases table numbers and series IDs. A four-digit catalogue
+number such as `6345` gains `.0`.
+
+## Errors
+
+`Effect.runPromise` rejects when the read fails. Inside Effect code, the
+error channel holds one of these tagged errors:
+
+- `InvalidRequest`: a catalogue number, table number, or series ID is not
+  valid.
+- `CatalogueNotFound`: the Time Series Directory has no series for the
+  catalogue number.
+- `TableNotFound`: the catalogue number has no table with a requested table
+  number.
+- `SeriesNotFound`: the Time Series Directory has no record of a series ID.
+- `WorkbookLayoutError`: a table does not have the ABS time series
+  layout.
+- `HttpClientError`: a request failed or the ABS answered with a status
+  outside 2xx.
+
+`WorkbookLayoutError` names the table ID and, when a sheet fails the check,
+the sheet. A file that does not open as `.xlsx` gives the error without a
+sheet. Each data sheet must hold `Series ID` in column A of row 10, and the
+five metadata labels `Unit`, `Series Type`, `Data Type`, `Frequency`, and
+`Collection Month` in column A of rows 2 to 9. Below row 10, column A must
+hold a date, a blank, or a note that contains `Trend Break` or `see`.
+
+## Your Own `HttpClient`
+
+Any `HttpClient` layer works. The library sends its own User-Agent, follows
+redirects, and fails on any status outside 2xx, so the client only needs to
+send requests.
+
 ## Links
 
 - [Command-line tool](https://www.npmjs.com/package/ausecon)
 - [Source and glossary](https://github.com/jackemcpherson/ausecon)
+- [Changelog](https://github.com/jackemcpherson/ausecon/blob/main/CHANGELOG.md)
 - [`readabs` R package](https://github.com/MattCowgill/readabs)
 - [`readrba` R package](https://github.com/MattCowgill/readrba)
 

package/dist/index.d.ts

@@ -1,4 +1,5 @@
-import { Schema } from "effect";
+import { Data, Effect, Schema } from "effect";
+import { HttpClient, HttpClientError } from "effect/http";
 /**
  * An ABS catalogue number, such as `6202.0` or `6291.0.55.001`.
  */
@@ -18,6 +19,179 @@ export type SeriesId = typeof SeriesId.Type;
 export declare const TableNumber: Schema.brand<Schema.String, "TableNumber">;
 /** A decoded ABS table number. */
 export type TableNumber = typeof TableNumber.Type;
+/**
+ * An ABS table ID: the file name of a table without its extension, such as
+ * `6202001` or `62020x28`.
+ */
+export declare const TableId: Schema.brand<Schema.String, "TableId">;
+/** An ABS table ID. */
+export type TableId = typeof TableId.Type;
+/**
+ * A request to read ABS time series, as the caller writes it.
+ *
+ * The `Catalogue` variant reads the latest release of a catalogue number,
+ * optionally limited to some table numbers. The `Series` variant reads one or
+ * more series IDs. The read normalises and decodes these strings, so `6345`
+ * and lower-case series IDs are accepted.
+ */
+export type AbsRequest = Data.TaggedEnum<{
+  Catalogue: {
+    readonly catalogueNumber: string;
+    readonly tableNumbers?: ReadonlyArray<string>;
+  };
+  Series: {
+    readonly seriesIds: ReadonlyArray<string>;
+  };
+}>;
+/**
+ * Constructors for {@link AbsRequest}.
+ *
+ * @example
+ * ```typescript
+ * const wages = AbsRequest.Catalogue({ catalogueNumber: "6345.0" });
+ * const series = AbsRequest.Series({ seriesIds: ["A2603039T"] });
+ * ```
+ */
+export declare const AbsRequest: {
+  readonly $is: <Tag extends "Catalogue" | "Series">(tag: Tag) => (u: unknown) => u is Extract<{
+    readonly _tag: "Catalogue";
+    readonly catalogueNumber: string;
+    readonly tableNumbers?: ReadonlyArray<string>;
+  }, {
+    readonly _tag: Tag;
+  }> | Extract<{
+    readonly _tag: "Series";
+    readonly seriesIds: ReadonlyArray<string>;
+  }, {
+    readonly _tag: Tag;
+  }>;
+  readonly $match: {
+    <Cases extends { readonly [Tag in "Catalogue" | "Series"]: (args: Extract<{
+      readonly _tag: "Catalogue";
+      readonly catalogueNumber: string;
+      readonly tableNumbers?: ReadonlyArray<string>;
+    } | {
+      readonly _tag: "Series";
+      readonly seriesIds: ReadonlyArray<string>;
+    }, {
+      readonly _tag: Tag;
+    }>) => any; }>(cases: Cases): (value: {
+      readonly _tag: "Catalogue";
+      readonly catalogueNumber: string;
+      readonly tableNumbers?: ReadonlyArray<string>;
+    } | {
+      readonly _tag: "Series";
+      readonly seriesIds: ReadonlyArray<string>;
+    }) => import("effect/Unify").Unify<ReturnType<Cases["Catalogue" | "Series"]>>;
+    <Cases extends { readonly [Tag in "Catalogue" | "Series"]: (args: Extract<{
+      readonly _tag: "Catalogue";
+      readonly catalogueNumber: string;
+      readonly tableNumbers?: ReadonlyArray<string>;
+    } | {
+      readonly _tag: "Series";
+      readonly seriesIds: ReadonlyArray<string>;
+    }, {
+      readonly _tag: Tag;
+    }>) => any; }>(value: {
+      readonly _tag: "Catalogue";
+      readonly catalogueNumber: string;
+      readonly tableNumbers?: ReadonlyArray<string>;
+    } | {
+      readonly _tag: "Series";
+      readonly seriesIds: ReadonlyArray<string>;
+    }, cases: Cases): import("effect/Unify").Unify<ReturnType<Cases["Catalogue" | "Series"]>>;
+  };
+  readonly Catalogue: Data.TaggedEnum.ConstructorFrom<{
+    readonly _tag: "Catalogue";
+    readonly catalogueNumber: string;
+    readonly tableNumbers?: ReadonlyArray<string>;
+  }, "_tag">;
+  readonly Series: Data.TaggedEnum.ConstructorFrom<{
+    readonly _tag: "Series";
+    readonly seriesIds: ReadonlyArray<string>;
+  }, "_tag">;
+};
+/** A decoded catalogue number request. */
+export declare const CatalogueRequest: Schema.TaggedStruct<"Catalogue", {
+  readonly catalogueNumber: Schema.brand<Schema.String, "CatalogueNumber">;
+  readonly tableNumbers: Schema.$Array<Schema.brand<Schema.String, "TableNumber">>;
+}>;
+/** A decoded catalogue number request. */
+export type CatalogueRequest = typeof CatalogueRequest.Type;
+/** A decoded series ID request. */
+export declare const SeriesRequest: Schema.TaggedStruct<"Series", {
+  readonly seriesIds: Schema.NonEmptyArray<Schema.brand<Schema.String, "SeriesId">>;
+}>;
+/** A decoded series ID request. */
+export type SeriesRequest = typeof SeriesRequest.Type;
+/** A decoded {@link AbsRequest}. */
+export declare const DecodedAbsRequest: Schema.Union<readonly [Schema.TaggedStruct<"Catalogue", {
+  readonly catalogueNumber: Schema.brand<Schema.String, "CatalogueNumber">;
+  readonly tableNumbers: Schema.$Array<Schema.brand<Schema.String, "TableNumber">>;
+}>, Schema.TaggedStruct<"Series", {
+  readonly seriesIds: Schema.NonEmptyArray<Schema.brand<Schema.String, "SeriesId">>;
+}>]>;
+/** A decoded {@link AbsRequest}. */
+export type DecodedAbsRequest = typeof DecodedAbsRequest.Type;
+/**
+ * Options for a read.
+ *
+ * Set `hasMetadata` to `false` to return only series ID, date, and value.
+ */
+interface ReadOptions {
+  readonly hasMetadata?: boolean;
+}
+/**
+ * One value of a series for one period, with the series metadata.
+ *
+ * The fields are the 12 `readabs` columns in camelCase, in the same order.
+ */
+export declare const Observation: Schema.Struct<{
+  readonly tableId: Schema.brand<Schema.String, "TableId">;
+  readonly sheet: Schema.String;
+  readonly tableTitle: Schema.String;
+  readonly date: Schema.String;
+  readonly seriesDescription: Schema.String;
+  readonly value: Schema.Number;
+  readonly seriesType: Schema.String;
+  readonly dataType: Schema.String;
+  readonly collectionMonth: Schema.String;
+  readonly frequency: Schema.String;
+  readonly seriesId: Schema.String;
+  readonly unit: Schema.String;
+}>;
+/** One value of a series for one period, with the series metadata. */
+export type Observation = typeof Observation.Type;
+/** One value of a series for one period, without the series metadata. */
+export declare const ObservationValue: Schema.Struct<{
+  readonly seriesId: Schema.String;
+  readonly date: Schema.String;
+  readonly value: Schema.Number;
+}>;
+/** One value of a series for one period, without the series metadata. */
+export type ObservationValue = typeof ObservationValue.Type;
+declare const InvalidRequest_base: Schema.Class<InvalidRequest, Schema.TaggedStruct<"InvalidRequest", {
+  readonly message: Schema.String;
+}>, import("effect/Cause").YieldableError>;
+/**
+ * The request input failed schema decoding.
+ */
+export declare class InvalidRequest extends InvalidRequest_base {}
+declare const CatalogueNotFound_base: Schema.Class<CatalogueNotFound, Schema.TaggedStruct<"CatalogueNotFound", {
+  readonly catalogueNumber: Schema.brand<Schema.String, "CatalogueNumber">;
+}>, import("effect/Cause").YieldableError>;
+/**
+ * The Time Series Directory has no series for the catalogue number.
+ */
+export declare class CatalogueNotFound extends CatalogueNotFound_base {}
+declare const TableNotFound_base: Schema.Class<TableNotFound, Schema.TaggedStruct<"TableNotFound", {
+  readonly catalogueNumber: Schema.brand<Schema.String, "CatalogueNumber">;
+  readonly tableNumbers: Schema.$Array<Schema.brand<Schema.String, "TableNumber">>;
+}>, import("effect/Cause").YieldableError>;
+/**
+ * One or more requested table numbers match no table title.
+ */
+export declare class TableNotFound extends TableNotFound_base {}
 declare const SeriesNotFound_base: Schema.Class<SeriesNotFound, Schema.TaggedStruct<"SeriesNotFound", {
   readonly seriesIds: Schema.$Array<Schema.brand<Schema.String, "SeriesId">>;
 }>, import("effect/Cause").YieldableError>;
@@ -25,4 +199,45 @@ declare const SeriesNotFound_base: Schema.Class<SeriesNotFound, Schema.TaggedStr
  * The Time Series Directory has no record of one or more requested series.
  */
 export declare class SeriesNotFound extends SeriesNotFound_base {}
-export {};
+declare const WorkbookLayoutError_base: Schema.Class<WorkbookLayoutError, Schema.TaggedStruct<"WorkbookLayoutError", {
+  readonly tableId: Schema.brand<Schema.String, "TableId">;
+  readonly sheet: Schema.optionalKey<Schema.String>;
+  readonly reason: Schema.String;
+}>, import("effect/Cause").YieldableError>;
+/**
+ * A table does not have the ABS time series layout.
+ *
+ * `sheet` is absent when the table cannot be opened as `.xlsx` at all.
+ */
+export declare class WorkbookLayoutError extends WorkbookLayoutError_base {}
+/** Every error a read can fail with. */
+type ReadAbsError = InvalidRequest | CatalogueNotFound | TableNotFound | SeriesNotFound | WorkbookLayoutError | HttpClientError.HttpClientError;
+/**
+ * Read ABS time series from the latest release.
+ *
+ * A `Catalogue` request reads the tables of a catalogue number. A `Series`
+ * request reads series IDs. The read queries the Time Series Directory,
+ * downloads each table once, and returns one flat list of observations.
+ *
+ * The Effect requires an `HttpClient`. The library never picks one, so it runs
+ * on any runtime. The client must follow redirects, or return them so that
+ * the library can follow them.
+ *
+ * @param request - What to read.
+ * @param options - Set `hasMetadata: false` to return only series ID, date,
+ * and value.
+ * @returns An Effect that yields the observations.
+ *
+ * @example
+ * ```typescript
+ * const observations = yield* readAbs(AbsRequest.Catalogue({ catalogueNumber: "6345.0" }));
+ * ```
+ */
+export declare function readAbs(request: AbsRequest, options: {
+  readonly hasMetadata: false;
+}): Effect.Effect<ReadonlyArray<ObservationValue>, ReadAbsError, HttpClient.HttpClient>;
+export declare function readAbs(request: AbsRequest, options?: {
+  readonly hasMetadata?: true;
+}): Effect.Effect<ReadonlyArray<Observation>, ReadAbsError, HttpClient.HttpClient>;
+export declare function readAbs(request: AbsRequest, options?: ReadOptions): Effect.Effect<ReadonlyArray<Observation> | ReadonlyArray<ObservationValue>, ReadAbsError, HttpClient.HttpClient>;
+export type { ReadAbsError, ReadOptions };

package/dist/index.js

@@ -1,4 +1,7 @@
-import { Schema } from "effect";
+import { Data, Effect, Match, Option, Schema, Struct } from "effect";
+import { HttpClient, HttpClientError, HttpClientRequest } from "effect/http";
+import { read, utils } from "xlsx";
+import { XMLParser } from "fast-xml-parser";
 // src/types.ts
 /**
 * An ABS catalogue number, such as `6202.0` or `6291.0.55.001`.
@@ -14,7 +17,528 @@ const SeriesId = Schema.String.pipe(Schema.check(Schema.isPattern(/^[A-Z]\d+[A-Z
 */
 const TableNumber = Schema.String.pipe(Schema.check(Schema.isPattern(/^[A-Z]?\d+[A-Za-z]?(?:_[A-Za-z0-9]+)?$/)), Schema.brand("TableNumber"));
 /**
+* An ABS table ID: the file name of a table without its extension, such as
+* `6202001` or `62020x28`.
+*/
+const TableId = Schema.String.pipe(Schema.brand("TableId"));
+/**
+* Constructors for {@link AbsRequest}.
+*
+* @example
+* ```typescript
+* const wages = AbsRequest.Catalogue({ catalogueNumber: "6345.0" });
+* const series = AbsRequest.Series({ seriesIds: ["A2603039T"] });
+* ```
+*/
+const AbsRequest = Data.taggedEnum();
+/** A decoded catalogue number request. */
+const CatalogueRequest = Schema.TaggedStruct("Catalogue", {
+	catalogueNumber: CatalogueNumber,
+	tableNumbers: Schema.Array(TableNumber)
+});
+/** A decoded series ID request. */
+const SeriesRequest = Schema.TaggedStruct("Series", { seriesIds: Schema.NonEmptyArray(SeriesId) });
+/** A decoded {@link AbsRequest}. */
+const DecodedAbsRequest = Schema.Union([CatalogueRequest, SeriesRequest]);
+/**
+* One value of a series for one period, with the series metadata.
+*
+* The fields are the 12 `readabs` columns in camelCase, in the same order.
+*/
+const Observation = Schema.Struct({
+	tableId: TableId,
+	sheet: Schema.String,
+	tableTitle: Schema.String,
+	date: Schema.String,
+	seriesDescription: Schema.String,
+	value: Schema.Number,
+	seriesType: Schema.String,
+	dataType: Schema.String,
+	collectionMonth: Schema.String,
+	frequency: Schema.String,
+	seriesId: Schema.String,
+	unit: Schema.String
+});
+/** One value of a series for one period, without the series metadata. */
+const ObservationValue = Schema.Struct({
+	seriesId: Schema.String,
+	date: Schema.String,
+	value: Schema.Number
+});
+/**
+* The request input failed schema decoding.
+*/
+var InvalidRequest = class extends Schema.TaggedError()("InvalidRequest", { message: Schema.String }) {};
+/**
+* The Time Series Directory has no series for the catalogue number.
+*/
+var CatalogueNotFound = class extends Schema.TaggedError()("CatalogueNotFound", { catalogueNumber: CatalogueNumber }) {};
+/**
+* One or more requested table numbers match no table title.
+*/
+var TableNotFound = class extends Schema.TaggedError()("TableNotFound", {
+	catalogueNumber: CatalogueNumber,
+	tableNumbers: Schema.Array(TableNumber)
+}) {};
+/**
 * The Time Series Directory has no record of one or more requested series.
 */
 var SeriesNotFound = class extends Schema.TaggedError()("SeriesNotFound", { seriesIds: Schema.Array(SeriesId) }) {};
-export { CatalogueNumber, SeriesId, SeriesNotFound, TableNumber };
+/**
+* A table does not have the ABS time series layout.
+*
+* `sheet` is absent when the table cannot be opened as `.xlsx` at all.
+*/
+var WorkbookLayoutError = class extends Schema.TaggedError()("WorkbookLayoutError", {
+	tableId: TableId,
+	sheet: Schema.optionalKey(Schema.String),
+	reason: Schema.String
+}) {};
+// src/abs/http.ts
+/**
+* The User-Agent sent with every request. The ABS answers 403 to a request
+* without one.
+*/
+const USER_AGENT = "ausecon (+https://github.com/jackemcpherson/ausecon)";
+/**
+* The caller's `HttpClient`, set up for ABS requests.
+*
+* It sends the project User-Agent, follows redirects that the underlying
+* client returns, and fails on any status outside 2xx.
+*/
+const absHttpClient = Effect.gen(function* () {
+	return (yield* HttpClient.HttpClient).pipe(HttpClient.mapRequest(HttpClientRequest.setHeader("user-agent", USER_AGENT)), HttpClient.followRedirects(), HttpClient.filterStatusOk);
+});
+// src/abs/table-parse.ts
+/** Sheets that `readabs` skips by exact name. */
+const SKIPPED_SHEETS = /* @__PURE__ */ new Set([
+	"Index",
+	"Inquiries",
+	"Enquiries",
+	"Contents",
+	"Technical note",
+	"Technical notes",
+	"Note",
+	"Notes"
+]);
+/** `readabs` also skips any sheet whose name contains `Table` and a space. */
+const TABLE_SHEET = /Table /;
+/** Zero-based row of the `Series ID` label: spreadsheet row 10. */
+const SERIES_ID_ROW = 9;
+/** Zero-based row of the first observation: spreadsheet row 11. */
+const FIRST_OBSERVATION_ROW = 10;
+const MILLISECONDS_PER_DAY = 864e5;
+/** Excel serial day zero, 1899-12-30, in UTC. */
+const EXCEL_EPOCH = Date.UTC(1899, 11, 30);
+/** The column A labels of the metadata rows that an observation carries. */
+const METADATA_LABELS = {
+	unit: "Unit",
+	seriesType: "Series Type",
+	dataType: "Data Type",
+	frequency: "Frequency",
+	collectionMonth: "Collection Month"
+};
+const cellAt = (sheet, row, column) => {
+	return sheet[utils.encode_cell({
+		r: row,
+		c: column
+	})];
+};
+const cellText = (cell) => String(cell?.v ?? "");
+const cellNumber = (cell) => cell?.t === "n" ? Number(cell.v) : void 0;
+/** Convert an Excel serial day number to an ISO `YYYY-MM-DD` date in UTC. */
+const excelSerialToIsoDate = (serial) => new Date(EXCEL_EPOCH + Math.floor(serial) * MILLISECONDS_PER_DAY).toISOString().slice(0, 10);
+/** Whether column A text is a note that `readabs` drops, such as `Trend Break` or `see ...`. */
+const isNote = (text) => text.includes("Trend Break") || text.includes("see");
+/** Whether a column A cell below row 10 is neither a date, blank, nor a note. */
+const isUnexpectedRowLabel = (cell) => {
+	const text = cellText(cell).trim();
+	return cell?.t !== "n" && text !== "" && !isNote(text);
+};
+/** Whether `readabs` reads a sheet as data. */
+const isDataSheet = (sheetName) => !SKIPPED_SHEETS.has(sheetName) && !TABLE_SHEET.test(sheetName);
+/** The zero-based rows from `first` to `last`, inclusive. */
+const rowsBetween = (first, last) => Array.from({ length: Math.max(last - first + 1, 0) }, (_, index) => first + index);
+/**
+* Check that a sheet has the ABS time series layout.
+*
+* Column A of row 10 must hold `Series ID`, rows 2 to 9 must hold each
+* metadata label that an observation carries, and each column A cell below
+* row 10 must hold a date, a blank, or a note.
+*/
+const checkLayout = (sheet, name, table) => {
+	const layoutError = (reason) => Effect.fail(new WorkbookLayoutError({
+		tableId: table.tableId,
+		sheet: name,
+		reason
+	}));
+	const ref = sheet["!ref"];
+	if (ref === void 0 || cellText(cellAt(sheet, SERIES_ID_ROW, 0)) !== "Series ID") return layoutError("Column A of row 10 is not `Series ID`.");
+	const labelRows = new Map(rowsBetween(1, 8).map((row) => [cellText(cellAt(sheet, row, 0)), row]));
+	const rowOf = (field) => Option.fromUndefinedOr(labelRows.get(METADATA_LABELS[field]));
+	const metadataRows = Option.all({
+		unit: rowOf("unit"),
+		seriesType: rowOf("seriesType"),
+		dataType: rowOf("dataType"),
+		frequency: rowOf("frequency"),
+		collectionMonth: rowOf("collectionMonth")
+	});
+	if (Option.isNone(metadataRows)) return layoutError(`Rows 2 to 9 lack the metadata rows: ${Struct.keys(METADATA_LABELS).filter((field) => Option.isNone(rowOf(field))).join(", ")}.`);
+	const range = utils.decode_range(ref);
+	const rows = rowsBetween(FIRST_OBSERVATION_ROW, range.e.r);
+	const unexpectedRow = rows.find((row) => isUnexpectedRowLabel(cellAt(sheet, row, 0)));
+	if (unexpectedRow !== void 0) {
+		const text = cellText(cellAt(sheet, unexpectedRow, 0));
+		return layoutError(`Cell A${unexpectedRow + 1} holds \`${text}\`, which is not a date, a blank, or a note.`);
+	}
+	return Effect.succeed({
+		name,
+		sheet,
+		lastColumn: range.e.c,
+		metadataRows: metadataRows.value,
+		periods: rows.flatMap((row) => {
+			const serial = cellNumber(cellAt(sheet, row, 0));
+			return serial === void 0 ? [] : [[row, excelSerialToIsoDate(serial)]];
+		})
+	});
+};
+/** Read the observations of one checked data sheet. */
+const sheetObservations = (data, table) => {
+	const { sheet, metadataRows } = data;
+	return rowsBetween(1, data.lastColumn).flatMap((column) => {
+		const seriesId = cellText(cellAt(sheet, SERIES_ID_ROW, column));
+		if (seriesId === "") return [];
+		const metadata = (field) => cellText(cellAt(sheet, metadataRows[field], column));
+		const seriesDescription = cellText(cellAt(sheet, 0, column));
+		const seriesType = metadata("seriesType");
+		const dataType = metadata("dataType");
+		const collectionMonth = metadata("collectionMonth");
+		const frequency = metadata("frequency");
+		const unit = metadata("unit");
+		return data.periods.flatMap(([row, date]) => {
+			const value = cellNumber(cellAt(sheet, row, column));
+			return value === void 0 ? [] : [{
+				tableId: table.tableId,
+				sheet: data.name,
+				tableTitle: table.tableTitle,
+				date,
+				seriesDescription,
+				value,
+				seriesType,
+				dataType,
+				collectionMonth,
+				frequency,
+				seriesId,
+				unit
+			}];
+		});
+	});
+};
+/**
+* Read the observations of an ABS time series table.
+*
+* Skips the sheets `readabs` skips. Each data sheet must have `Series ID` in
+* column A of row 10. Below it, column A holds a date, a blank, or a note that
+* contains `Trend Break` or `see`; any other text fails the read. Rows with a
+* blank or a note, and blank or text value cells, are left out. Dates come
+* from Excel serial numbers, which is how `.xlsx` stores date cells.
+*
+* @param bytes - The `.xlsx` file contents.
+* @param table - The table ID and title to put on each observation.
+* @returns An Effect that yields the observations, or fails with
+* `WorkbookLayoutError`.
+*/
+const parseTable = Effect.fn("parseTable")(function* (bytes, table) {
+	const file = yield* Effect.try({
+		try: () => read(bytes, {
+			type: "array",
+			cellDates: false,
+			cellHTML: false
+		}),
+		catch: (cause) => new WorkbookLayoutError({
+			tableId: table.tableId,
+			reason: `The table cannot be opened as .xlsx: ${String(cause)}`
+		})
+	});
+	return (yield* Effect.forEach(Object.entries(file.Sheets).filter(([name]) => isDataSheet(name)), ([name, sheet]) => checkLayout(sheet, name, table))).flatMap((data) => sheetObservations(data, table));
+});
+// src/abs/download.ts
+/** The most tables downloaded at once. */
+const DOWNLOAD_CONCURRENCY = 4;
+/**
+* Download and parse tables.
+*
+* Downloads each distinct `TableURL` once, with a concurrency limit, after
+* replacing spaces in the URL with `+`.
+*
+* @param tables - The tables to read.
+* @param isWanted - Whether to keep an observation of a table.
+* @returns An Effect that yields the kept observations of every table, in
+* table order.
+*/
+const readTables = Effect.fn("readTables")(function* (tables, isWanted) {
+	const client = yield* absHttpClient;
+	const distinct = [...new Map(tables.map((table) => [table.tableUrl, table])).values()];
+	return (yield* Effect.forEach(distinct, (table) => client.get(table.tableUrl.replaceAll(" ", "+")).pipe(Effect.flatMap((response) => response.arrayBuffer), Effect.flatMap((body) => parseTable(new Uint8Array(body), table)), Effect.map((observations) => observations.filter((observation) => isWanted(table, observation)))), { concurrency: DOWNLOAD_CONCURRENCY })).flat();
+});
+// src/abs/latest-release.ts
+/**
+* Keep only rows that link to a table, from the latest release of each table.
+*
+* A table is one `TableURL`, as `readabs` groups it in `read_abs`. Rows with an
+* empty `TableURL` link to no table.
+*
+* @param rows - Directory rows.
+* @returns The rows with a `TableURL` whose release date is the latest for
+* their table.
+*/
+const latestRows = (rows) => {
+	const linked = rows.filter((row) => row.tableUrl.trim() !== "");
+	const latestDates = new Map([...Map.groupBy(linked, (row) => row.tableUrl)].map(([tableUrl, group]) => [tableUrl, group.reduce((latest, row) => row.releaseDate > latest ? row.releaseDate : latest, "")]));
+	return linked.filter((row) => latestDates.get(row.tableUrl) === row.releaseDate);
+};
+/** The file name of a table URL without its extension. */
+const tableIdOf = (tableUrl) => TableId.make(tableUrl.replace(/[?#].*$/, "").replace(/^.*\//, "").replace(/\.[^.]*$/, ""));
+/**
+* List the distinct tables of directory rows, in row order.
+*
+* @param rows - Directory rows, already filtered to the latest release.
+* @returns One entry per `TableURL`, titled by its first row.
+*/
+const tablesOf = (rows) => [...Map.groupBy(rows, (row) => row.tableUrl).values()].flatMap(([first]) => first === void 0 ? [] : [{
+	tableId: tableIdOf(first.tableUrl),
+	tableTitle: first.tableTitle,
+	tableUrl: first.tableUrl
+}]);
+// src/abs/request.ts
+const FOUR_DIGIT_CATALOGUE_NUMBER = /^\d{4}$/;
+const normaliseCatalogueNumber = (catalogueNumber) => {
+	const trimmed = catalogueNumber.trim();
+	return FOUR_DIGIT_CATALOGUE_NUMBER.test(trimmed) ? `${trimmed}.0` : trimmed;
+};
+const normaliseCode = (code) => code.trim().toUpperCase();
+/**
+* Normalise caller input before schema decoding.
+*
+* Appends `.0` to a four-digit catalogue number and upper-cases table numbers
+* and series IDs.
+*/
+const normalise = AbsRequest.$match({
+	Catalogue: (request) => AbsRequest.Catalogue({
+		catalogueNumber: normaliseCatalogueNumber(request.catalogueNumber),
+		tableNumbers: (request.tableNumbers ?? []).map(normaliseCode)
+	}),
+	Series: (request) => AbsRequest.Series({ seriesIds: request.seriesIds.map(normaliseCode) })
+});
+/**
+* Normalise and decode a request.
+*
+* @param request - The request as the caller wrote it.
+* @returns An Effect that yields the decoded request, or fails with
+* `InvalidRequest`.
+*/
+const decodeRequest = Effect.fn("decodeRequest")(function* (request) {
+	return yield* Schema.decodeUnknownEffect(DecodedAbsRequest)(normalise(request));
+}, Effect.mapError((error) => new InvalidRequest({ message: error.message })));
+// src/abs/table-number-parse.ts
+const TITLE_PREFIX = /^(?:Tables|Table|TABLE|Data cube)[ _]/;
+const NUMBER = String.raw`[A-Za-z]?\d+[A-Za-z]?(?:_[A-Za-z0-9]+)?`;
+const NUMBER_LIST = new RegExp(String.raw`^${NUMBER}(?:(?:,? and |, )${NUMBER})+(?=[.: ]|$)`);
+const LIST_SEPARATOR = /,? and |, /;
+const NUMBER_RANGE = /^([A-Za-z]?)(\d+)([A-Za-z]?) to ([A-Za-z]?)(\d+)([A-Za-z]?)(?=[.: ]|$)/;
+const NUMBER_END = /[.: ]|$/;
+/** A table number in upper case without leading zeros, for comparison. */
+const comparable = (tableNumber) => tableNumber.toUpperCase().replace(/^([A-Z]?)0+(?=\d)/, "$1");
+const NOTHING = () => false;
+const oneOf = (numbers) => (tableNumber) => numbers.map(comparable).includes(tableNumber);
+const NUMBER_PARTS = /^([A-Z]?)(\d+)([A-Z]?)$/;
+/**
+* Test whether a range names a table number. A range whose ends carry the
+* same letters, such as `01 to 17` or `2a to 9a`, names each number from one
+* end to the other. A range whose letters differ, such as `2a to 4b`, names
+* only its two ends, as ADR 0005 records.
+*
+* The test compares numbers rather than listing the range, so a title with a
+* huge range costs no memory.
+*/
+const rangeMatch = (range) => {
+	const [, firstPrefix = "", firstDigits = "", firstSuffix = "", lastPrefix = "", lastDigits = "", lastSuffix = ""] = range;
+	const prefix = firstPrefix.toUpperCase();
+	const suffix = firstSuffix.toUpperCase();
+	if (prefix !== lastPrefix.toUpperCase() || suffix !== lastSuffix.toUpperCase()) return oneOf([`${firstPrefix}${firstDigits}${firstSuffix}`, `${lastPrefix}${lastDigits}${lastSuffix}`]);
+	const first = Number(firstDigits);
+	const last = Number(lastDigits);
+	return (tableNumber) => {
+		const [, numberPrefix, digits = "", numberSuffix] = NUMBER_PARTS.exec(tableNumber) ?? [];
+		const number = Number(digits);
+		return numberPrefix === prefix && numberSuffix === suffix && number >= first && number <= last;
+	};
+};
+/** Parse the table numbers a title names into a test. */
+const titleMatch = (title) => {
+	const prefix = TITLE_PREFIX.exec(title);
+	if (prefix === null) return NOTHING;
+	const rest = title.slice(prefix[0].length);
+	const range = NUMBER_RANGE.exec(rest);
+	if (range !== null) return rangeMatch(range);
+	const list = NUMBER_LIST.exec(rest);
+	if (list !== null) return oneOf(list[0].split(LIST_SEPARATOR));
+	const end = rest.search(NUMBER_END);
+	const number = rest.slice(0, end);
+	return oneOf([/[.:]/.test(rest.charAt(end)) ? number : number.replace(/_.*$/, "")]);
+};
+/**
+* Test whether a table title names a table number.
+*
+* @param title - The `TableTitle` of a table.
+* @param tableNumber - The requested table number.
+* @returns Whether the title names the table number.
+*/
+const titleNamesTable = (title, tableNumber) => titleMatch(title)(comparable(tableNumber));
+// src/abs/time-series-directory-parse.ts
+/** A Time Series Directory response that is not a series index. */
+var DirectoryDocumentError = class extends Schema.TaggedError()("DirectoryDocumentError", { reason: Schema.String }) {};
+const DIRECTORY_DATE = /^(\d{2})\/(\d{2})\/(\d{4})$/;
+const SeriesElement = Schema.Struct({
+	ProductReleaseDate: Schema.String.pipe(Schema.check(Schema.isPattern(DIRECTORY_DATE))),
+	TableURL: Schema.String,
+	TableTitle: Schema.String,
+	TableOrder: Schema.String.pipe(Schema.check(Schema.isPattern(/^\d+$/))),
+	SeriesID: Schema.String
+});
+const SeriesIndex = Schema.Struct({
+	NumPages: Schema.optionalKey(Schema.String.pipe(Schema.check(Schema.isPattern(/^[1-9]\d*$/)))),
+	Series: Schema.optionalKey(Schema.Array(SeriesElement))
+});
+const DirectoryDocument = Schema.Struct({
+	TimeSeriesIndex: Schema.optionalKey(Schema.Union([Schema.Literal(""), SeriesIndex])),
+	Error: Schema.optionalKey(Schema.String)
+});
+const parser = new XMLParser({
+	ignoreDeclaration: true,
+	isArray: (name) => name === "Series",
+	parseTagValue: false
+});
+/** Convert a `DD/MM/YYYY` directory date to an ISO `YYYY-MM-DD` string. */
+const isoDate = (directoryDate) => directoryDate.replace(DIRECTORY_DATE, "$3-$2-$1");
+const toRow = (series) => ({
+	releaseDate: isoDate(series.ProductReleaseDate),
+	tableUrl: series.TableURL,
+	tableTitle: series.TableTitle,
+	tableOrder: Number(series.TableOrder),
+	seriesId: series.SeriesID
+});
+/**
+* Parse one Time Series Directory page.
+*
+* A page without `NumPages` is the only page. An empty index parses to a page
+* with no rows.
+*
+* @param xml - The response body.
+* @returns An Effect that yields the page, or fails with
+* `DirectoryDocumentError` for an `<Error>` document or an unexpected layout.
+*/
+const parseDirectoryPage = Effect.fn("parseDirectoryPage")(function* (xml) {
+	const parsed = yield* Effect.try({
+		try: () => parser.parse(xml, true),
+		catch: (cause) => new DirectoryDocumentError({ reason: `The XML is malformed: ${String(cause)}` })
+	});
+	const document = yield* Schema.decodeUnknownEffect(DirectoryDocument)(parsed).pipe(Effect.mapError((error) => new DirectoryDocumentError({ reason: `Unexpected layout: ${error.message}` })));
+	if (document.Error !== void 0) return yield* new DirectoryDocumentError({ reason: `The Time Series Directory answered: ${document.Error}` });
+	if (document.TimeSeriesIndex === void 0) return yield* new DirectoryDocumentError({ reason: "No TimeSeriesIndex element." });
+	if (document.TimeSeriesIndex === "") return {
+		pageCount: 1,
+		rows: []
+	};
+	return {
+		pageCount: Number(document.TimeSeriesIndex.NumPages ?? "1"),
+		rows: (document.TimeSeriesIndex.Series ?? []).map(toRow)
+	};
+});
+// src/abs/time-series-directory.ts
+/** The documented Time Series Directory base URL. */
+const DIRECTORY_URL = "https://abs.gov.au/servlet/TSSearchServlet";
+/** The most Time Series Directory pages fetched at once. */
+const PAGE_CONCURRENCY = 4;
+const fetchPage = (client, request) => Effect.gen(function* () {
+	const response = yield* client.execute(request);
+	const xml = yield* response.text;
+	return yield* parseDirectoryPage(xml).pipe(Effect.mapError((error) => decodeError(request, response, error.reason)));
+});
+const decodeError = (request, response, description) => new HttpClientError.HttpClientError({ reason: new HttpClientError.DecodeError({
+	request,
+	response,
+	description
+}) });
+/**
+* Fetch every Time Series Directory row for one query.
+*
+* Reads `NumPages` from page 1, then fetches the other pages with a small
+* concurrency limit. An unknown catalogue number or series ID yields no rows.
+*
+* @param query - The query parameter, `catno` or `sid`.
+* @param value - The decoded catalogue number or series ID.
+* @returns An Effect that yields the rows of every page, in page order.
+*/
+const fetchDirectory = Effect.fn("fetchDirectory")(function* (query, value) {
+	const client = yield* absHttpClient;
+	const request = HttpClientRequest.get(DIRECTORY_URL).pipe(HttpClientRequest.setUrlParam(query, value));
+	const firstPage = yield* fetchPage(client, request);
+	const pageNumbers = Array.from({ length: firstPage.pageCount - 1 }, (_, index) => index + 2);
+	return [firstPage, ...yield* Effect.forEach(pageNumbers, (pageNumber) => fetchPage(client, request.pipe(HttpClientRequest.setUrlParam("pg", `${pageNumber}`))), { concurrency: PAGE_CONCURRENCY })].flatMap((page) => page.rows);
+});
+// src/abs/time-series.ts
+/**
+* Limit the tables of a catalogue number to the requested table numbers.
+*
+* A request without table numbers reads every table. Otherwise the read keeps
+* each table whose title names a requested table number, as ADR 0005 records,
+* and fails with `TableNotFound` if any requested table number names no table.
+*/
+const selectTables = (request, tables) => {
+	if (request.tableNumbers.length === 0) return Effect.succeed(tables);
+	const matches = request.tableNumbers.map((tableNumber) => [tableNumber, tables.filter((table) => titleNamesTable(table.tableTitle, tableNumber))]);
+	const unmatched = matches.flatMap(([tableNumber, found]) => found.length === 0 ? [tableNumber] : []);
+	if (unmatched.length > 0) return Effect.fail(new TableNotFound({
+		catalogueNumber: request.catalogueNumber,
+		tableNumbers: unmatched
+	}));
+	const selected = new Set(matches.flatMap(([, found]) => found));
+	return Effect.succeed(tables.filter((table) => selected.has(table)));
+};
+/** The tables a catalogue request reads, from the latest release. */
+const catalogueTables = (request) => Effect.gen(function* () {
+	const rows = latestRows(yield* fetchDirectory("catno", request.catalogueNumber));
+	if (rows.length === 0) return yield* new CatalogueNotFound({ catalogueNumber: request.catalogueNumber });
+	return yield* selectTables(request, tablesOf(rows));
+});
+const readCatalogue = (request) => catalogueTables(request).pipe(Effect.flatMap((tables) => readTables(tables, () => true)));
+/** The most series IDs looked up in the Time Series Directory at once. */
+const SERIES_CONCURRENCY = 4;
+/**
+* The directory row a series is read from: the lowest `TableOrder` of the
+* latest release, as `readabs` picks it, so no series appears twice.
+*/
+const seriesRow = (seriesId) => fetchDirectory("sid", seriesId).pipe(Effect.map((rows) => [seriesId, latestRows(rows).toSorted((first, second) => first.tableOrder - second.tableOrder).at(0)]));
+const readSeries = (request) => Effect.gen(function* () {
+	const found = yield* Effect.forEach(new Set(request.seriesIds), seriesRow, { concurrency: SERIES_CONCURRENCY });
+	const missing = found.flatMap(([seriesId, row]) => row === void 0 ? [seriesId] : []);
+	if (missing.length > 0) return yield* new SeriesNotFound({ seriesIds: missing });
+	const rows = found.flatMap(([, row]) => row === void 0 ? [] : [row]);
+	const wanted = new Set(rows.map((row) => `${row.tableUrl} ${row.seriesId}`));
+	return yield* readTables(tablesOf(rows), (table, observation) => wanted.has(`${table.tableUrl} ${observation.seriesId}`));
+});
+const readDecoded = Match.type().pipe(Match.tag("Catalogue", readCatalogue), Match.tag("Series", readSeries), Match.exhaustive);
+const withoutMetadata = (observation) => ({
+	seriesId: observation.seriesId,
+	date: observation.date,
+	value: observation.value
+});
+const read$1 = Effect.fn("readAbs")(function* (request, options) {
+	const observations = yield* readDecoded(yield* decodeRequest(request));
+	return options?.hasMetadata === false ? observations.map(withoutMetadata) : observations;
+});
+function readAbs(request, options) {
+	return read$1(request, options);
+}
+export { AbsRequest, CatalogueNotFound, CatalogueNumber, CatalogueRequest, DecodedAbsRequest, InvalidRequest, Observation, ObservationValue, SeriesId, SeriesNotFound, SeriesRequest, TableId, TableNotFound, TableNumber, WorkbookLayoutError, readAbs };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ausecon/core",
-  "version": "0.0.0",
+  "version": "0.1.0",
   "description": "Read Australian economic data from the ABS and RBA.",
   "license": "MIT",
   "repository": {
@@ -24,6 +24,7 @@
   },
   "dependencies": {
     "effect": "4.0.6",
+    "fast-xml-parser": "5.11.2",
     "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz"
   },
   "engines": {