@fin.cx/opendata 4.6.0 → 4.8.0

package/ts/financialstatements/html.ts

@@ -0,0 +1,107 @@
+import * as plugins from './plugins.js';
+import { FinancialStatementError } from './classes.financialstatementerror.js';
+
+export type THtmlDocument = plugins.parse5.DefaultTreeAdapterTypes.Document;
+export type THtmlElement = plugins.parse5.DefaultTreeAdapterTypes.Element;
+type THtmlParentNode = plugins.parse5.DefaultTreeAdapterTypes.ParentNode;
+type THtmlChildNode = plugins.parse5.DefaultTreeAdapterTypes.ChildNode;
+
+export const parseHtml = (htmlArg: string): THtmlDocument => plugins.parse5.parse(htmlArg);
+
+export const isElement = (nodeArg: THtmlChildNode): nodeArg is THtmlElement => 'tagName' in nodeArg;
+
+/** the element children of a node */
+export const elementChildren = (nodeArg: THtmlParentNode): THtmlElement[] =>
+  nodeArg.childNodes.filter(isElement);
+
+/** every element below a node, in document order */
+export const descendants = (nodeArg: THtmlParentNode): THtmlElement[] => {
+  const result: THtmlElement[] = [];
+  const stack: THtmlElement[] = elementChildren(nodeArg).reverse();
+  while (stack.length > 0) {
+    const element = stack.pop() as THtmlElement;
+    result.push(element);
+    const children = element.tagName === 'template' ? [] : elementChildren(element);
+    for (let index = children.length - 1; index >= 0; index--) {
+      stack.push(children[index]);
+    }
+  }
+  return result;
+};
+
+export const getAttribute = (elementArg: THtmlElement, nameArg: string): string | undefined =>
+  elementArg.attrs.find((attrArg) => attrArg.name === nameArg)?.value;
+
+export const hasClass = (elementArg: THtmlElement, classArg: string): boolean =>
+  (getAttribute(elementArg, 'class') ?? '').split(/\s+/).includes(classArg);
+
+/**
+ * the raw text of a node and everything below it, scripts and styles left out: the text nodes in
+ * document order, separated by spaces. Walks the tree with its own stack, so no nesting depth a page
+ * can have exhausts the call stack.
+ */
+const rawText = (nodeArg: THtmlChildNode | THtmlParentNode): string => {
+  const pieces: string[] = [];
+  const stack: Array<THtmlChildNode | THtmlParentNode> = [nodeArg];
+  for (let node = stack.pop(); node !== undefined; node = stack.pop()) {
+    if ('value' in node && node.nodeName === '#text') {
+      pieces.push(node.value);
+      continue;
+    }
+    if (!('childNodes' in node)) {
+      continue;
+    }
+    if ('tagName' in node && (node.tagName === 'script' || node.tagName === 'style')) {
+      continue;
+    }
+    for (let index = node.childNodes.length - 1; index >= 0; index--) {
+      stack.push(node.childNodes[index]);
+    }
+  }
+  return pieces.join(' ');
+};
+
+/** the text of a node with white space collapsed */
+export const textOf = (nodeArg: THtmlChildNode | THtmlParentNode): string =>
+  rawText(nodeArg).replace(/\s+/g, ' ').trim();
+
+/** the encodings an HTML statement may declare, by the names it may declare them with */
+const knownEncodings: Record<string, string> = {
+  'utf-8': 'utf-8',
+  utf8: 'utf-8',
+  'iso-8859-1': 'windows-1252',
+  'iso-8859-15': 'iso-8859-15',
+  latin1: 'windows-1252',
+  'windows-1252': 'windows-1252',
+  cp1252: 'windows-1252',
+};
+
+/**
+ * Decodes an HTML document: by its byte order mark, else by the charset its first 1,024 bytes
+ * declare, else as UTF-8. Bytes that are not text in that encoding are `unreadable`.
+ */
+export const decodeHtml = (bytesArg: Uint8Array): string => {
+  let encoding = 'utf-8';
+  let start = 0;
+  if (bytesArg[0] === 0xef && bytesArg[1] === 0xbb && bytesArg[2] === 0xbf) {
+    start = 3;
+  } else if ((bytesArg[0] === 0xff && bytesArg[1] === 0xfe) || (bytesArg[0] === 0xfe && bytesArg[1] === 0xff)) {
+    encoding = bytesArg[0] === 0xff ? 'utf-16le' : 'utf-16be';
+    start = 2;
+  } else {
+    const head = new TextDecoder('windows-1252').decode(bytesArg.subarray(0, 1024));
+    const declared = /<meta[^>]+charset\s*=\s*["']?\s*([\w-]+)/i.exec(head)?.[1].toLowerCase();
+    if (declared !== undefined) {
+      const known = knownEncodings[declared];
+      if (known === undefined) {
+        throw new FinancialStatementError('unreadable', `The document declares the encoding "${declared}", which is not read here.`);
+      }
+      encoding = known;
+    }
+  }
+  try {
+    return new TextDecoder(encoding, { fatal: true }).decode(bytesArg.subarray(start));
+  } catch (error) {
+    throw new FinancialStatementError('unreadable', `The document is not text in ${encoding}.`, { cause: error });
+  }
+};

package/ts/financialstatements/index.ts

@@ -0,0 +1,17 @@
+// This entry point stands alone: it imports nothing from the rest of the package and keeps nothing.
+// It reads published financial statements from the bytes a caller obtained: statements under the
+// HGB from the HTML or the PDF a register shows, and ESEF reports. The caller stores the document
+// and what was read from it; `parsed` is null rather than guessed.
+export * from './interfaces.js';
+export { FinancialStatementError, type TFinancialStatementErrorCode } from './classes.financialstatementerror.js';
+export {
+  maxEsefFacts,
+  maxEsefPackageEntries,
+  maxPdfPages,
+  maxStatementBytes,
+  unternehmensregisterNotice,
+  unternehmensregisterOrigin,
+} from './constants.js';
+export { parseStatementDocument } from './parse.statement.js';
+export { parseEsefPackage, parseEsefReport } from './parse.esef.js';
+export { readFinancialStatement } from './read.js';

package/ts/financialstatements/interfaces.ts

@@ -0,0 +1,268 @@
+/** An ISO date, `2024-12-31`. */
+export type TIsoDate = string;
+
+/**
+ * Where a statement was published:
+ * - `unternehmensregister`: the Unternehmensregister, for financial years beginning after 31.12.2021,
+ *   and for the balance sheets micro companies deposit there (§ 326 Abs. 2 HGB);
+ * - `bundesanzeiger`: the Bundesanzeiger, for earlier financial years;
+ * - `issuer`: the company's own publication, as listed issuers publish their ESEF reports;
+ * - `unknown`: a document whose origin the caller does not know.
+ */
+export type TFilingSource = 'unternehmensregister' | 'bundesanzeiger' | 'issuer' | 'unknown';
+
+/**
+ * What a filing is:
+ * - `annual_accounts`: the Jahresabschluss of one company (§§ 242, 264 HGB);
+ * - `consolidated_accounts`: a group's Konzernabschluss (§ 290 HGB);
+ * - `deposited_balance_sheet`: the balance sheet a micro company deposits instead of publishing
+ *   (§ 326 Abs. 2 HGB);
+ * - `annual_financial_report`: a listed issuer's Jahresfinanzbericht (§ 114 WpHG), as an ESEF report.
+ */
+export type TFilingKind = 'annual_accounts' | 'consolidated_accounts' | 'deposited_balance_sheet' | 'annual_financial_report';
+
+/** The size classes of § 267 and § 267a HGB, as a filing states them. */
+export type TSizeClass = 'micro' | 'small' | 'medium' | 'large';
+
+/**
+ * One published statement as its publisher lists it. Every field the caller does not know is null;
+ * none is derived from the document.
+ */
+export interface IFiling {
+  source: TFilingSource;
+  kind: TFilingKind;
+  /** the company's name as the publisher writes it */
+  companyName: string | null;
+  /** the register sheet, as the caller writes it: `Amtsgericht Bremen HRB 35230 HB` */
+  registerReference: string | null;
+  /** the company's Legal Entity Identifier */
+  lei: string | null;
+  fiscalYearStart: TIsoDate | null;
+  fiscalYearEnd: TIsoDate | null;
+  /**
+   * the day the publisher posted the document; for a deposited balance sheet, the day it was
+   * deposited. The Unternehmensregister's origin note names this day.
+   */
+  publishedOn: TIsoDate | null;
+  /** the publisher's own title of the document */
+  title: string | null;
+  sizeClass: TSizeClass | null;
+  /** the publisher's identifier of the document */
+  documentId: string | null;
+  /** where the publisher shows the document */
+  sourceUrl: string | null;
+}
+
+/** What a caller states about a filing: `source` and `kind`, and whatever else it knows. */
+export interface IFilingInput extends Pick<IFiling, 'source' | 'kind'>, Partial<Omit<IFiling, 'source' | 'kind'>> {}
+
+/**
+ * The media types this entry point reads:
+ * - `text/html`: a statement as the Unternehmensregister or the Bundesanzeiger shows it;
+ * - `application/pdf`: the same as a PDF with a text layer;
+ * - `application/xhtml+xml`: an ESEF report (inline XBRL) on its own;
+ * - `application/zip`: an ESEF report package (`.zip` or `.xbri`).
+ */
+export type TStatementMediaType = 'text/html' | 'application/pdf' | 'application/xhtml+xml' | 'application/zip';
+
+/** The document as it was obtained, byte for byte. */
+export interface IRawStatementDocument {
+  bytes: Uint8Array;
+  mediaType: TStatementMediaType;
+  /** the SHA-256 of `bytes`, lowercase hex */
+  sha256: string;
+}
+
+/**
+ * The unit a statement prints its amounts in. Amounts in this entry point are always in the
+ * currency itself, the unit's factor applied: `1.234` printed in TEUR is `'1234000'`.
+ */
+export interface IStatementUnit {
+  /** as this entry point names it */
+  label: 'EUR' | 'TEUR' | 'Mio. EUR';
+  /** the number of euros one printed unit stands for */
+  factor: '1' | '1000' | '1000000';
+}
+
+/** One of the two years a statement shows. */
+export interface IStatementPeriod {
+  /** the column heading as printed, `Geschäftsjahr Euro` or `31.12.2024`; null where none is printed */
+  heading: string | null;
+  /** the first day, where the statement states it */
+  start: TIsoDate | null;
+  /** the balance sheet date, the last day of the period, where the statement states it */
+  end: TIsoDate | null;
+}
+
+/**
+ * The figures most readers look for first, as decimal strings in the currency. Each is null where
+ * the statement does not print it:
+ * - `totalAssets`: the balance sheet total;
+ * - `equity`: the equity section of the balance sheet, its printed total or, where the statement
+ *   prints only its parts, their sum;
+ * - `netIncome`: the net income or loss (Jahresüberschuss or Jahresfehlbetrag), from the income
+ *   statement or, where none is published, from the balance sheet's equity section;
+ * - `revenue`: the revenue (Umsatzerlöse); small companies may print only a gross result
+ *   (Rohergebnis, § 276 HGB), and then it is null.
+ */
+export interface IKeyFigures {
+  totalAssets: string | null;
+  equity: string | null;
+  netIncome: string | null;
+  revenue: string | null;
+}
+
+/**
+ * The kind of a printed line:
+ * - `position`: a position of the outline of § 266 or § 275 HGB, or a line the statement adds to it;
+ * - `subtotal`: a printed sum of the lines above it, such as Summe Anlagevermögen, or an
+ *   intermediate result of the income statement, such as Rohergebnis or Ergebnis nach Steuern;
+ *   no total adds it again;
+ * - `total`: the total of a side of the balance sheet, or the income statement's net income;
+ * - `memo`: a "davon" line, part of the line above it and never added.
+ */
+export type TStatementLineKind = 'position' | 'subtotal' | 'total' | 'memo';
+
+/** One printed line of a balance sheet or an income statement. */
+export interface IStatementPosition {
+  /** the outline path of the line, its printed codes joined: `A.I.1`; null for a line without a code */
+  code: string | null;
+  /** the code as printed: `1.` or `a)` */
+  printedCode: string | null;
+  /** the line's text as printed, without its code */
+  label: string;
+  /** the depth in the outline, 0 for the top level */
+  level: number;
+  kind: TStatementLineKind;
+  /** the amount for the current year as a decimal string in the currency, null where none is printed */
+  current: string | null;
+  /** the same for the prior year */
+  prior: string | null;
+}
+
+export interface IBalanceSheet {
+  unit: IStatementUnit;
+  /** Aktiva, in printed order, with the total last */
+  assets: IStatementPosition[];
+  /** Passiva, in printed order, with the total last */
+  equityAndLiabilities: IStatementPosition[];
+}
+
+/**
+ * The method of an income statement: `total_cost` (Gesamtkostenverfahren, § 275 Abs. 2 HGB) or
+ * `cost_of_sales` (Umsatzkostenverfahren, § 275 Abs. 3 HGB); null where its lines do not tell.
+ */
+export type TIncomeStatementMethod = 'total_cost' | 'cost_of_sales';
+
+export interface IIncomeStatement {
+  unit: IStatementUnit;
+  method: TIncomeStatementMethod | null;
+  /**
+   * how the statement prints its expenses: `negative`, with a minus, so that its lines add up as
+   * printed, or `positive`, as amounts to subtract. The positions keep the amounts as printed.
+   */
+  expenses: 'negative' | 'positive';
+  positions: IStatementPosition[];
+}
+
+/**
+ * A statement under the HGB read from the HTML or the PDF a register shows. Everything in it was
+ * printed and adds up: each position with parts equals their sum, each side of the balance sheet its
+ * total, both totals each other, and the income statement's lines its net income. A prior year whose
+ * figures do not add up, and an income statement that does not, are left out with a note.
+ */
+export interface IHgbStatement {
+  standard: 'hgb';
+  currency: 'EUR';
+  periods: { current: IStatementPeriod; prior: IStatementPeriod | null };
+  keyFigures: { current: IKeyFigures; prior: IKeyFigures | null };
+  balanceSheet: IBalanceSheet;
+  /** null where the document publishes none, as small companies may (§ 326 Abs. 1 HGB) */
+  incomeStatement: IIncomeStatement | null;
+  /** what was left out and why, in English */
+  notes: string[];
+}
+
+/** The period of an ESEF fact: an instant (`start` null) or a duration. */
+export interface IEsefPeriod {
+  start: TIsoDate | null;
+  end: TIsoDate;
+}
+
+/** One numeric fact of an ESEF report (`ix:nonFraction`). */
+export interface IEsefFact {
+  /** the concept with the prefix the report declares for it: `ifrs-full:Revenue` */
+  concept: string;
+  /** the namespace of the concept */
+  namespace: string;
+  contextId: string;
+  period: IEsefPeriod;
+  /** the explicit dimension members of the context, dimension to member, both as written */
+  dimensions: Record<string, string>;
+  /** the unit: an ISO 4217 code such as `EUR`, `shares`, `pure`, or a ratio such as `EUR/shares` */
+  unit: string | null;
+  /** the value as a decimal string, scale and sign applied; null for a nil fact */
+  value: string | null;
+  /** the `decimals` attribute as written: `0`, `-3`, `INF` */
+  decimals: string | null;
+  /** the fact's `id`, where it has one */
+  id: string | null;
+}
+
+/**
+ * A listed issuer's annual financial report in the European Single Electronic Format: the numeric
+ * facts its inline XBRL tags, and the key figures of its consolidated statements (IFRS).
+ */
+export interface IEsefStatement {
+  standard: 'esef';
+  /** the LEI the report's contexts identify the issuer with */
+  lei: string | null;
+  /** the path of the report in its package; null for a report read on its own */
+  reportPath: string | null;
+  /** the language the report declares */
+  language: string | null;
+  /** the currency of the key figures, ISO 4217 */
+  currency: string | null;
+  periods: { current: IStatementPeriod; prior: IStatementPeriod | null };
+  keyFigures: { current: IKeyFigures; prior: IKeyFigures | null };
+  facts: IEsefFact[];
+  /** what could not be read and why, in English */
+  notes: string[];
+}
+
+export type TParsedStatement = IHgbStatement | IEsefStatement;
+
+/** Why a document was kept without a parsed statement. */
+export interface IStatementParseFailure {
+  code: 'invalid_package' | 'unreadable' | 'no_text' | 'unparseable' | 'too_large';
+  message: string;
+}
+
+/**
+ * A statement as a caller keeps it: the filing, the document byte for byte, and what could be read
+ * from it. `parsed` is null rather than guessed: a document this entry point cannot read with
+ * certainty keeps its raw bytes, and `parseFailure` says why.
+ */
+export interface IFinancialStatement {
+  filing: IFiling;
+  raw: IRawStatementDocument;
+  parsed: TParsedStatement | null;
+  parseFailure: IStatementParseFailure | null;
+  /** when the document was obtained, milliseconds since the epoch */
+  obtainedAt: number;
+  /**
+   * the origin note the publisher marks the document with, for a copy to keep it: for the
+   * Unternehmensregister "Auszug aus dem Unternehmensregister" and the day the document was posted
+   * or deposited (Nutzungsbedingungen § 3 Abs. 5); null for other publishers
+   */
+  notice: string | null;
+}
+
+export interface IReadFinancialStatementOptions {
+  filing: IFilingInput;
+  bytes: Uint8Array;
+  /** the media type of `bytes`; parameters such as `; charset=utf-8` are ignored */
+  mediaType: string;
+  /** clock for `obtainedAt`; default `Date.now` */
+  now?: () => number;
+}

package/ts/financialstatements/layout.html.ts

@@ -0,0 +1,130 @@
+import * as plugins from './plugins.js';
+import { descendants, elementChildren, getAttribute, hasClass, isElement, parseHtml, textOf, type THtmlElement } from './html.js';
+import { normalizeSpaces } from './amounts.js';
+import type { ILayoutCell, TLayoutBlock } from './layout.js';
+
+type THtmlChildNode = plugins.parse5.DefaultTreeAdapterTypes.ChildNode;
+
+const headingTags = new Set(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']);
+const skippedTags = new Set(['head', 'script', 'style', 'template', 'noscript', 'svg', 'math', 'img', 'button', 'select', 'input']);
+
+/** elements that end a line of text before and after them */
+const blockTags = new Set([
+  'address', 'article', 'aside', 'blockquote', 'body', 'caption', 'center', 'dd', 'details', 'dialog', 'div',
+  'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form', 'header', 'hgroup', 'hr', 'li', 'main',
+  'nav', 'ol', 'p', 'pre', 'section', 'summary', 'tbody', 'td', 'tfoot', 'th', 'thead', 'tr', 'ul',
+]);
+
+/** colspan and rowspan as a statement may write them, kept to 1 to 50 */
+const spanOf = (valueArg: string | undefined): number => {
+  const value = Number.parseInt(valueArg ?? '1', 10);
+  return Number.isFinite(value) ? Math.min(Math.max(value, 1), 50) : 1;
+};
+
+/** the rows of a table without nested tables, every cell placed in the columns it covers */
+const tableRows = (tableArg: THtmlElement): ILayoutCell[][] => {
+  const rowElements = descendants(tableArg).filter((elementArg) => elementArg.tagName === 'tr');
+  const occupied: Array<Set<number>> = rowElements.map(() => new Set<number>());
+  const rows: ILayoutCell[][] = [];
+  rowElements.forEach((rowElement, rowIndex) => {
+    const cells: ILayoutCell[] = [];
+    let column = 0;
+    for (const cellElement of elementChildren(rowElement)) {
+      if (cellElement.tagName !== 'td' && cellElement.tagName !== 'th') {
+        continue;
+      }
+      while (occupied[rowIndex].has(column)) {
+        column++;
+      }
+      const colspan = spanOf(getAttribute(cellElement, 'colspan'));
+      const rowspan = spanOf(getAttribute(cellElement, 'rowspan'));
+      for (let spannedRow = rowIndex; spannedRow < Math.min(rowIndex + rowspan, rowElements.length); spannedRow++) {
+        for (let spannedColumn = column; spannedColumn < column + colspan; spannedColumn++) {
+          occupied[spannedRow].add(spannedColumn);
+        }
+      }
+      const text = normalizeSpaces(textOf(cellElement));
+      if (text !== '') {
+        cells.push({ text, first: column, last: column + colspan - 1 });
+      }
+      column += colspan;
+    }
+    if (cells.length > 0) {
+      rows.push(cells);
+    }
+  });
+  return rows;
+};
+
+/**
+ * The blocks of an HTML statement in document order: headings, lines of text and tables. Reads the
+ * publication itself (`div.publication_container`, as the Unternehmensregister and the Bundesanzeiger
+ * show it) where the page has one, else the whole body. A table that only lays out other tables is
+ * read through to them.
+ */
+export const htmlLayout = (htmlArg: string): TLayoutBlock[] => {
+  const document = parseHtml(htmlArg);
+  const elements = descendants(document);
+  const root =
+    elements.find((elementArg) => elementArg.tagName === 'div' && hasClass(elementArg, 'publication_container')) ??
+    elements.find((elementArg) => elementArg.tagName === 'body');
+  if (!root) {
+    return [];
+  }
+  const blocks: TLayoutBlock[] = [];
+  let run: string[] = [];
+  const flush = () => {
+    const text = normalizeSpaces(run.join(' '));
+    run = [];
+    if (text !== '') {
+      blocks.push({ kind: 'text', text, heading: false });
+    }
+  };
+  const stack: Array<{ children: THtmlChildNode[]; index: number; block: boolean }> = [
+    { children: root.childNodes, index: 0, block: true },
+  ];
+  while (stack.length > 0) {
+    const frame = stack[stack.length - 1];
+    if (frame.index >= frame.children.length) {
+      stack.pop();
+      if (frame.block) {
+        flush();
+      }
+      continue;
+    }
+    const child = frame.children[frame.index++];
+    if (!isElement(child)) {
+      if ('value' in child && child.nodeName === '#text') {
+        run.push(child.value);
+      }
+      continue;
+    }
+    const tag = child.tagName;
+    if (skippedTags.has(tag)) {
+      continue;
+    }
+    if (tag === 'br') {
+      flush();
+    } else if (headingTags.has(tag)) {
+      flush();
+      const text = normalizeSpaces(textOf(child));
+      if (text !== '') {
+        blocks.push({ kind: 'text', text, heading: true });
+      }
+    } else if (tag === 'table' && !descendants(child).some((elementArg) => elementArg.tagName === 'table')) {
+      flush();
+      const rows = tableRows(child);
+      if (rows.length > 0) {
+        blocks.push({ kind: 'table', rows });
+      }
+    } else {
+      const block = tag === 'table' || blockTags.has(tag);
+      if (block) {
+        flush();
+      }
+      stack.push({ children: child.childNodes, index: 0, block });
+    }
+  }
+  flush();
+  return blocks;
+};