@fin.cx/opendata 4.7.0 → 4.8.0

package/ts/financialstatements/layout.pdf.ts

@@ -0,0 +1,268 @@
+import * as plugins from './plugins.js';
+import { FinancialStatementError } from './classes.financialstatementerror.js';
+import { maxPdfPages } from './constants.js';
+import { normalizeSpaces, readsAsAmount } from './amounts.js';
+import { isMemoLabel, splitOutlineCode } from './outline.js';
+import type { ILayoutCell, TLayoutBlock } from './layout.js';
+
+/** a run of text on a page, in page coordinates with y growing downwards */
+interface IPiece {
+  text: string;
+  x0: number;
+  x1: number;
+  y: number;
+  height: number;
+}
+
+/** pieces of one line close enough to read as one: a label, an amount, a column heading */
+interface ISegment {
+  text: string;
+  x0: number;
+  x1: number;
+  height: number;
+  amount: boolean;
+}
+
+/** the amounts of a page that share a right edge: one column */
+interface ICluster {
+  x0: number;
+  x1: number;
+  members: number;
+}
+
+/** a row of a page as it is built, with where its label starts */
+interface IPageRow {
+  cells: ILayoutCell[];
+  labelX0: number | null;
+}
+
+/** the largest gap between pieces of one segment, and the gap that reads as a space, in font heights */
+const segmentGap = 0.6;
+const spaceGap = 0.15;
+/** amounts whose right edges lie further apart than this are in different columns, in points */
+const clusterGap = 12;
+
+/** a label that a wrapped line continues: it ends on a hyphen, a comma or a word that needs another */
+const continuedLabelPattern =
+  /(?:[-,]|\b(?:und|oder|sowie|aus|für|an|gegenüber|mit|von|zu|bei|auf|in|im|am|der|die|das|des|dem|den|ein|eine|einem|einen|einer|eines|ähnliche|ähnlichen|sonstige|sonstigen|denen))$/i;
+
+const isContinuation = (previousLabelArg: string, labelArg: string): boolean =>
+  !isMemoLabel(labelArg) &&
+  splitOutlineCode(labelArg).code === null &&
+  (continuedLabelPattern.test(previousLabelArg) || /^[a-zäöüß]/.test(labelArg));
+
+const linesOf = (piecesArg: IPiece[]): ISegment[][] => {
+  const sorted = [...piecesArg].sort((leftArg, rightArg) => leftArg.y - rightArg.y || leftArg.x0 - rightArg.x0);
+  const lines: IPiece[][] = [];
+  let lineY = Number.NEGATIVE_INFINITY;
+  for (const piece of sorted) {
+    const current = lines[lines.length - 1];
+    if (current && Math.abs(piece.y - lineY) <= Math.max(2, 0.3 * piece.height)) {
+      current.push(piece);
+    } else {
+      lines.push([piece]);
+      lineY = piece.y;
+    }
+  }
+  return lines.map((lineArg) => {
+    const segments: ISegment[] = [];
+    for (const piece of lineArg.sort((leftArg, rightArg) => leftArg.x0 - rightArg.x0)) {
+      const segment = segments[segments.length - 1];
+      const gap = segment ? piece.x0 - segment.x1 : Number.POSITIVE_INFINITY;
+      const height = Math.max(piece.height, segment?.height ?? 0);
+      if (segment && gap <= segmentGap * height) {
+        segment.text += gap > spaceGap * height ? ` ${piece.text}` : piece.text;
+        segment.x1 = Math.max(segment.x1, piece.x1);
+        segment.height = height;
+      } else {
+        segments.push({ text: piece.text, x0: piece.x0, x1: piece.x1, height: piece.height, amount: false });
+      }
+    }
+    for (const segment of segments) {
+      segment.text = normalizeSpaces(segment.text);
+      segment.amount = readsAsAmount(segment.text);
+    }
+    return segments.filter((segmentArg) => segmentArg.text !== '');
+  });
+};
+
+/**
+ * The columns of a page: amounts sharing a right edge, at least two of them and one a number, so
+ * that dashes set as bullets make no column.
+ */
+const clustersOf = (linesArg: ISegment[][]): ICluster[] => {
+  const amounts = linesArg.flat().filter((segmentArg) => segmentArg.amount).sort((leftArg, rightArg) => leftArg.x1 - rightArg.x1);
+  const clusters: Array<ICluster & { lastX1: number; numbers: number }> = [];
+  for (const amount of amounts) {
+    const cluster = clusters[clusters.length - 1];
+    const number = /\d/.test(amount.text) ? 1 : 0;
+    if (cluster && amount.x1 - cluster.lastX1 <= clusterGap) {
+      cluster.x0 = Math.min(cluster.x0, amount.x0);
+      cluster.x1 = Math.max(cluster.x1, amount.x1);
+      cluster.lastX1 = amount.x1;
+      cluster.members++;
+      cluster.numbers += number;
+    } else {
+      clusters.push({ x0: amount.x0, x1: amount.x1, lastX1: amount.x1, members: 1, numbers: number });
+    }
+  }
+  return clusters
+    .filter((clusterArg) => clusterArg.members >= 2 && clusterArg.numbers > 0)
+    .map(({ x0, x1, members }) => ({ x0, x1, members }));
+};
+
+const overlaps = (segmentArg: ISegment, clusterArg: ICluster): boolean =>
+  segmentArg.x1 > clusterArg.x0 + 1 && segmentArg.x0 < clusterArg.x1 - 1;
+
+/** one page as a table: column 0 the labels, columns 1 to n its amount columns */
+const pageTable = (linesArg: ISegment[][]): ILayoutCell[][] => {
+  const clusters = clustersOf(linesArg);
+  const labelEnd = clusters.length > 0 ? clusters[0].x0 - 1 : Number.POSITIVE_INFINITY;
+  const clusterOfAmount = (segmentArg: ISegment): number =>
+    clusters.findIndex((clusterArg) => segmentArg.x1 >= clusterArg.x0 && segmentArg.x1 <= clusterArg.x1 + 1);
+  const rows: IPageRow[] = [];
+  for (const line of linesArg) {
+    const columns = new Map<number, { text: string[]; first: number; last: number }>();
+    const place = (columnArg: number, lastArg: number, textArg: string) => {
+      const cell = columns.get(columnArg) ?? { text: [], first: columnArg, last: lastArg };
+      cell.text.push(textArg);
+      cell.last = Math.max(cell.last, lastArg);
+      columns.set(columnArg, cell);
+    };
+    const amountColumns = line.map((segmentArg) => (segmentArg.amount ? clusterOfAmount(segmentArg) : -1));
+    const hasAmounts = amountColumns.some((columnArg) => columnArg >= 0);
+    let labelX0: number | null = null;
+    line.forEach((segment, index) => {
+      if (amountColumns[index] >= 0) {
+        place(amountColumns[index] + 1, amountColumns[index] + 1, segment.text);
+        return;
+      }
+      const covered = clusters.flatMap((clusterArg, clusterIndex) => (overlaps(segment, clusterArg) ? [clusterIndex + 1] : []));
+      if (segment.x0 < labelEnd) {
+        labelX0 ??= segment.x0;
+        place(0, hasAmounts || covered.length === 0 ? 0 : Math.max(...covered), segment.text);
+      } else if (covered.length > 0) {
+        place(covered[0], covered[covered.length - 1], segment.text);
+      } else {
+        const center = (segment.x0 + segment.x1) / 2;
+        let nearest = 0;
+        clusters.forEach((clusterArg, clusterIndex) => {
+          const distance = Math.abs((clusterArg.x0 + clusterArg.x1) / 2 - center);
+          const best = Math.abs((clusters[nearest].x0 + clusters[nearest].x1) / 2 - center);
+          if (distance < best) {
+            nearest = clusterIndex;
+          }
+        });
+        place(nearest + 1, nearest + 1, segment.text);
+      }
+    });
+    const cells = [...columns.values()]
+      .sort((leftArg, rightArg) => leftArg.first - rightArg.first)
+      .map((cellArg) => ({ text: normalizeSpaces(cellArg.text.join(' ')), first: cellArg.first, last: cellArg.last }));
+    if (cells.length > 0) {
+      rows.push({ cells, labelX0 });
+    }
+  }
+  return joinWrappedLabels(rows);
+};
+
+/**
+ * Joins labels a page wraps over two lines: a coded line without amounts whose next line carries on
+ * its label and holds the amounts, and a line whose label carries on on the next line, indented and
+ * without amounts.
+ */
+const joinWrappedLabels = (rowsArg: IPageRow[]): ILayoutCell[][] => {
+  const result: IPageRow[] = [];
+  const labelOf = (rowArg: IPageRow): ILayoutCell | undefined =>
+    rowArg.cells[0]?.first === 0 && rowArg.cells[0].last === 0 ? rowArg.cells[0] : undefined;
+  for (const row of rowsArg) {
+    const previous = result[result.length - 1];
+    const label = labelOf(row);
+    const previousLabel = previous ? labelOf(previous) : undefined;
+    if (previous && label && previousLabel && previous.labelX0 !== null && row.labelX0 !== null) {
+      const amounts = row.cells.length > 1;
+      const previousAmounts = previous.cells.length > 1;
+      const coded = splitOutlineCode(previousLabel.text).code !== null;
+      if (
+        isContinuation(previousLabel.text, label.text) &&
+        ((amounts && !previousAmounts && coded) || (!amounts && row.labelX0 > previous.labelX0 + 1))
+      ) {
+        previousLabel.text = `${previousLabel.text} ${label.text}`;
+        previous.cells.push(...row.cells.slice(1));
+        continue;
+      }
+    }
+    result.push(row);
+  }
+  return result.map((rowArg) => rowArg.cells);
+};
+
+/**
+ * The blocks of a PDF statement: each page one table. Reads the text layer only; a PDF without one,
+ * such as a scan, is `no_text`.
+ */
+export const pdfLayout = async (bytesArg: Uint8Array): Promise<TLayoutBlock[]> => {
+  const pdfjs = await plugins.loadPdfjs();
+  const packageDirectory = plugins.path.dirname(plugins.createRequire(import.meta.url).resolve('pdfjs-dist/package.json'));
+  const loadingTask = pdfjs.getDocument({
+    data: new Uint8Array(bytesArg),
+    cMapUrl: plugins.path.join(packageDirectory, 'cmaps') + plugins.path.sep,
+    cMapPacked: true,
+    standardFontDataUrl: plugins.path.join(packageDirectory, 'standard_fonts') + plugins.path.sep,
+    useSystemFonts: false,
+    disableFontFace: true,
+    enableXfa: false,
+    verbosity: pdfjs.VerbosityLevel.ERRORS,
+  });
+  try {
+    let pdfDocument: Awaited<typeof loadingTask.promise>;
+    try {
+      pdfDocument = await loadingTask.promise;
+    } catch (error) {
+      const reason = error instanceof pdfjs.PasswordException ? 'is protected by a password' : 'does not open as a PDF';
+      throw new FinancialStatementError('unreadable', `The document ${reason}.`, { cause: error });
+    }
+    if (pdfDocument.numPages > maxPdfPages) {
+      throw new FinancialStatementError('too_large', `The PDF has ${pdfDocument.numPages} pages, more than the ${maxPdfPages} read here.`);
+    }
+    const blocks: TLayoutBlock[] = [];
+    let textFound = false;
+    for (let pageNumber = 1; pageNumber <= pdfDocument.numPages; pageNumber++) {
+      const page = await pdfDocument.getPage(pageNumber);
+      const viewport = page.getViewport({ scale: 1 });
+      const content = await page.getTextContent();
+      const pieces: IPiece[] = [];
+      for (const item of content.items) {
+        if (!('str' in item) || item.str.trim() === '') {
+          continue;
+        }
+        const [a, b, c, d, x, y] = pdfjs.Util.transform(viewport.transform, item.transform) as number[];
+        if (Math.abs(b) > 0.01 * Math.abs(a) || Math.abs(c) > 0.01 * Math.abs(d)) {
+          continue;
+        }
+        const height = item.height > 0 ? item.height : Math.abs(d);
+        pieces.push({ text: item.str, x0: x, x1: x + item.width, y, height });
+      }
+      page.cleanup();
+      if (pieces.length === 0) {
+        continue;
+      }
+      textFound = true;
+      const rows = pageTable(linesOf(pieces));
+      if (rows.length > 0) {
+        blocks.push({ kind: 'table', rows });
+      }
+    }
+    if (!textFound) {
+      throw new FinancialStatementError('no_text', 'The PDF has no text layer; it may be a scan.');
+    }
+    return blocks;
+  } catch (error) {
+    if (error instanceof FinancialStatementError) {
+      throw error;
+    }
+    throw new FinancialStatementError('unreadable', 'The PDF could not be read.', { cause: error });
+  } finally {
+    await loadingTask.destroy();
+  }
+};

package/ts/financialstatements/layout.ts

@@ -0,0 +1,16 @@
+/**
+ * A statement as this entry point reads it, whatever it came as: text in printed order, and tables
+ * of cells. An HTML table keeps its own columns; a PDF page becomes one table whose column 0 holds
+ * the labels and whose other columns are the columns its amounts are aligned in.
+ */
+export interface ILayoutCell {
+  text: string;
+  /** the first column the cell covers */
+  first: number;
+  /** the last column the cell covers */
+  last: number;
+}
+
+export type TLayoutBlock =
+  | { kind: 'text'; text: string; heading: boolean }
+  | { kind: 'table'; rows: ILayoutCell[][] };

package/ts/financialstatements/mediatypes.ts

@@ -0,0 +1,14 @@
+import type { TStatementMediaType } from './interfaces.js';
+
+/** the media types read here, by the names callers and servers give them */
+const mediaTypes: Record<string, TStatementMediaType> = {
+  'text/html': 'text/html',
+  'application/pdf': 'application/pdf',
+  'application/xhtml+xml': 'application/xhtml+xml',
+  'application/zip': 'application/zip',
+  'application/x-zip-compressed': 'application/zip',
+};
+
+/** a media type as read here, its parameters ignored; null for one that is not read */
+export const statementMediaTypeOf = (mediaTypeArg: string): TStatementMediaType | null =>
+  mediaTypes[mediaTypeArg.split(';')[0].trim().toLowerCase()] ?? null;

package/ts/financialstatements/outline.ts

@@ -0,0 +1,44 @@
+/**
+ * The outline codes of §§ 266 and 275 HGB as statements print them, from the top level down:
+ * capital letters (`A.`), roman numerals (`I.`), arabic numerals (`1.`), small letters (`a)`) and
+ * doubled small letters (`aa)`).
+ */
+export interface IOutlineCode {
+  /** the code as printed: `A.`, `a)` */
+  printed: string;
+  /** the code without its punctuation: `A`, `a` */
+  key: string;
+  /** 0 for capital letters to 4 for doubled small letters */
+  rank: number;
+}
+
+/** capital letters stop before I, V and X, which are roman numerals; a code is followed by a space */
+const codePatterns: Array<{ rank: number; pattern: RegExp }> = [
+  { rank: 4, pattern: /^(([a-z])\2)\)(?:\s+|$)/ },
+  { rank: 0, pattern: /^([A-H])[.)](?:\s+|$)/ },
+  { rank: 1, pattern: /^((?=[IVX])X{0,3}(?:IX|IV|V?I{0,3}))\.(?:\s+|$)/ },
+  { rank: 2, pattern: /^(\d{1,2})\.(?:\s+|$)/ },
+  { rank: 3, pattern: /^([a-z])[.)](?:\s+|$)/ },
+];
+
+/** splits the outline code off a printed label */
+export const splitOutlineCode = (labelArg: string): { code: IOutlineCode | null; label: string } => {
+  for (const { rank, pattern } of codePatterns) {
+    const match = pattern.exec(labelArg);
+    if (match) {
+      return {
+        code: { printed: match[0].trim(), key: match[1], rank },
+        label: labelArg.slice(match[0].length).trim(),
+      };
+    }
+  }
+  return { code: null, label: labelArg };
+};
+
+/** a "davon" line: part of the line above it, never added */
+export const isMemoLabel = (labelArg: string): boolean =>
+  /^[-–—•·(]?\s*(?:davon|darunter)\b/i.test(labelArg);
+
+/** a line that sums the lines above it: no label at all, or one such as Summe or Bilanzsumme */
+export const isTotalLabel = (labelArg: string): boolean =>
+  labelArg === '' || /^(?:Summe|Bilanzsumme|Gesamtsumme|Gesamt|Zwischensumme|Insgesamt)\b/i.test(labelArg);