@fin.cx/opendata 4.6.0 → 4.8.0

package/ts/financialstatements/parse.statement.ts

@@ -0,0 +1,26 @@
+import { FinancialStatementError } from './classes.financialstatementerror.js';
+import { maxStatementBytes } from './constants.js';
+import { decodeHtml } from './html.js';
+import { htmlLayout } from './layout.html.js';
+import { pdfLayout } from './layout.pdf.js';
+import { statementMediaTypeOf } from './mediatypes.js';
+import { readHgbStatement } from './parse.hgb.js';
+import type { IHgbStatement } from './interfaces.js';
+
+/**
+ * Reads a statement under the HGB from the HTML or the PDF a register shows: the balance sheet
+ * (§ 266 HGB) and, where one is published, the income statement (§ 275 HGB). Everything it returns
+ * was printed and adds up; where the current year's balance sheet cannot be read with certainty it
+ * throws `unparseable` rather than guess. A PDF is read from its text layer only.
+ */
+export const parseStatementDocument = async (bytesArg: Uint8Array, mediaTypeArg: string): Promise<IHgbStatement> => {
+  const mediaType = statementMediaTypeOf(mediaTypeArg);
+  if (mediaType !== 'text/html' && mediaType !== 'application/xhtml+xml' && mediaType !== 'application/pdf') {
+    throw new FinancialStatementError('unsupported_media_type', `A statement in "${mediaTypeArg}" is not read here; HTML and PDF are.`);
+  }
+  if (bytesArg.byteLength > maxStatementBytes) {
+    throw new FinancialStatementError('too_large', `The document has ${bytesArg.byteLength} bytes, more than the ${maxStatementBytes} read here.`);
+  }
+  const blocks = mediaType === 'application/pdf' ? await pdfLayout(bytesArg) : htmlLayout(decodeHtml(bytesArg));
+  return readHgbStatement(blocks);
+};

package/ts/financialstatements/plugins.ts

@@ -0,0 +1,29 @@
+// This entry point stands alone: it imports nothing from the rest of the package and needs no storage.
+
+// node native scope
+import * as path from 'node:path';
+import { createRequire } from 'node:module';
+
+export {
+  path,
+  createRequire,
+}
+
+// @push.rocks scope
+import * as smartarchiveZip from '@push.rocks/smartarchive/web/zip';
+import * as smartxmlXpath from '@push.rocks/smartxml/xpath';
+
+export {
+  smartarchiveZip,
+  smartxmlXpath,
+}
+
+// third party scope
+import * as parse5 from 'parse5';
+
+export {
+  parse5,
+}
+
+/** PDF.js for Node.js (its legacy build), loaded when the first PDF is read */
+export const loadPdfjs = () => import('pdfjs-dist/legacy/build/pdf.mjs');

package/ts/financialstatements/read.ts

@@ -0,0 +1,135 @@
+import { FinancialStatementError } from './classes.financialstatementerror.js';
+import { maxStatementBytes, unternehmensregisterNotice } from './constants.js';
+import { statementMediaTypeOf } from './mediatypes.js';
+import { parseEsefPackage, parseEsefReport } from './parse.esef.js';
+import { parseStatementDocument } from './parse.statement.js';
+import type {
+  IFiling,
+  IFinancialStatement,
+  IReadFinancialStatementOptions,
+  IStatementParseFailure,
+  TFilingKind,
+  TFilingSource,
+  TParsedStatement,
+  TSizeClass,
+} from './interfaces.js';
+
+const filingSources: TFilingSource[] = ['unternehmensregister', 'bundesanzeiger', 'issuer', 'unknown'];
+const filingKinds: TFilingKind[] = ['annual_accounts', 'consolidated_accounts', 'deposited_balance_sheet', 'annual_financial_report'];
+const sizeClasses: TSizeClass[] = ['micro', 'small', 'medium', 'large'];
+const parseFailureCodes = new Set<string>(['invalid_package', 'unreadable', 'no_text', 'unparseable', 'too_large']);
+
+/** the inline XBRL namespace, which marks an HTML document as an ESEF report */
+const inlineXbrlNamespace = 'http://www.xbrl.org/2013/inlineXBRL';
+
+const invalid = (messageArg: string): never => {
+  throw new FinancialStatementError('invalid_input', messageArg);
+};
+
+const isIsoDate = (valueArg: string): boolean => {
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(valueArg)) {
+    return false;
+  }
+  const date = new Date(`${valueArg}T00:00:00Z`);
+  return !Number.isNaN(date.getTime()) && date.toISOString().slice(0, 10) === valueArg;
+};
+
+const filingOf = (inputArg: IReadFinancialStatementOptions['filing']): IFiling => {
+  if (!filingSources.includes(inputArg?.source)) {
+    invalid(`The filing's source "${inputArg?.source}" is not one of ${filingSources.join(', ')}.`);
+  }
+  if (!filingKinds.includes(inputArg.kind)) {
+    invalid(`The filing's kind "${inputArg.kind}" is not one of ${filingKinds.join(', ')}.`);
+  }
+  if (inputArg.sizeClass != null && !sizeClasses.includes(inputArg.sizeClass)) {
+    invalid(`The filing's size class "${inputArg.sizeClass}" is not one of ${sizeClasses.join(', ')}.`);
+  }
+  for (const field of ['fiscalYearStart', 'fiscalYearEnd', 'publishedOn'] as const) {
+    const value = inputArg[field];
+    if (value != null && (typeof value !== 'string' || !isIsoDate(value))) {
+      invalid(`The filing's ${field} "${value}" is not an ISO date (YYYY-MM-DD).`);
+    }
+  }
+  if (inputArg.source === 'unternehmensregister' && inputArg.publishedOn == null) {
+    invalid('A filing from the Unternehmensregister needs publishedOn, the day it was posted or deposited, for its origin note.');
+  }
+  return {
+    source: inputArg.source,
+    kind: inputArg.kind,
+    companyName: inputArg.companyName ?? null,
+    registerReference: inputArg.registerReference ?? null,
+    lei: inputArg.lei ?? null,
+    fiscalYearStart: inputArg.fiscalYearStart ?? null,
+    fiscalYearEnd: inputArg.fiscalYearEnd ?? null,
+    publishedOn: inputArg.publishedOn ?? null,
+    title: inputArg.title ?? null,
+    sizeClass: inputArg.sizeClass ?? null,
+    documentId: inputArg.documentId ?? null,
+    sourceUrl: inputArg.sourceUrl ?? null,
+  };
+};
+
+const sha256Of = async (bytesArg: Uint8Array<ArrayBuffer>): Promise<string> => {
+  const digest = new Uint8Array(await globalThis.crypto.subtle.digest('SHA-256', bytesArg));
+  return [...digest].map((byteArg) => byteArg.toString(16).padStart(2, '0')).join('');
+};
+
+/**
+ * Keeps a published statement as a caller stores it: the filing as the caller states it, the
+ * document byte for byte with its SHA-256, and what could be read from it: an ESEF report by
+ * `parseEsefPackage` or `parseEsefReport`, any other HTML or PDF by `parseStatementDocument`. A
+ * document that cannot be read with certainty is kept with `parsed` null and `parseFailure` saying
+ * why. Throws `invalid_input`, `unsupported_media_type` and, for a document larger than
+ * `maxStatementBytes`, `too_large`, as nothing would be kept.
+ */
+export const readFinancialStatement = async (optionsArg: IReadFinancialStatementOptions): Promise<IFinancialStatement> => {
+  if (!(optionsArg?.bytes instanceof Uint8Array)) {
+    invalid('bytes must be a Uint8Array.');
+  }
+  const filing = filingOf(optionsArg.filing);
+  if (optionsArg.bytes.byteLength > maxStatementBytes) {
+    throw new FinancialStatementError(
+      'too_large',
+      `The document has ${optionsArg.bytes.byteLength} bytes, more than the ${maxStatementBytes} read here.`
+    );
+  }
+  const mediaType = statementMediaTypeOf(String(optionsArg.mediaType));
+  if (!mediaType) {
+    throw new FinancialStatementError(
+      'unsupported_media_type',
+      `A statement in "${optionsArg.mediaType}" is not read here; HTML, PDF, XHTML and ZIP packages are.`
+    );
+  }
+  const bytes = new Uint8Array(optionsArg.bytes);
+  const sha256 = await sha256Of(bytes);
+  let parsed: TParsedStatement | null = null;
+  let parseFailure: IStatementParseFailure | null = null;
+  try {
+    if (mediaType === 'application/zip') {
+      parsed = await parseEsefPackage(bytes);
+    } else if (
+      mediaType === 'application/xhtml+xml' ||
+      (mediaType === 'text/html' && new TextDecoder('windows-1252').decode(bytes.subarray(0, 65536)).includes(inlineXbrlNamespace))
+    ) {
+      parsed = parseEsefReport(bytes);
+    } else {
+      parsed = await parseStatementDocument(bytes, mediaType);
+    }
+  } catch (error) {
+    if (!(error instanceof FinancialStatementError) || !parseFailureCodes.has(error.code)) {
+      throw error;
+    }
+    parseFailure = { code: error.code as IStatementParseFailure['code'], message: error.message };
+  }
+  return {
+    filing,
+    raw: { bytes, mediaType, sha256 },
+    parsed,
+    parseFailure,
+    obtainedAt: (optionsArg.now ?? Date.now)(),
+    notice:
+      filing.source === 'unternehmensregister'
+        ? unternehmensregisterNotice(filing.publishedOn as string, filing.kind === 'deposited_balance_sheet')
+        : null,
+  };
+};

package/ts/handelsregister/classes.handelsregisterclient.ts

@@ -120,14 +120,18 @@ const decodeUtf8 = (bytesArg: Uint8Array): string => new TextDecoder('utf-8').de
  * plain HTTP and without storage.
  *
  * Every call opens its own visit to the portal and forgets it afterwards: nothing is cached or
- * shared between calls. The client never repeats a request by itself, never logs, and asks its
- * gate before every request the portal's terms count:
+ * shared between calls. The client never logs, and asks its gate before every request the portal's
+ * terms count:
  * - `search()` makes one counted search;
  * - `getEntry()` makes one counted search and, when the portal offers structured content for the
  *   entry, one counted retrieval.
  *
  * Each search also loads the search form and the result list, two pages the terms do not count.
  *
+ * The client repeats a request only when the portal closed the connection without answering it,
+ * and only once: the portal's front end does that to have the request sent again. A counted request
+ * sent once more asks the gate again and counts again.
+ *
  * The portal's terms allow single lookups for information only, at most 60 searches or
  * retrievals per hour, and no systematic retrieval to build or update a copy of the registers.
  */
@@ -288,6 +292,12 @@ export class HandelsregisterClient {
     }
   }
 
+  /** asks the gate for a counted request the portal dropped before it is sent once more, and counts it */
+  private async countAgain(kindArg: TRegisterCountedRequestKind, stepArg: THandelsregisterStep, progressArg: ICallProgress): Promise<void> {
+    await this.acquireSlot(kindArg, stepArg, progressArg);
+    progressArg.countedRequests += 1;
+  }
+
   private async runSearch(planArg: ISearchPlan, progressArg: ICallProgress, sessionArg: PortalSession): Promise<ISearchOutcome> {
     await this.acquireSlot('search', 'search', progressArg);
     try {
@@ -333,7 +343,9 @@ export class HandelsregisterClient {
 
       progressArg.step = 'search';
       progressArg.countedRequests += 1;
-      const resultPage = await sessionArg.post(sessionArg.resolve(form.action, formPage.url), fields);
+      const resultPage = await sessionArg.post(sessionArg.resolve(form.action, formPage.url), fields, () =>
+        this.countAgain('search', 'search', progressArg)
+      );
 
       progressArg.step = 'result_list';
       const resultText = decodeUtf8(resultPage.body);
@@ -406,7 +418,9 @@ export class HandelsregisterClient {
         forms.setField(fields, name, value);
       }
       progressArg.countedRequests += 1;
-      const answer = await sessionArg.post(sessionArg.resolve(resultForm.action, outcomeArg.pageUrl), fields);
+      const answer = await sessionArg.post(sessionArg.resolve(resultForm.action, outcomeArg.pageUrl), fields, () =>
+        this.countAgain('retrieval', 'structured_content', progressArg)
+      );
       if (/html/i.test(answer.contentType)) {
         this.throwIfBlocked(decodeUtf8(answer.body));
         throw new HandelsregisterError('unexpected_response', 'The portal answered with a page instead of the structured content.');

package/ts/handelsregister/classes.handelsregistererror.ts

@@ -7,6 +7,8 @@ import type { IRegisterMatch } from './interfaces.js';
  * - `aborted`: the caller's signal aborted the call;
  * - `timeout`: a request to the portal took longer than the client's limit;
  * - `unreachable`: the portal could not be reached (name resolution, connection, TLS);
+ * - `dropped`: the portal closed the connection without answering, also when the request was sent
+ *   once more. Its front end does so when it lost its back end's answer;
  * - `refused`: the portal answered with an HTTP error, for example 403 when it blocks an address,
  *   429 or 503 when it is overloaded (`status` carries the code), or with its notice that it refuses
  *   retrievals from this address (`portalMessage` carries its words);
@@ -23,6 +25,7 @@ export type THandelsregisterErrorCode =
   | 'aborted'
   | 'timeout'
   | 'unreachable'
+  | 'dropped'
   | 'refused'
   | 'too_large'
   | 'unexpected_response'

package/ts/handelsregister/classes.portalsession.ts

@@ -25,6 +25,26 @@ export interface IPortalResponse {
 const redirectStatuses = new Set([301, 302, 303]);
 const maxRedirects = 3;
 
+/**
+ * error codes of a connection the other side closed or reset before it answered: undici's
+ * `UND_ERR_SOCKET` (the `fetch` of Node.js: "other side closed") and the system's `ECONNRESET` and
+ * `EPIPE`
+ */
+const closedConnectionCodes = new Set(['UND_ERR_SOCKET', 'ECONNRESET', 'EPIPE']);
+
+/** whether a failed `fetch` failed because the connection was closed before any answer arrived */
+const closedWithoutAnswer = (errorArg: unknown): boolean => {
+  let current: unknown = errorArg;
+  for (let depth = 0; depth < 4 && current instanceof Error; depth++) {
+    const code = (current as Error & { code?: unknown }).code;
+    if (typeof code === 'string' && closedConnectionCodes.has(code)) {
+      return true;
+    }
+    current = current.cause;
+  }
+  return false;
+};
+
 /**
  * One visit to the register portal: a cookie jar, the fixed language and a limit for every
  * request. It lives for one call of the client and is dropped afterwards, so nothing is shared
@@ -45,17 +65,25 @@ export class PortalSession {
     return new URL(targetArg, baseArg).toString();
   }
 
-  public async get(urlArg: string): Promise<IPortalResponse> {
-    return this.request('GET', urlArg);
+  /**
+   * Loads a page. Like `post()`, it sends the request once more when the portal closes the
+   * connection without answering; `beforeRepeatArg` runs before that.
+   */
+  public async get(urlArg: string, beforeRepeatArg?: () => Promise<void>): Promise<IPortalResponse> {
+    return this.request('GET', urlArg, undefined, beforeRepeatArg);
   }
 
-  /** submits a form as `application/x-www-form-urlencoded` */
-  public async post(urlArg: string, fieldsArg: Array<[string, string]>): Promise<IPortalResponse> {
-    return this.request('POST', urlArg, new URLSearchParams(fieldsArg).toString());
+  /**
+   * Submits a form as `application/x-www-form-urlencoded`. When the portal closes the connection
+   * without answering, the form is sent once more after `beforeRepeatArg`; a redirect it answers
+   * with is followed the same way, without `beforeRepeatArg`.
+   */
+  public async post(urlArg: string, fieldsArg: Array<[string, string]>, beforeRepeatArg?: () => Promise<void>): Promise<IPortalResponse> {
+    return this.request('POST', urlArg, new URLSearchParams(fieldsArg).toString(), beforeRepeatArg);
   }
 
   private fail(
-    codeArg: 'aborted' | 'timeout' | 'unreachable' | 'refused' | 'too_large' | 'unexpected_response',
+    codeArg: 'aborted' | 'timeout' | 'unreachable' | 'dropped' | 'refused' | 'too_large' | 'unexpected_response',
     messageArg: string,
     extraArg: { status?: number; cause?: unknown } = {}
   ): HandelsregisterError {
@@ -66,7 +94,12 @@ export class PortalSession {
     });
   }
 
-  private async request(methodArg: 'GET' | 'POST', urlArg: string, bodyArg?: string): Promise<IPortalResponse> {
+  private async request(
+    methodArg: 'GET' | 'POST',
+    urlArg: string,
+    bodyArg?: string,
+    beforeRepeatArg?: () => Promise<void>
+  ): Promise<IPortalResponse> {
     let method = methodArg;
     let url = urlArg;
     let body = bodyArg;
@@ -74,7 +107,7 @@ export class PortalSession {
       if (new URL(url).origin !== this.origin) {
         throw this.fail('unexpected_response', 'The portal pointed to another site.');
       }
-      const response = await this.send(method, url, body);
+      const response = await this.sendRepeating(method, url, body, redirects === 0 ? beforeRepeatArg : undefined);
       if (redirectStatuses.has(response.status)) {
         const location = response.headers.get('location');
         await response.body?.cancel();
@@ -100,6 +133,39 @@ export class PortalSession {
     }
   }
 
+  /**
+   * Sends a request, and once more when the portal closed the connection without answering it. The
+   * portal's front end, Apache httpd with mod_proxy_http before the application servers, closes a
+   * kept-alive connection without any answer when it could not read its back end's answer, so that
+   * the client sends the request again, as browsers do. Every request of this client reads; sending
+   * one again changes nothing at the portal. A second close without an answer ends in `dropped`.
+   */
+  private async sendRepeating(
+    methodArg: 'GET' | 'POST',
+    urlArg: string,
+    bodyArg: string | undefined,
+    beforeRepeatArg?: () => Promise<void>
+  ): Promise<Response> {
+    try {
+      return await this.send(methodArg, urlArg, bodyArg);
+    } catch (error) {
+      if (!(error instanceof HandelsregisterError) || error.code !== 'dropped') {
+        throw error;
+      }
+    }
+    await beforeRepeatArg?.();
+    try {
+      return await this.send(methodArg, urlArg, bodyArg);
+    } catch (error) {
+      if (error instanceof HandelsregisterError && error.code === 'dropped') {
+        throw this.fail('dropped', 'The register portal closed the connection without answering, also when the request was sent once more.', {
+          cause: error.cause,
+        });
+      }
+      throw error;
+    }
+  }
+
   private async send(methodArg: 'GET' | 'POST', urlArg: string, bodyArg?: string): Promise<Response> {
     if (this.settings.signal?.aborted) {
       throw this.fail('aborted', 'The call was aborted.', { cause: this.settings.signal.reason });
@@ -125,7 +191,10 @@ export class PortalSession {
         signal: this.requestSignal(),
       });
     } catch (error) {
-      throw this.failureOf(error);
+      const failure = this.failureOf(error);
+      throw failure.code === 'unreachable' && closedWithoutAnswer(error)
+        ? this.fail('dropped', 'The register portal closed the connection without answering.', { cause: error })
+        : failure;
     }
     for (const cookie of response.headers.getSetCookie()) {
       const pair = cookie.split(';', 1)[0];

package/ts/handelsregister/interfaces.ts

@@ -92,7 +92,10 @@ export interface IRegisterSearchResult {
   portalNotice?: string;
   /** epoch ms: when the result list arrived */
   retrievedAt: number;
-  /** requests of this call that count against the portal's limit of 60 per hour (always 1) */
+  /**
+   * requests of this call that count against the portal's limit of 60 per hour: 1, or 2 when the
+   * portal dropped the search without an answer and it was sent once more
+   */
   countedRequests: number;
   notice: IStructuredContentNotice;
 }
@@ -202,7 +205,11 @@ export interface IRegisterEntry {
   structuredContent: IRegisterStructuredContent | null;
   /** epoch ms: when the last answer of the portal arrived */
   retrievedAt: number;
-  /** requests of this call that count against the portal's limit: 2, or 1 without structured content */
+  /**
+   * requests of this call that count against the portal's limit: 2, or 1 without structured content;
+   * one more for the search or the retrieval when the portal dropped it without an answer and it was
+   * sent once more
+   */
   countedRequests: number;
   notice: IStructuredContentNotice;
 }