@fin.cx/opendata 3.6.0 → 4.1.0

package/ts/handelsregister/classes.handelsregisterclient.ts

@@ -0,0 +1,474 @@
+import { HandelsregisterError, type THandelsregisterStep } from './classes.handelsregistererror.js';
+import { PortalSession, type ICallProgress, type IPortalResponse } from './classes.portalsession.js';
+import {
+  defaultMaxResponseBytes,
+  defaultPortalUrl,
+  defaultRequestTimeoutMs,
+  defaultUserAgent,
+  registerTypes,
+  structuredContentNotice,
+} from './constants.js';
+import { registerCourts } from './data.codelists.js';
+import * as forms from './forms.js';
+import * as html from './html.js';
+import type {
+  IHandelsregisterClientOptions,
+  IRegisterCallOptions,
+  IRegisterCourt,
+  IRegisterEntry,
+  IRegisterMatch,
+  IRegisterReferenceInput,
+  IRegisterRequestGate,
+  IRegisterSearchResult,
+  IRegisterStructuredContent,
+  TRegisterCountedRequestKind,
+  TRegisterSearchQuery,
+  TRegisterType,
+} from './interfaces.js';
+import { readResultList, type IParsedResultList, type IResultRow } from './parse.searchresults.js';
+import { parseStructuredContent } from './parse.structuredcontent.js';
+import { parseRegisterNumber, type IRegisterNumberParts } from './references.js';
+
+const searchFormPath = '/rp_web/erweitertesuche/welcome.xhtml';
+const searchFormId = 'form';
+const field = {
+  keywords: 'form:schlagwoerter',
+  keywordMode: 'form:schlagwortOptionen',
+  seat: 'form:NiederlassungSitz',
+  registerType: 'form:registerArt_input',
+  registerNumber: 'form:registerNummer',
+  court: 'form:registergericht_input',
+  pageSize: 'form:ergebnisseProSeite_input',
+  includeClosed: 'form:auchGeloeschte_input',
+  searchButton: 'form:btnSuche',
+};
+const searchFormMessagesId = 'form:messagesSuchMaske';
+/**
+ * The page the portal shows when it refuses a retrieval: for the TOR network and for addresses it
+ * has blocked (the portal's FAQ, question 6)
+ */
+const blockedRetrievalPattern = /Leider können wir den Dokumentenabruf[^.]*\./;
+const keywordModes = { all: '1', any: '2', exact: '3' } as const;
+const maxKeywordLength = 500;
+
+/** a result list and the address of the page that holds it */
+interface ISearchOutcome {
+  list: IParsedResultList;
+  pageUrl: string;
+}
+
+/** what the portal's search form is filled with */
+interface ISearchPlan {
+  keywords: string;
+  keywordMode: string;
+  seat: string;
+  registerType: string;
+  digits: string;
+  courtCode: string;
+  includeClosed: boolean;
+}
+
+const invalidQuery = (messageArg: string): HandelsregisterError =>
+  new HandelsregisterError('invalid_query', messageArg, { countedRequests: 0 });
+
+/** the error with the step and the counted requests of the call it ended */
+const inCall = (errorArg: HandelsregisterError, progressArg: ICallProgress): HandelsregisterError =>
+  new HandelsregisterError(errorArg.code, errorArg.message, {
+    status: errorArg.status,
+    retryAt: errorArg.retryAt,
+    step: errorArg.step ?? progressArg.step,
+    countedRequests: progressArg.countedRequests,
+    matches: errorArg.matches,
+    portalMessage: errorArg.portalMessage,
+    cause: errorArg.cause,
+  });
+
+const checkType = (typeArg: string | undefined): TRegisterType | undefined => {
+  if (typeArg === undefined) {
+    return undefined;
+  }
+  const type = registerTypes.find((knownArg) => knownArg === typeArg);
+  if (!type) {
+    throw invalidQuery(`Unknown register type; use one of ${registerTypes.join(', ')}.`);
+  }
+  return type;
+};
+
+const checkCourt = (courtCodeArg: string | undefined): string => {
+  if (courtCodeArg === undefined) {
+    return '';
+  }
+  if (!registerCourts.some((courtArg) => courtArg.code === courtCodeArg)) {
+    throw invalidQuery('Unknown register court code; see registerCourts.');
+  }
+  return courtCodeArg;
+};
+
+const checkNumber = (numberArg: string): IRegisterNumberParts => {
+  const parts = parseRegisterNumber(numberArg);
+  if (!parts) {
+    throw invalidQuery('A register number is digits, optionally followed by a location suffix such as HB.');
+  }
+  return parts;
+};
+
+const decodeUtf8 = (bytesArg: Uint8Array): string => new TextDecoder('utf-8').decode(bytesArg);
+
+/**
+ * Looks up entries of the German commercial, cooperative, partnership, association and civil-law
+ * partnership registers on the register portal of the federal states (handelsregister.de), over
+ * plain HTTP and without storage.
+ *
+ * Every call opens its own visit to the portal and forgets it afterwards: nothing is cached or
+ * shared between calls. The client never repeats a request by itself, never logs, and asks its
+ * gate before every request the portal's terms count:
+ * - `search()` makes one counted search;
+ * - `getEntry()` makes one counted search and, when the portal offers structured content for the
+ *   entry, one counted retrieval.
+ *
+ * Each search also loads the search form and the result list, two pages the terms do not count.
+ *
+ * The portal's terms allow single lookups for information only, at most 60 searches or
+ * retrievals per hour, and no systematic retrieval to build or update a copy of the registers.
+ */
+export class HandelsregisterClient {
+  private gate: IRegisterRequestGate;
+  private portalUrl: string;
+  private userAgent: string;
+  private requestTimeoutMs: number;
+  private maxResponseBytes: number;
+  private now: () => number;
+
+  constructor(optionsArg: IHandelsregisterClientOptions) {
+    if (!optionsArg || !optionsArg.gate) {
+      throw new TypeError('HandelsregisterClient needs a gate; see SlidingWindowGate.');
+    }
+    this.gate = optionsArg.gate;
+    this.portalUrl = optionsArg.portalUrl ?? defaultPortalUrl;
+    this.userAgent = optionsArg.userAgent ?? defaultUserAgent;
+    this.requestTimeoutMs = optionsArg.requestTimeoutMs ?? defaultRequestTimeoutMs;
+    this.maxResponseBytes = optionsArg.maxResponseBytes ?? defaultMaxResponseBytes;
+    this.now = optionsArg.now ?? Date.now;
+    const protocol = new URL(this.portalUrl).protocol;
+    if (protocol !== 'https:' && protocol !== 'http:') {
+      throw new TypeError('portalUrl must be an http or https URL.');
+    }
+  }
+
+  /**
+   * Searches the registers by name or by register number. Returns every row the portal lists, in
+   * its order, and whether the portal cut the list at its maximum. One counted search.
+   *
+   * A number is searched by its digits: rows with any location suffix are returned.
+   */
+  public async search(queryArg: TRegisterSearchQuery, optionsArg: IRegisterCallOptions = {}): Promise<IRegisterSearchResult> {
+    const plan = this.planSearch(queryArg);
+    const progress: ICallProgress = { countedRequests: 0 };
+    const { list } = await this.runSearch(plan, progress, this.openSession(optionsArg, progress));
+    const result: IRegisterSearchResult = {
+      matches: list.rows.map((rowArg) => rowArg.match),
+      truncated: list.truncated,
+      retrievedAt: this.now(),
+      countedRequests: progress.countedRequests,
+      notice: structuredContentNotice,
+    };
+    if (list.portalNotice) {
+      result.portalNotice = list.portalNotice;
+    }
+    return result;
+  }
+
+  /**
+   * Retrieves one register sheet: its row in the result list and, when the portal offers it, the
+   * parsed structured content (SI). Closed sheets are included, so a deleted entry is found with
+   * its status. One counted search, plus one counted retrieval when there is structured content.
+   *
+   * Fails with `not_found` when no sheet has the number at that court (with the entries whose
+   * location suffix differs in `matches`), and with `ambiguous` when a number without suffix fits
+   * several sheets. Never picks one of several.
+   */
+  public async getEntry(referenceArg: IRegisterReferenceInput, optionsArg: IRegisterCallOptions = {}): Promise<IRegisterEntry> {
+    const type = checkType(referenceArg.type);
+    if (!type) {
+      throw invalidQuery('A reference needs a register type.');
+    }
+    const courtCode = checkCourt(referenceArg.courtCode);
+    if (!courtCode) {
+      throw invalidQuery('A reference needs a register court.');
+    }
+    const number = checkNumber(referenceArg.number);
+    const progress: ICallProgress = { countedRequests: 0 };
+    const session = this.openSession(optionsArg, progress);
+    const outcome = await this.runSearch(
+      { keywords: '', keywordMode: keywordModes.all, seat: '', registerType: type, digits: number.digits, courtCode, includeClosed: true },
+      progress,
+      session
+    );
+
+    const row = this.pickRow(outcome.list, { courtCode, type, number }, progress);
+    let structuredContent: IRegisterStructuredContent | null = null;
+    let retrievedAt = this.now();
+    if (row.match.documents.includes('SI')) {
+      structuredContent = await this.retrieveStructuredContent(row, outcome, session, progress);
+      retrievedAt = this.now();
+    }
+    return {
+      match: row.match,
+      structuredContent,
+      retrievedAt,
+      countedRequests: progress.countedRequests,
+      notice: structuredContentNotice,
+    };
+  }
+
+  private openSession(optionsArg: IRegisterCallOptions, progressArg: ICallProgress): PortalSession {
+    return new PortalSession(
+      {
+        portalUrl: this.portalUrl,
+        userAgent: this.userAgent,
+        requestTimeoutMs: this.requestTimeoutMs,
+        maxResponseBytes: this.maxResponseBytes,
+        signal: optionsArg.signal,
+      },
+      progressArg
+    );
+  }
+
+  private planSearch(queryArg: TRegisterSearchQuery): ISearchPlan {
+    if ('name' in queryArg) {
+      const keywords = queryArg.name.replace(/\s+/g, ' ').trim();
+      if (!keywords) {
+        throw invalidQuery('A name search needs at least one keyword.');
+      }
+      if (keywords.length > maxKeywordLength) {
+        throw invalidQuery(`A name search takes at most ${maxKeywordLength} characters.`);
+      }
+      const mode = queryArg.match ?? 'all';
+      if (!(mode in keywordModes)) {
+        throw invalidQuery('match is all, any or exact.');
+      }
+      return {
+        keywords,
+        keywordMode: keywordModes[mode],
+        seat: (queryArg.seat ?? '').replace(/\s+/g, ' ').trim(),
+        registerType: checkType(queryArg.type) ?? '',
+        digits: '',
+        courtCode: checkCourt(queryArg.courtCode),
+        includeClosed: queryArg.includeClosed === true,
+      };
+    }
+    const type = checkType(queryArg.type);
+    if (!type) {
+      throw invalidQuery('A number search needs a register type.');
+    }
+    return {
+      keywords: '',
+      keywordMode: keywordModes.all,
+      seat: '',
+      registerType: type,
+      digits: checkNumber(queryArg.number).digits,
+      courtCode: checkCourt(queryArg.courtCode),
+      includeClosed: queryArg.includeClosed === true,
+    };
+  }
+
+  /**
+   * Asks the gate for one counted request. A refusal that is a `HandelsregisterError` reaches the
+   * caller with the step it guarded and the requests the call had counted before it; any other
+   * error of the gate reaches the caller unchanged.
+   */
+  private async acquireSlot(kindArg: TRegisterCountedRequestKind, stepArg: THandelsregisterStep, progressArg: ICallProgress): Promise<void> {
+    try {
+      await this.gate.acquire({ kind: kindArg });
+    } catch (error) {
+      if (error instanceof HandelsregisterError) {
+        throw inCall(error, { step: stepArg, countedRequests: progressArg.countedRequests });
+      }
+      throw error;
+    }
+  }
+
+  private async runSearch(planArg: ISearchPlan, progressArg: ICallProgress, sessionArg: PortalSession): Promise<ISearchOutcome> {
+    await this.acquireSlot('search', 'search', progressArg);
+    try {
+      progressArg.step = 'search_form';
+      const formPage = await sessionArg.get(sessionArg.resolve(searchFormPath));
+      const formDocument = this.htmlDocumentOf(formPage, 'The search form is not a web page.');
+      if (html.documentLanguage(formDocument) !== 'de') {
+        throw new HandelsregisterError('unexpected_response', 'The search form is not in German.');
+      }
+      const form = forms.readForm(formDocument, searchFormId);
+      if (!form) {
+        throw new HandelsregisterError('unexpected_response', 'The page holds no search form.');
+      }
+      const missing = Object.values(field).find((nameArg) => !form.controlNames.has(nameArg));
+      if (missing) {
+        throw new HandelsregisterError('unexpected_response', 'The search form lacks a field this version fills.');
+      }
+      const courts = this.courtsOf(form);
+      if (planArg.courtCode && !courts.some((courtArg) => courtArg.code === planArg.courtCode)) {
+        throw new HandelsregisterError('invalid_query', 'The portal does not offer this register court.');
+      }
+      if (planArg.registerType && !(form.options.get(field.registerType) ?? []).some((optionArg) => optionArg.value === planArg.registerType)) {
+        throw new HandelsregisterError('unexpected_response', 'The search form does not offer this register type.');
+      }
+      if (!(form.options.get(field.pageSize) ?? []).some((optionArg) => optionArg.value === '100')) {
+        throw new HandelsregisterError('unexpected_response', 'The search form does not offer 100 results per page.');
+      }
+
+      const fields = [...form.fields];
+      forms.setField(fields, field.keywords, planArg.keywords);
+      forms.setField(fields, field.keywordMode, planArg.keywordMode);
+      forms.setField(fields, field.seat, planArg.seat);
+      forms.setField(fields, field.registerType, planArg.registerType);
+      forms.setField(fields, field.registerNumber, planArg.digits);
+      forms.setField(fields, field.court, planArg.courtCode);
+      forms.setField(fields, field.pageSize, '100');
+      if (planArg.includeClosed) {
+        forms.setField(fields, field.includeClosed, 'on');
+      } else {
+        forms.removeField(fields, field.includeClosed);
+      }
+      forms.setField(fields, field.searchButton, '');
+
+      progressArg.step = 'search';
+      progressArg.countedRequests += 1;
+      const resultPage = await sessionArg.post(sessionArg.resolve(form.action, formPage.url), fields);
+
+      progressArg.step = 'result_list';
+      const resultText = decodeUtf8(resultPage.body);
+      try {
+        return { list: readResultList(resultText, courts), pageUrl: resultPage.url };
+      } catch (error) {
+        if (error instanceof HandelsregisterError && error.code === 'unexpected_response') {
+          this.throwIfBlocked(resultText);
+          const message = this.searchFormMessage(resultText);
+          if (message) {
+            throw new HandelsregisterError('unexpected_response', 'The portal answered the search with a message instead of a result list.', {
+              portalMessage: message,
+            });
+          }
+        }
+        throw error;
+      }
+    } catch (error) {
+      throw error instanceof HandelsregisterError ? inCall(error, progressArg) : error;
+    }
+  }
+
+  private pickRow(
+    listArg: IParsedResultList,
+    referenceArg: { courtCode: string; type: TRegisterType; number: IRegisterNumberParts },
+    progressArg: ICallProgress
+  ): IResultRow {
+    const sameNumber = listArg.rows.filter((rowArg) => {
+      const reference = rowArg.match.reference;
+      const number = parseRegisterNumber(reference.number);
+      return reference.court.code === referenceArg.courtCode && reference.type === referenceArg.type && number?.digits === referenceArg.number.digits;
+    });
+    const fitting = referenceArg.number.suffix
+      ? sameNumber.filter((rowArg) => parseRegisterNumber(rowArg.match.reference.number)?.suffix === referenceArg.number.suffix)
+      : sameNumber;
+    const matches: IRegisterMatch[] = sameNumber.map((rowArg) => rowArg.match);
+    if (fitting.length === 0) {
+      throw new HandelsregisterError('not_found', 'No register sheet has this number at this court.', {
+        step: 'result_list',
+        countedRequests: progressArg.countedRequests,
+        matches,
+      });
+    }
+    if (fitting.length > 1) {
+      throw new HandelsregisterError('ambiguous', 'Several register sheets have this number at this court.', {
+        step: 'result_list',
+        countedRequests: progressArg.countedRequests,
+        matches,
+      });
+    }
+    return fitting[0];
+  }
+
+  private async retrieveStructuredContent(
+    rowArg: IResultRow,
+    outcomeArg: ISearchOutcome,
+    sessionArg: PortalSession,
+    progressArg: ICallProgress
+  ): Promise<IRegisterStructuredContent> {
+    await this.acquireSlot('retrieval', 'structured_content', progressArg);
+    try {
+      progressArg.step = 'structured_content';
+      const resultForm = outcomeArg.list.form;
+      const submission = rowArg.documentSubmissions.get('SI');
+      if (!resultForm || !submission) {
+        throw new HandelsregisterError('unexpected_response', 'The result list offers structured content without a way to retrieve it.');
+      }
+      const fields: Array<[string, string]> = [...resultForm.fields];
+      for (const [name, value] of submission) {
+        forms.setField(fields, name, value);
+      }
+      progressArg.countedRequests += 1;
+      const answer = await sessionArg.post(sessionArg.resolve(resultForm.action, outcomeArg.pageUrl), fields);
+      if (/html/i.test(answer.contentType)) {
+        this.throwIfBlocked(decodeUtf8(answer.body));
+        throw new HandelsregisterError('unexpected_response', 'The portal answered with a page instead of the structured content.');
+      }
+      let xml: string;
+      try {
+        xml = new TextDecoder('utf-8', { fatal: true }).decode(answer.body);
+      } catch (error) {
+        throw new HandelsregisterError('invalid_structured_content', 'The structured content is not valid UTF-8.', { cause: error });
+      }
+      const content = parseStructuredContent(xml);
+      this.checkContentBelongsToRow(content, rowArg.match);
+      return content;
+    } catch (error) {
+      throw error instanceof HandelsregisterError ? inCall(error, progressArg) : error;
+    }
+  }
+
+  /** the structured content must describe the sheet of the row it was retrieved for */
+  private checkContentBelongsToRow(contentArg: IRegisterStructuredContent, matchArg: IRegisterMatch): void {
+    const reference = matchArg.reference;
+    const contentNumber = contentArg.register.number ? parseRegisterNumber(contentArg.register.number) : undefined;
+    const rowNumber = parseRegisterNumber(reference.number);
+    if (
+      (contentArg.register.courtCode !== undefined && contentArg.register.courtCode !== reference.court.code) ||
+      (contentArg.register.type !== undefined && contentArg.register.type !== reference.type) ||
+      (contentNumber !== undefined && contentNumber.digits !== rowNumber?.digits)
+    ) {
+      throw new HandelsregisterError('unexpected_response', 'The structured content belongs to another register sheet.');
+    }
+  }
+
+  private htmlDocumentOf(pageArg: IPortalResponse, messageArg: string): html.THtmlDocument {
+    if (!/html/i.test(pageArg.contentType)) {
+      throw new HandelsregisterError('unexpected_response', messageArg);
+    }
+    return html.parseHtml(decodeUtf8(pageArg.body));
+  }
+
+  /** the register courts the search form offers */
+  private courtsOf(formArg: forms.IPortalForm): IRegisterCourt[] {
+    return (formArg.options.get(field.court) ?? [])
+      .filter((optionArg) => optionArg.value !== '')
+      .map((optionArg) => ({ code: optionArg.value, name: optionArg.label }));
+  }
+
+  /** refuses with the portal's words when the page is its notice of a blocked retrieval */
+  private throwIfBlocked(htmlArg: string): void {
+    const document = html.parseHtml(htmlArg);
+    const notice = blockedRetrievalPattern.exec(html.textOf(document));
+    if (notice) {
+      throw new HandelsregisterError('refused', 'The register portal refused the retrieval from this address.', {
+        portalMessage: notice[0],
+      });
+    }
+  }
+
+  /** the message the search form shows, when the portal answered with the form again */
+  private searchFormMessage(htmlArg: string): string | undefined {
+    const document = html.parseHtml(htmlArg);
+    const messages = html.findById(document, searchFormMessagesId);
+    const text = messages ? html.textOf(messages) : '';
+    return text ? text : undefined;
+  }
+}

package/ts/handelsregister/classes.handelsregistererror.ts

@@ -0,0 +1,82 @@
+import type { IRegisterMatch } from './interfaces.js';
+
+/**
+ * Why a call failed:
+ * - `invalid_query`: the query or reference cannot be sent as it is (nothing was sent);
+ * - `rate_limited`: `SlidingWindowGate` refused the request; `retryAt` says when the next is possible;
+ * - `aborted`: the caller's signal aborted the call;
+ * - `timeout`: a request to the portal took longer than the client's limit;
+ * - `unreachable`: the portal could not be reached (name resolution, connection, TLS);
+ * - `refused`: the portal answered with an HTTP error, for example 403 when it blocks an address,
+ *   429 or 503 when it is overloaded (`status` carries the code), or with its notice that it refuses
+ *   retrievals from this address (`portalMessage` carries its words);
+ * - `too_large`: an answer exceeded the client's size limit;
+ * - `unexpected_response`: an answer did not have the shape this version reads: the portal changed
+ *   its pages, answered in another language, or showed an error page;
+ * - `not_found`: no register sheet matches the reference;
+ * - `ambiguous`: several register sheets match the reference; `matches` lists them;
+ * - `invalid_structured_content`: the structured register content could not be read.
+ */
+export type THandelsregisterErrorCode =
+  | 'invalid_query'
+  | 'rate_limited'
+  | 'aborted'
+  | 'timeout'
+  | 'unreachable'
+  | 'refused'
+  | 'too_large'
+  | 'unexpected_response'
+  | 'not_found'
+  | 'ambiguous'
+  | 'invalid_structured_content';
+
+/** the step of a call that failed */
+export type THandelsregisterStep = 'search_form' | 'search' | 'result_list' | 'structured_content';
+
+export interface IHandelsregisterErrorDetails {
+  status?: number;
+  retryAt?: number;
+  step?: THandelsregisterStep;
+  countedRequests?: number;
+  matches?: IRegisterMatch[];
+  portalMessage?: string;
+  cause?: unknown;
+}
+
+/**
+ * The typed error of the register client. Its message names the failure and never contains
+ * register data, so it can be logged.
+ */
+export class HandelsregisterError extends Error {
+  public readonly code: THandelsregisterErrorCode;
+  /** the HTTP status of a refusal */
+  public readonly status?: number;
+  /** epoch ms of the next possible request (`rate_limited`) */
+  public readonly retryAt?: number;
+  /** the step that failed */
+  public readonly step?: THandelsregisterStep;
+  /**
+   * counted requests the call had sent before it failed. A request that was sent may have counted
+   * at the portal even when its answer did not arrive.
+   */
+  public readonly countedRequests?: number;
+  /**
+   * the candidates of an `ambiguous` reference, and for `not_found` the entries with the same
+   * number whose location suffix differs
+   */
+  public readonly matches?: IRegisterMatch[];
+  /** a message the portal showed instead of the expected page, German */
+  public readonly portalMessage?: string;
+
+  constructor(codeArg: THandelsregisterErrorCode, messageArg: string, detailsArg: IHandelsregisterErrorDetails = {}) {
+    super(messageArg, detailsArg.cause === undefined ? undefined : { cause: detailsArg.cause });
+    this.name = 'HandelsregisterError';
+    this.code = codeArg;
+    this.status = detailsArg.status;
+    this.retryAt = detailsArg.retryAt;
+    this.step = detailsArg.step;
+    this.countedRequests = detailsArg.countedRequests;
+    this.matches = detailsArg.matches;
+    this.portalMessage = detailsArg.portalMessage;
+  }
+}