@fin.cx/opendata 3.5.1 → 4.0.0

package/ts/classes.handelsregister.ts

@@ -1,360 +0,0 @@
-import type { BusinessRecord } from './classes.businessrecord.js';
-import type { OpenData } from './classes.main.opendata.js';
-import * as plugins from './plugins.js';
-
-/**
- * the HandlesRegister exposed as a class
- */
-export class HandelsRegister {
-  private openDataRef: OpenData;
-  private asyncExecutionStack = new plugins.lik.AsyncExecutionStack();
-  private downloadDir: string;
-  private uniqueDowloadFolder: string;
-
-  // Puppeteer wrapper instance
-  public smartbrowserInstance = new plugins.smartbrowser.SmartBrowser();
-
-  constructor(openDataRef: OpenData, downloadDirArg: string) {
-    this.openDataRef = openDataRef;
-    this.downloadDir = downloadDirArg;
-    this.uniqueDowloadFolder = plugins.path.join(this.downloadDir, plugins.smartunique.uniSimple());
-  }
-
-  public async start() {
-    // Start the browser
-    await plugins.smartfile.fs.ensureDir(this.uniqueDowloadFolder);
-    await this.smartbrowserInstance.start();
-  }
-
-  public async stop() {
-    // Stop the browser
-    await plugins.smartfile.fs.remove(this.uniqueDowloadFolder);
-    await this.smartbrowserInstance.stop();
-  }
-
-  /**
-   * Creates a new page and configures it to allow file downloads
-   * to a predefined path.
-   */
-  public getNewPage = async () => {
-    const page = await this.smartbrowserInstance.headlessBrowser.newPage();
-
-    // 1) Create a DevTools session for this page
-    const cdpSession = await page.target().createCDPSession();
-
-    // 2) Allow file downloads and set the download path
-    await cdpSession.send('Page.setDownloadBehavior', {
-      behavior: 'allow',
-      downloadPath: this.uniqueDowloadFolder, // <-- Change this to your desired absolute path
-    });
-
-    // Optionally set viewport and go to page
-    await page.setViewport({ width: 1920, height: 1080 });
-    await page.goto('https://www.handelsregister.de/');
-    return page;
-  };
-
-  private navigateToPage = async (
-    pageArg: plugins.smartbrowser.smartpuppeteer.puppeteer.Page,
-    pageNameArg: string
-  ) => {
-    try {
-      await pageArg.evaluate((pageNameArg2) => {
-        const elements = Array.from(document.querySelectorAll('.ui-menuitem-text > span'));
-        const targetElement = elements.find((el) => el.textContent?.trim() === pageNameArg2);
-        if (targetElement) {
-          (targetElement as HTMLElement).click();
-        }
-      }, pageNameArg);
-      console.log(`Navigated to the ${pageNameArg} page successfully.`);
-    } catch (error) {
-      console.error(`Failed to navigate to the ${pageNameArg} page:`, error);
-    }
-  };
-
-  private waitForResults = async (pageArg: plugins.smartbrowser.smartpuppeteer.puppeteer.Page) => {
-    await pageArg
-      .waitForSelector('#ergebnissForm\\:selectedSuchErgebnisFormTable_data', {
-        timeout: 30000,
-      })
-      .catch(async (err) => {
-        await pageArg.screenshot({ path: `${this.downloadDir}/error.png` as `${string}.png` });
-        throw err;
-      });
-
-    const businessRecords: BusinessRecord['data'][] = await pageArg.evaluate(() => {
-      const rows = document.querySelectorAll(
-        '#ergebnissForm\\:selectedSuchErgebnisFormTable_data > tr'
-      );
-      const records: BusinessRecord['data'][] = [];
-
-      rows.forEach((row) => {
-        const nameElement = row.querySelector('td.ui-panelgrid-cell span.marginLeft20');
-        const cityElement = row.querySelector('td.ui-panelgrid-cell.sitzSuchErgebnisse span');
-        const statusElement = row.querySelector('td.ui-panelgrid-cell span.verticalText');
-        const registrationCourtElement = row.querySelector(
-          'td.ui-panelgrid-cell.fontTableNameSize'
-        );
-
-        const name = nameElement?.textContent?.trim();
-        const city = cityElement?.textContent?.trim();
-        const status = statusElement?.textContent?.trim();
-        const registrationId = registrationCourtElement?.textContent?.trim();
-
-        // Push parsed data into records array
-        records.push({
-          name,
-          city,
-          registrationId,
-          businessType: status,
-        });
-      });
-
-      return records;
-    });
-    return businessRecords;
-  };
-
-  private clickFindButton = async (pageArg: plugins.smartbrowser.smartpuppeteer.puppeteer.Page, resultsLimitArg: number = 100) => {
-    try {
-      // Wait for the button with the text "Find" to appear
-      await pageArg.waitForSelector('span.ui-button-text.ui-c', { timeout: 5000 });
-
-      // adjust to 100 results per page
-      await pageArg.select('#form\\:ergebnisseProSeite_input', `${resultsLimitArg}`);
-
-      // Locate and click the button using its text
-      await pageArg.evaluate(() => {
-        const buttons = Array.from(document.querySelectorAll('span.ui-button-text.ui-c'));
-        const targetButton = buttons.find((button) => button.textContent?.trim() === 'Find');
-        if (targetButton) {
-          const parentButton = targetButton.closest('button') || targetButton;
-          (parentButton as HTMLElement).click();
-        }
-      });
-
-      console.log('Find button clicked successfully!');
-    } catch (error) {
-      console.error('Failed to find or click the "Find" button:', error);
-    }
-  };
-
-  private async downloadFile(
-    pageArg: plugins.smartbrowser.smartpuppeteer.puppeteer.Page,
-    typeArg: 'SI' | 'AD'
-  ) {
-    // Trigger the file download by clicking on the relevant link
-    await pageArg.evaluate((typeArg2) => {
-      // Locate the table body
-      const tableBody = document.querySelector(
-        '#ergebnissForm\\:selectedSuchErgebnisFormTable_data'
-      );
-      if (!tableBody) {
-        throw new Error('Table body not found');
-      }
-
-      // Locate the first row
-      const firstRow = tableBody.querySelector('tr:nth-child(1)');
-      if (!firstRow) {
-        throw new Error('First row not found');
-      }
-
-      // Locate the last cell in the first row
-      const lastCell = firstRow.querySelector('td:last-child');
-      if (!lastCell) {
-        throw new Error('Last cell not found in the first row');
-      }
-
-      // Locate the download links
-      const adLink = lastCell.querySelector('a:first-of-type');
-      const siLink = lastCell.querySelector('a:last-of-type');
-      if (!siLink) {
-        throw new Error('SI link not found in the last cell');
-      }
-
-      // Simulate a click on the last <a> element
-      switch (typeArg2) {
-        case 'AD':
-          (adLink as HTMLElement).click();
-          break;
-        case 'SI':
-          (siLink as HTMLElement).click();
-          break;
-        default:
-          throw new Error('Invalid file type');
-      }
-    }, typeArg);
-
-
-    await plugins.smartfile.fs.waitForFileToBeReady(this.uniqueDowloadFolder);
-
-    const files = await plugins.smartfile.fs.fileTreeToObject(this.uniqueDowloadFolder, '**/*');
-    const file = files[0];
-
-    // lets clear the folder for the next download
-    await plugins.smartfile.fs.ensureEmptyDir(this.uniqueDowloadFolder);
-    switch (typeArg) {
-      case 'AD':
-        await file.rename(`ad.pdf`);
-        break;
-      case 'SI':
-        await file.rename(`si.xml`);
-        break;
-        break;
-    }
-    return file;
-  }
-
-  /**
-   * Helper method to parse the German registration string
-   */
-  private async parseGermanRegistration(
-    input: string
-  ): Promise<BusinessRecord['data']['germanParsedRegistration']> {
-    // e.g. District court Berlin (Charlottenburg) HRB 123456
-    const regex =
-      /District court (\p{L}[\p{L}\s-]*?(?:\s*\([\p{L}\s-]+\))?)\s+(HRA|HRB|GnR|VR|PR|GsR)\s+(\d+)/u;
-    const match = input.match(regex);
-
-    if (match) {
-      return {
-        court: match[1],
-        type: match[2] as 'HRA' | 'HRB', // Adjust if needed
-        number: match[3],
-      };
-    }
-  }
-
-  /**
-   * Search for a company by name and return basic info
-   */
-  public async searchCompany(companyNameArg: string, resultsLimitArg: number = 100) {
-    return this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
-      const page = await this.getNewPage();
-      await this.navigateToPage(page, 'Normal search');
-
-      try {
-        // Wait for the textarea to appear
-        await page.waitForSelector('#form\\:schlagwoerter', { timeout: 5000 });
-
-        // Enter text into the textarea
-        const inputText = companyNameArg;
-        await page.evaluate((text) => {
-          const textarea = document.querySelector<HTMLTextAreaElement>('#form\\:schlagwoerter');
-          if (textarea) {
-            textarea.value = text; // Set the value
-            // Trigger the change event manually if required
-            const event = new Event('change', { bubbles: true });
-            textarea.dispatchEvent(event);
-          }
-        }, inputText);
-
-        console.log('Text entered successfully!');
-      } catch (error) {
-        console.error('Failed to find or enter text into the textarea:', error);
-      }
-
-      try {
-        // Wait for the radio button's label to appear
-        await page.waitForSelector('label[for="form:schlagwortOptionen:0"]', { timeout: 5000 });
-
-        // Click the label to select the radio button
-        await page.evaluate(() => {
-          const label = document.querySelector<HTMLLabelElement>(
-            'label[for="form:schlagwortOptionen:0"]'
-          );
-          if (label) {
-            label.click();
-          }
-        });
-
-        console.log('Radio button clicked successfully!');
-      } catch (error) {
-        console.error('Failed to find or click the radio button:', error);
-      }
-
-      await this.clickFindButton(page, resultsLimitArg);
-
-      const businessRecords = await this.waitForResults(page);
-
-      // Parse out the registration info
-      for (const record of businessRecords) {
-        if (record.registrationId) {
-          record.germanParsedRegistration = await this.parseGermanRegistration(
-            record.registrationId
-          );
-        }
-      }
-
-      await page.close();
-      return businessRecords;
-    }, 60000);
-  }
-
-  /**
-   * Search for a specific company (known register type/number/court),
-   * then click on an element that triggers a file download.
-   */
-  public async getSpecificCompany(companyArg: BusinessRecord['data']['germanParsedRegistration']) {
-    return this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
-      const page = await this.getNewPage();
-      await this.navigateToPage(page, 'Normal search');
-      await page.waitForSelector('#form\\:schlagwoerter', { timeout: 5000 });
-
-      // 1) Type of Register (e.g. HRB, HRA, etc.)
-      await page.waitForSelector('#form\\:registerArt_label');
-      await page.click('#form\\:registerArt_label');
-      await page.waitForSelector('#form\\:registerArt_items');
-      await page.evaluate((type) => {
-        const options = Array.from(document.querySelectorAll('#form\\:registerArt_items li'));
-        const targetOption = options.find((option) => option.textContent?.trim() === type);
-        (targetOption as HTMLElement)?.click();
-      }, companyArg.type);
-
-      // 2) Register number
-      await page.waitForSelector('#form\\:registerNummer');
-      await page.type('#form\\:registerNummer', companyArg.number);
-
-      // 3) Register court
-      await page.waitForSelector('#form\\:registergericht_label');
-      await page.click('#form\\:registergericht_label');
-      await page.waitForSelector('#form\\:registergericht_items');
-      await page.evaluate((court) => {
-        const options = Array.from(document.querySelectorAll('#form\\:registergericht_items li'));
-        const targetOption = options.find((option) => option.textContent?.trim() === court);
-        (targetOption as HTMLElement)?.click();
-      }, companyArg.court);
-
-      // Click 'Find'
-      await this.clickFindButton(page);
-
-      // Optionally grab the results, just for logging
-      const businessRecords = await this.waitForResults(page);
-      console.log(businessRecords);
-
-      const files: plugins.smartfile.SmartFile[] = [];
-
-      // download files
-      files.push(await this.downloadFile(page, 'SI'));
-      files.push(await this.downloadFile(page, 'AD'));
-
-      // At this point, the file should have been downloaded automatically
-      // to the path specified by `Page.setDownloadBehavior`
-      await page.close();
-
-      return {
-        businessRecords,
-        files,
-      };
-    }, 60000);
-  }
-
-  /**
-   * get specific company by full name
-   */
-  public async getSpecificCompanyByName(companyNameArg: string) {
-    const businessRecords = await this.searchCompany(companyNameArg, 1);
-    const result = this.getSpecificCompany(businessRecords[0].germanParsedRegistration);
-    return result;
-  }
-}

package/ts/classes.jsonldata.ts

@@ -1,113 +0,0 @@
-import * as plugins from './plugins.js';
-import type { OpenData } from './classes.main.opendata.js';
-
-export type SeedEntryType = {
-  all_attributes: {
-    _registerArt: string;
-    _registerNummer: string;
-    additional_data: {
-      AD: boolean;
-      CD: boolean;
-      DK: boolean;
-      HD: boolean;
-      SI: boolean;
-      UT: boolean;
-      VÖ: boolean;
-    };
-    federal_state: string;
-    native_company_number: string;
-    registered_office: string;
-    registrar: string;
-  };
-  company_number: string;
-  current_status: string;
-  jurisdiction_code: string;
-  name: string;
-  officers: {
-    name: string;
-    other_attributes: {
-      city: string;
-      firstname: string;
-      flag: string;
-      lastname: string;
-    };
-    position: string;
-    start_date: string; // ISO 8601 date string
-    type: string;
-  }[];
-  registered_address: string;
-  retrieved_at: string; // ISO 8601 date string
-};
-
-export class JsonlDataProcessor<T> {
-  private germanBusinessDataDir: string;
-  public forEachFunction: (entryArg: T) => Promise<void>;
-
-  constructor(germanBusinessDataDirArg: string, forEachFunctionArg: typeof this.forEachFunction) {
-    this.germanBusinessDataDir = germanBusinessDataDirArg;
-    this.forEachFunction = forEachFunctionArg;
-  }
-
-  // TODO: define a mapper as argument instead of hard-coding it
-  public async processDataFromUrl(
-    dataUrlArg = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2'
-  ) {
-    const done = plugins.smartpromise.defer();
-    const dataExists = await plugins.smartfile.fs.isDirectory(this.germanBusinessDataDir);
-    if (!dataExists) {
-      await plugins.smartfile.fs.ensureDir(this.germanBusinessDataDir);
-    } else {
-    }
-
-    const smartarchive = await plugins.smartarchive.SmartArchive.fromArchiveUrl(dataUrlArg);
-    const jsonlDataStream = await smartarchive.exportToStreamOfStreamFiles();
-    let totalRecordsCounter = 0;
-    let nextRest: string = '';
-    jsonlDataStream.pipe(
-      new plugins.smartstream.SmartDuplex({
-        objectMode: true,
-        writeFunction: async (chunkArg: plugins.smartfile.StreamFile, streamToolsArg) => {
-          const readStream = await chunkArg.createReadStream();
-          readStream.pipe(
-            new plugins.smartstream.SmartDuplex({
-              objectMode: true,
-              writeFunction: async (chunkArg: Buffer, streamToolsArg) => {
-                const currentString = nextRest + chunkArg.toString();
-                const lines = currentString.split('\n');
-                nextRest = lines.pop();
-                console.log(`Got another ${lines.length} records.`);
-                const concurrentProcessor = new plugins.smartarray.ConcurrentProcessor<string>(
-                  async (line) => {
-                    let entry: T;
-                    if (!line) return;
-                    try {
-                      entry = JSON.parse(line);
-                    } catch (err) {
-                      console.log(line);
-                      await plugins.smartdelay.delayFor(10000);
-                    }
-                    if (!entry) return;
-                    totalRecordsCounter++;
-                    if (totalRecordsCounter % 10000 === 0)
-                      console.log(`${totalRecordsCounter} total records.`);
-                    await this.forEachFunction(entry);
-                  },
-                  1000
-                );
-                await concurrentProcessor.process(lines);
-              },
-              finalFunction: async (streamToolsArg) => {
-                console.log(`finished processing ${totalRecordsCounter} records.`);
-                if (nextRest) {
-                  JSON.parse(nextRest);
-                };
-                done.resolve();
-              },
-            })
-          );
-        },
-      })
-    );
-    await done.promise;
-  }
-}

package/ts/classes.main.opendata.ts

@@ -1,87 +0,0 @@
-import { BusinessRecord } from './classes.businessrecord.js';
-import { HandelsRegister } from './classes.handelsregister.js';
-import { JsonlDataProcessor, type SeedEntryType } from './classes.jsonldata.js';
-import * as paths from './paths.js';
-import * as plugins from './plugins.js';
-
-export interface IOpenDataConfig {
-  downloadDir: string;
-  germanBusinessDataDir: string;
-  nogitDir: string;
-}
-
-export class OpenData {
-  public db: plugins.smartdata.SmartdataDb;
-  private serviceQenv: plugins.qenv.Qenv;
-  private config: IOpenDataConfig;
-
-  public jsonLDataProcessor: JsonlDataProcessor<SeedEntryType>;
-  public handelsregister: HandelsRegister;
-
-  public CBusinessRecord = plugins.smartdata.setDefaultManagerForDoc(this, BusinessRecord);
-
-  constructor(configArg: IOpenDataConfig) {
-    if (!configArg) {
-      throw new Error('@fin.cx/opendata: Configuration is required. You must provide downloadDir, germanBusinessDataDir, and nogitDir paths.');
-    }
-    if (!configArg.downloadDir || !configArg.germanBusinessDataDir || !configArg.nogitDir) {
-      throw new Error('@fin.cx/opendata: All directory paths are required (downloadDir, germanBusinessDataDir, nogitDir).');
-    }
-    this.config = configArg;
-    this.serviceQenv = new plugins.qenv.Qenv(paths.packageDir, this.config.nogitDir);
-  }
-
-  public async start() {
-    // Ensure configured directories exist
-    await plugins.smartfile.fs.ensureDir(this.config.nogitDir);
-    await plugins.smartfile.fs.ensureDir(this.config.downloadDir);
-    await plugins.smartfile.fs.ensureDir(this.config.germanBusinessDataDir);
-
-    this.db = new plugins.smartdata.SmartdataDb({
-      mongoDbUrl: await this.serviceQenv.getEnvVarOnDemand('MONGODB_URL'),
-      mongoDbName: await this.serviceQenv.getEnvVarOnDemand('MONGODB_NAME'),
-      mongoDbUser: await this.serviceQenv.getEnvVarOnDemand('MONGODB_USER'),
-      mongoDbPass: await this.serviceQenv.getEnvVarOnDemand('MONGODB_PASS'),
-    });
-    await this.db.init();
-    this.jsonLDataProcessor = new JsonlDataProcessor(
-      this.config.germanBusinessDataDir,
-      async (entryArg) => {
-        const businessRecord = new this.CBusinessRecord();
-        businessRecord.id = await this.CBusinessRecord.getNewId();
-        businessRecord.data.name = entryArg.name;
-        businessRecord.data.germanParsedRegistration = {
-          court: entryArg.all_attributes.registered_office,
-          number: entryArg.all_attributes._registerNummer,
-          type: entryArg.all_attributes._registerArt as 'HRA' | 'HRB',
-        };
-        await businessRecord.save();
-      }
-    );
-    this.handelsregister = new HandelsRegister(this, this.config.downloadDir);
-    await this.handelsregister.start();
-  }
-
-  public async buildInitialDb() {
-    await this.jsonLDataProcessor.processDataFromUrl();
-  }
-
-  public async slowValidateDb() {
-
-  }
-
-  public async validateSearchByName() {
-
-  }
-
-  public async searchDbByBusinessNameAndPostalCode(businessNameArg: string, postalCodeArg: string) {
-
-  }
-
-
-
-  public async stop() {
-    await this.db.close();
-    await this.handelsregister.stop();
-  }
-}