@fin.cx/opendata 1.2.0 → 1.4.4

package/ts/classes.handelsregister.ts

@@ -1,12 +1,17 @@
+import type { BusinessRecord } from './classes.businessrecord.js';
 import type { OpenData } from './classes.main.opendata.js';
 import * as plugins from './plugins.js';
+import * as paths from './paths.js';
 
 /**
  * the HandlesRegister exposed as a class
  */
 export class HandelsRegister {
   private openDataRef: OpenData;
+  private asyncExecutionStack = new plugins.lik.AsyncExecutionStack();
+  private uniqueDowloadFolder = plugins.path.join(paths.downloadDir, plugins.smartunique.uniSimple()); 
 
+  // Puppeteer wrapper instance
   public smartbrowserInstance = new plugins.smartbrowser.SmartBrowser();
 
   constructor(openDataRef: OpenData) {
@@ -14,96 +19,72 @@ export class HandelsRegister {
   }
 
   public async start() {
+    // Start the browser
+    await plugins.smartfile.fs.ensureDir(this.uniqueDowloadFolder);
     await this.smartbrowserInstance.start();
   }
 
   public async stop() {
+    // Stop the browser
+    await plugins.smartfile.fs.remove(this.uniqueDowloadFolder);
     await this.smartbrowserInstance.stop();
   }
 
   /**
-   * Search for a company by name
+   * Creates a new page and configures it to allow file downloads
+   * to a predefined path.
    */
-  public async getDataForCompany(companyNameArg: string) {
+  public getNewPage = async () => {
     const page = await this.smartbrowserInstance.headlessBrowser.newPage();
-    await page.setViewport({ width: 1920, height: 1080 });
-    await page.goto('https://www.handelsregister.de/');
-    await page.evaluate(() => {
-      const elements = Array.from(document.querySelectorAll('.ui-menuitem-text > span'));
-      const targetElement = elements.find((el) => el.textContent?.trim() === 'Normal search');
-      if (targetElement) {
-        (targetElement as HTMLElement).click();
-      }
-    });
 
-    try {
-      // Wait for the textarea to appear
-      await page.waitForSelector('#form\\:schlagwoerter', { timeout: 5000 });
+    // 1) Create a DevTools session for this page
+    const cdpSession = await page.target().createCDPSession();
 
-      // Enter text into the textarea using page.evaluate
-      const inputText = companyNameArg;
-      await page.evaluate((text) => {
-        const textarea = document.querySelector<HTMLTextAreaElement>('#form\\:schlagwoerter');
-        if (textarea) {
-          textarea.value = text; // Set the value
-          // Trigger the change event manually if required
-          const event = new Event('change', { bubbles: true });
-          textarea.dispatchEvent(event);
-        }
-      }, inputText);
+    // 2) Allow file downloads and set the download path
+    await cdpSession.send('Page.setDownloadBehavior', {
+      behavior: 'allow',
+      downloadPath: this.uniqueDowloadFolder, // <-- Change this to your desired absolute path
+    });
 
-      console.log('Text entered successfully!');
-    } catch (error) {
-      console.error('Failed to find or enter text into the textarea:', error);
-    }
+    // Optionally set viewport and go to page
+    await page.setViewport({ width: 1920, height: 1080 });
+    await page.goto('https://www.handelsregister.de/');
+    return page;
+  };
 
+  private navigateToPage = async (
+    pageArg: plugins.smartbrowser.smartpuppeteer.puppeteer.Page,
+    pageNameArg: string
+  ) => {
     try {
-      // Wait for the radio button's label to appear
-      await page.waitForSelector('label[for="form:schlagwortOptionen:0"]', { timeout: 5000 });
-
-      // Click the label to select the radio button
-      await page.evaluate(() => {
-        const label = document.querySelector<HTMLLabelElement>(
-          'label[for="form:schlagwortOptionen:0"]'
-        );
-        if (label) {
-          label.click();
+      await pageArg.evaluate((pageNameArg2) => {
+        const elements = Array.from(document.querySelectorAll('.ui-menuitem-text > span'));
+        const targetElement = elements.find((el) => el.textContent?.trim() === pageNameArg2);
+        if (targetElement) {
+          (targetElement as HTMLElement).click();
         }
-      });
-
-      console.log('Radio button clicked successfully!');
+      }, pageNameArg);
+      console.log(`Navigated to the ${pageNameArg} page successfully.`);
     } catch (error) {
-      console.error('Failed to find or click the radio button:', error);
+      console.error(`Failed to navigate to the ${pageNameArg} page:`, error);
     }
+  };
 
-    try {
-      // Wait for the button with the text "Find" to appear
-      await page.waitForSelector('span.ui-button-text.ui-c', { timeout: 5000 });
-
-      // Locate and click the button using its text
-      await page.evaluate(() => {
-        const buttons = Array.from(document.querySelectorAll('span.ui-button-text.ui-c'));
-        const targetButton = buttons.find((button) => button.textContent?.trim() === 'Find');
-        if (targetButton) {
-          const parentButton = targetButton.closest('button') || targetButton;
-          (parentButton as HTMLElement).click();
-        }
+  private waitForResults = async (pageArg: plugins.smartbrowser.smartpuppeteer.puppeteer.Page) => {
+    await pageArg
+      .waitForSelector('#ergebnissForm\\:selectedSuchErgebnisFormTable_data', {
+        timeout: 30000,
+      })
+      .catch(async (err) => {
+        await pageArg.screenshot({ path: paths.downloadDir + '/error.png' });
+        throw err;
       });
 
-      console.log('Find button clicked successfully!');
-    } catch (error) {
-      console.error('Failed to find or click the "Find" button:', error);
-    }
-
-    await page.waitForSelector('#ergebnissForm\\:selectedSuchErgebnisFormTable_data', {
-      timeout: 10000,
-    });
-
-    const businessRecords = await page.evaluate(() => {
+    const businessRecords: BusinessRecord['data'][] = await pageArg.evaluate(() => {
       const rows = document.querySelectorAll(
         '#ergebnissForm\\:selectedSuchErgebnisFormTable_data > tr'
       );
-      const records = [];
+      const records: BusinessRecord['data'][] = [];
 
       rows.forEach((row) => {
         const nameElement = row.querySelector('td.ui-panelgrid-cell span.marginLeft20');
@@ -116,23 +97,262 @@ export class HandelsRegister {
         const name = nameElement?.textContent?.trim();
         const city = cityElement?.textContent?.trim();
         const status = statusElement?.textContent?.trim();
-        const registrationCourt = registrationCourtElement?.textContent?.trim();
+        const registrationId = registrationCourtElement?.textContent?.trim();
 
         // Push parsed data into records array
         records.push({
           name,
           city,
-          registrationCourt,
+          registrationId,
           businessType: status,
         });
       });
 
       return records;
     });
+    return businessRecords;
+  };
+
+  private clickFindButton = async (pageArg: plugins.smartbrowser.smartpuppeteer.puppeteer.Page, resultsLimitArg: number = 100) => {
+    try {
+      // Wait for the button with the text "Find" to appear
+      await pageArg.waitForSelector('span.ui-button-text.ui-c', { timeout: 5000 });
 
-    await page.close();
+      // adjust to 100 results per page
+      await pageArg.select('#form\\:ergebnisseProSeite_input', `${resultsLimitArg}`);
 
-    // Finally, we return an object, which triggers a JSON file download
-    return businessRecords;
+      // Locate and click the button using its text
+      await pageArg.evaluate(() => {
+        const buttons = Array.from(document.querySelectorAll('span.ui-button-text.ui-c'));
+        const targetButton = buttons.find((button) => button.textContent?.trim() === 'Find');
+        if (targetButton) {
+          const parentButton = targetButton.closest('button') || targetButton;
+          (parentButton as HTMLElement).click();
+        }
+      });
+
+      console.log('Find button clicked successfully!');
+    } catch (error) {
+      console.error('Failed to find or click the "Find" button:', error);
+    }
+  };
+
+  private async downloadFile(
+    pageArg: plugins.smartbrowser.smartpuppeteer.puppeteer.Page,
+    typeArg: 'SI' | 'AD'
+  ) {
+    // Trigger the file download by clicking on the relevant link
+    await pageArg.evaluate((typeArg2) => {
+      // Locate the table body
+      const tableBody = document.querySelector(
+        '#ergebnissForm\\:selectedSuchErgebnisFormTable_data'
+      );
+      if (!tableBody) {
+        throw new Error('Table body not found');
+      }
+
+      // Locate the first row
+      const firstRow = tableBody.querySelector('tr:nth-child(1)');
+      if (!firstRow) {
+        throw new Error('First row not found');
+      }
+
+      // Locate the last cell in the first row
+      const lastCell = firstRow.querySelector('td:last-child');
+      if (!lastCell) {
+        throw new Error('Last cell not found in the first row');
+      }
+
+      // Locate the download links
+      const adLink = lastCell.querySelector('a:first-of-type');
+      const siLink = lastCell.querySelector('a:last-of-type');
+      if (!siLink) {
+        throw new Error('SI link not found in the last cell');
+      }
+
+      // Simulate a click on the last <a> element
+      switch (typeArg2) {
+        case 'AD':
+          (adLink as HTMLElement).click();
+          break;
+        case 'SI':
+          (siLink as HTMLElement).click();
+          break;
+        default:
+          throw new Error('Invalid file type');
+      }
+    }, typeArg);
+
+
+    await plugins.smartfile.fs.waitForFileToBeReady(this.uniqueDowloadFolder);
+
+    const files = await plugins.smartfile.fs.fileTreeToObject(this.uniqueDowloadFolder, '**/*');
+    const file = files[0];
+
+    // lets clear the folder for the next download
+    await plugins.smartfile.fs.ensureEmptyDir(this.uniqueDowloadFolder);
+    switch (typeArg) {
+      case 'AD':
+        await file.rename(`ad.pdf`);
+        break;
+      case 'SI':
+        await file.rename(`si.xml`);
+        break;
+        break;
+    }
+    return file;
+  }
+
+  /**
+   * Helper method to parse the German registration string
+   */
+  private async parseGermanRegistration(
+    input: string
+  ): Promise<BusinessRecord['data']['germanParsedRegistration']> {
+    // e.g. District court Berlin (Charlottenburg) HRB 123456
+    const regex =
+      /District court (\p{L}[\p{L}\s-]*?(?:\s*\([\p{L}\s-]+\))?)\s+(HRA|HRB|GnR|VR|PR|GsR)\s+(\d+)/u;
+    const match = input.match(regex);
+
+    if (match) {
+      return {
+        court: match[1],
+        type: match[2] as 'HRA' | 'HRB', // Adjust if needed
+        number: match[3],
+      };
+    }
+  }
+
+  /**
+   * Search for a company by name and return basic info
+   */
+  public async searchCompany(companyNameArg: string, resultsLimitArg: number = 100) {
+    return this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      const page = await this.getNewPage();
+      await this.navigateToPage(page, 'Normal search');
+
+      try {
+        // Wait for the textarea to appear
+        await page.waitForSelector('#form\\:schlagwoerter', { timeout: 5000 });
+
+        // Enter text into the textarea
+        const inputText = companyNameArg;
+        await page.evaluate((text) => {
+          const textarea = document.querySelector<HTMLTextAreaElement>('#form\\:schlagwoerter');
+          if (textarea) {
+            textarea.value = text; // Set the value
+            // Trigger the change event manually if required
+            const event = new Event('change', { bubbles: true });
+            textarea.dispatchEvent(event);
+          }
+        }, inputText);
+
+        console.log('Text entered successfully!');
+      } catch (error) {
+        console.error('Failed to find or enter text into the textarea:', error);
+      }
+
+      try {
+        // Wait for the radio button's label to appear
+        await page.waitForSelector('label[for="form:schlagwortOptionen:0"]', { timeout: 5000 });
+
+        // Click the label to select the radio button
+        await page.evaluate(() => {
+          const label = document.querySelector<HTMLLabelElement>(
+            'label[for="form:schlagwortOptionen:0"]'
+          );
+          if (label) {
+            label.click();
+          }
+        });
+
+        console.log('Radio button clicked successfully!');
+      } catch (error) {
+        console.error('Failed to find or click the radio button:', error);
+      }
+
+      await this.clickFindButton(page, resultsLimitArg);
+
+      const businessRecords = await this.waitForResults(page);
+
+      // Parse out the registration info
+      for (const record of businessRecords) {
+        if (record.registrationId) {
+          record.germanParsedRegistration = await this.parseGermanRegistration(
+            record.registrationId
+          );
+        }
+      }
+
+      await page.close();
+      return businessRecords;
+    }, 60000);
+  }
+
+  /**
+   * Search for a specific company (known register type/number/court),
+   * then click on an element that triggers a file download.
+   */
+  public async getSpecificCompany(companyArg: BusinessRecord['data']['germanParsedRegistration']) {
+    return this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      const page = await this.getNewPage();
+      await this.navigateToPage(page, 'Normal search');
+      await page.waitForSelector('#form\\:schlagwoerter', { timeout: 5000 });
+
+      // 1) Type of Register (e.g. HRB, HRA, etc.)
+      await page.waitForSelector('#form\\:registerArt_label');
+      await page.click('#form\\:registerArt_label');
+      await page.waitForSelector('#form\\:registerArt_items');
+      await page.evaluate((type) => {
+        const options = Array.from(document.querySelectorAll('#form\\:registerArt_items li'));
+        const targetOption = options.find((option) => option.textContent?.trim() === type);
+        (targetOption as HTMLElement)?.click();
+      }, companyArg.type);
+
+      // 2) Register number
+      await page.waitForSelector('#form\\:registerNummer');
+      await page.type('#form\\:registerNummer', companyArg.number);
+
+      // 3) Register court
+      await page.waitForSelector('#form\\:registergericht_label');
+      await page.click('#form\\:registergericht_label');
+      await page.waitForSelector('#form\\:registergericht_items');
+      await page.evaluate((court) => {
+        const options = Array.from(document.querySelectorAll('#form\\:registergericht_items li'));
+        const targetOption = options.find((option) => option.textContent?.trim() === court);
+        (targetOption as HTMLElement)?.click();
+      }, companyArg.court);
+
+      // Click 'Find'
+      await this.clickFindButton(page);
+
+      // Optionally grab the results, just for logging
+      const businessRecords = await this.waitForResults(page);
+      console.log(businessRecords);
+
+      const files: plugins.smartfile.SmartFile[] = [];
+
+      // download files
+      files.push(await this.downloadFile(page, 'SI'));
+      files.push(await this.downloadFile(page, 'AD'));
+
+      // At this point, the file should have been downloaded automatically
+      // to the path specified by `Page.setDownloadBehavior`
+      await page.close();
+
+      return {
+        businessRecords,
+        files,
+      };
+    }, 60000);
+  }
+
+  /**
+   * get specific company by full name
+   */
+  public async getSpecificCompanyByName(companyNameArg: string) {
+    const businessRecords = await this.searchCompany(companyNameArg, 1);
+    const result = this.getSpecificCompany(businessRecords[0].germanParsedRegistration);
+    return result;
   }
 }

package/ts/classes.jsonldata.ts

@@ -2,16 +2,55 @@ import * as plugins from './plugins.js';
 import * as paths from './paths.js';
 import type { OpenData } from './classes.main.opendata.js';
 
-export class JsonlDataProcessor {
-  public openDataRef: OpenData;
-  constructor(openDataRefArg: OpenData) {
-    this.openDataRef = openDataRefArg;
+export type SeedEntryType = {
+  all_attributes: {
+    _registerArt: string;
+    _registerNummer: string;
+    additional_data: {
+      AD: boolean;
+      CD: boolean;
+      DK: boolean;
+      HD: boolean;
+      SI: boolean;
+      UT: boolean;
+      VÖ: boolean;
+    };
+    federal_state: string;
+    native_company_number: string;
+    registered_office: string;
+    registrar: string;
+  };
+  company_number: string;
+  current_status: string;
+  jurisdiction_code: string;
+  name: string;
+  officers: {
+    name: string;
+    other_attributes: {
+      city: string;
+      firstname: string;
+      flag: string;
+      lastname: string;
+    };
+    position: string;
+    start_date: string; // ISO 8601 date string
+    type: string;
+  }[];
+  registered_address: string;
+  retrieved_at: string; // ISO 8601 date string
+};
+
+export class JsonlDataProcessor<T> {
+  public forEachFunction: (entryArg: T) => Promise<void>;
+  constructor(forEachFunctionArg: typeof this.forEachFunction) {
+    this.forEachFunction = forEachFunctionArg;
   }
 
   // TODO: define a mapper as argument instead of hard-coding it
-  public async processDataFromUrl(dataUrlArg = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2') {
+  public async processDataFromUrl(
+    dataUrlArg = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2'
+  ) {
     const done = plugins.smartpromise.defer();
-    const promiseArray: Promise<any>[] = [];
     const dataExists = await plugins.smartfile.fs.isDirectory(paths.germanBusinessDataDir);
     if (!dataExists) {
       await plugins.smartfile.fs.ensureDir(paths.germanBusinessDataDir);
@@ -19,10 +58,6 @@ export class JsonlDataProcessor {
     }
 
     const smartarchive = await plugins.smartarchive.SmartArchive.fromArchiveUrl(dataUrlArg);
-    promiseArray
-      .push
-      // smartarchive.exportToFs(paths.germanBusinessDataDir, 'de_companies_ocdata.jsonl')
-      ();
     const jsonlDataStream = await smartarchive.exportToStreamOfStreamFiles();
     let totalRecordsCounter = 0;
     let nextRest: string = '';
@@ -39,42 +74,37 @@ export class JsonlDataProcessor {
                 const lines = currentString.split('\n');
                 nextRest = lines.pop();
                 console.log(`Got another ${lines.length} records.`);
-                for (const line of lines) {
-                  let entry: any;
-                  if (!line) continue;
-                  try {
-                    entry = JSON.parse(line);
-                  } catch (err) {
-                    console.log(line);
-                    await plugins.smartdelay.delayFor(10000);
-                  }
-                  if (!entry) continue;
-                  totalRecordsCounter++;
-                  if (totalRecordsCounter % 10000 === 0) console.log(`${totalRecordsCounter} total records.`);
-                  const businessRecord = new this.openDataRef.CBusinessRecord();
-                  businessRecord.id = await this.openDataRef.CBusinessRecord.getNewId();
-                  businessRecord.data.name = entry.name;
-                  await businessRecord.save();
-                }
+                const concurrentProcessor = new plugins.smartarray.ConcurrentProcessor<string>(
+                  async (line) => {
+                    let entry: T;
+                    if (!line) return;
+                    try {
+                      entry = JSON.parse(line);
+                    } catch (err) {
+                      console.log(line);
+                      await plugins.smartdelay.delayFor(10000);
+                    }
+                    if (!entry) return;
+                    totalRecordsCounter++;
+                    if (totalRecordsCounter % 10000 === 0)
+                      console.log(`${totalRecordsCounter} total records.`);
+                    await this.forEachFunction(entry);
+                  },
+                  1000
+                );
+                await concurrentProcessor.process(lines);
               },
               finalFunction: async (streamToolsArg) => {
                 console.log(`finished processing ${totalRecordsCounter} records.`);
                 if (!nextRest) return;
                 JSON.parse(nextRest);
-              }
+                done.resolve();
+              },
             })
           );
         },
       })
     );
-  }
-
-  public async getBusinessRecordByName(nameArg: string) {
-    const businessRecord = await this.openDataRef.CBusinessRecord.getInstance({
-      data: {
-        name: { $regex: `${nameArg}`, $options: "i" } as any,
-      }
-    });
-    return businessRecord;
+    await done.promise;
   }
 }

package/ts/classes.main.opendata.ts

@@ -1,32 +1,61 @@
 import { BusinessRecord } from './classes.businessrecord.js';
-import  { HandelsRegister } from './classes.handelsregister.js';
-import { JsonlDataProcessor } from './classes.jsonldata.js';
+import { HandelsRegister } from './classes.handelsregister.js';
+import { JsonlDataProcessor, type SeedEntryType } from './classes.jsonldata.js';
 import * as paths from './paths.js';
 import * as plugins from './plugins.js';
 
 export class OpenData {
   public db: plugins.smartdata.SmartdataDb;
   private serviceQenv = new plugins.qenv.Qenv(paths.packageDir, paths.nogitDir);
-  
-  public jsonLDataProcessor: JsonlDataProcessor;
+
+  public jsonLDataProcessor: JsonlDataProcessor<SeedEntryType>;
   public handelsregister: HandelsRegister;
-  
+
   public CBusinessRecord = plugins.smartdata.setDefaultManagerForDoc(this, BusinessRecord);
 
   public async start() {
     this.db = new plugins.smartdata.SmartdataDb({
-        mongoDbUrl: await this.serviceQenv.getEnvVarOnDemand('MONGODB_URL'),
-        mongoDbName: await this.serviceQenv.getEnvVarOnDemand('MONGODB_NAME'),
-        mongoDbUser: await this.serviceQenv.getEnvVarOnDemand('MONGODB_USER'),
-        mongoDbPass: await this.serviceQenv.getEnvVarOnDemand('MONGODB_PASS'),
+      mongoDbUrl: await this.serviceQenv.getEnvVarOnDemand('MONGODB_URL'),
+      mongoDbName: await this.serviceQenv.getEnvVarOnDemand('MONGODB_NAME'),
+      mongoDbUser: await this.serviceQenv.getEnvVarOnDemand('MONGODB_USER'),
+      mongoDbPass: await this.serviceQenv.getEnvVarOnDemand('MONGODB_PASS'),
     });
     await this.db.init();
-    this.jsonLDataProcessor = new JsonlDataProcessor(this);
+    this.jsonLDataProcessor = new JsonlDataProcessor(async (entryArg) => {
+      const businessRecord = new this.CBusinessRecord();
+      businessRecord.id = await this.CBusinessRecord.getNewId();
+      businessRecord.data.name = entryArg.name;
+      businessRecord.data.germanParsedRegistration = {
+        court: entryArg.all_attributes.registered_office,
+        number: entryArg.all_attributes._registerNummer,
+        type: entryArg.all_attributes._registerArt as 'HRA' | 'HRB',
+      };
+      await businessRecord.save();
+    });
     this.handelsregister = new HandelsRegister(this);
     await this.handelsregister.start();
   }
+
+  public async buildInitialDb() {
+    await this.jsonLDataProcessor.processDataFromUrl();
+  }
+
+  public async slowValidateDb() {
+
+  }
+
+  public async validateSearchByName() {
+
+  }
+
+  public async searchDbByBusinessNameAndPostalCode(businessNameArg: string, postalCodeArg: string) {
+
+  }
+
+
+
   public async stop() {
     await this.db.close();
     await this.handelsregister.stop();
   }
-}
+}

package/ts/paths.ts

@@ -8,4 +8,8 @@ export const packageDir = plugins.path.join(
 export const nogitDir = plugins.path.join(packageDir, './.nogit/');
 plugins.smartfile.fs.ensureDirSync(nogitDir);
 
+export const downloadDir = plugins.path.join(nogitDir, 'downloads');
+plugins.smartfile.fs.ensureDirSync(downloadDir);
+
+
 export const germanBusinessDataDir = plugins.path.join(nogitDir, 'germanbusinessdata');

package/ts/plugins.ts

@@ -6,8 +6,10 @@ export {
 }
 
 // @push.rocks scope
+import * as lik from '@push.rocks/lik';
 import * as qenv from '@push.rocks/qenv';
 import * as smartarchive from '@push.rocks/smartarchive';
+import * as smartarray from '@push.rocks/smartarray';
 import * as smartbrowser from '@push.rocks/smartbrowser';
 import * as smartdata from '@push.rocks/smartdata';
 import * as smartdelay from '@push.rocks/smartdelay';
@@ -16,10 +18,14 @@ import * as smartpath from '@push.rocks/smartpath';
 import * as smartpromise from '@push.rocks/smartpromise';
 import * as smartrequest from '@push.rocks/smartrequest';
 import * as smartstream from '@push.rocks/smartstream';
+import * as smartunique from '@push.rocks/smartunique';
+import * as smartxml from '@push.rocks/smartxml'; 
 
 export {
+  lik,
   qenv,
   smartarchive,
+  smartarray,
   smartbrowser,
   smartdata,
   smartdelay,
@@ -28,4 +34,13 @@ export {
   smartpromise,
   smartrequest,
   smartstream,
-}
+  smartunique,
+  smartxml,
+}
+
+// @tsclass scope
+import * as tsclass from '@tsclass/tsclass';
+
+export {
+  tsclass,
+}